Skip to content

Commit 8138258

Browse files
rtomaykortomayko
committed
Merge pull request #4 from github/es-snapshots
Support ES v1.x snapshot backup and restore
2 parents 40c1e15 + a0e90ef commit 8138258

7 files changed

Lines changed: 194 additions & 84 deletions

File tree

‎bin/ghe-backup‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -122,13 +122,9 @@ echo "Backing up GitHub Pages ..."
122122
ghe-backup-pages-${GHE_BACKUP_STRATEGY} ||
123123
failures="$failures pages"
124124

125-
if [ "$GHE_VERSION_MAJOR" -lt 2 ]; then
126-
echo "Backing up Elasticsearch indices ..."
127-
ghe-backup-es-${GHE_BACKUP_STRATEGY} ||
128-
failures="$failures elasticsearch"
129-
else
130-
echo "Skipping Elasticsearch backup under v2.x (not yet supported)"
131-
fi
125+
echo "Backing up Elasticsearch indices ..."
126+
ghe-backup-es-${GHE_BACKUP_STRATEGY} ||
127+
failures="$failures elasticsearch"
132128

133129
# If we're using the tarball backup strategy, bring the appliance out of
134130
# maintenance mode now instead of waiting until after pruning stale snapshots.

‎libexec/ghe-backup-es-rsync‎

Lines changed: 8 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -10,76 +10,13 @@ set -e
1010
cd $(dirname "$0")/..
1111
. libexec/ghe-backup-config
1212

13-
# Set up remote host and root elastic backup directory based on config
14-
host="$GHE_HOSTNAME"
15-
1613
# Perform a host-check and establish GHE_REMOTE_XXX variables.
17-
ghe_remote_version_required "$host"
18-
19-
# Verify rsync is available.
20-
if ! rsync --version 1>/dev/null 2>&1; then
21-
echo "Error: rsync not found." 1>&2
22-
exit 1
23-
fi
24-
25-
# Make sure root backup dir exists if this is the first run
26-
mkdir -p "$GHE_SNAPSHOT_DIR/elasticsearch"
27-
28-
# Verify that the /data/elasticsearch directory exists.
29-
if ! ghe-ssh "$host" -- "[ -d '$GHE_REMOTE_DATA_USER_DIR/elasticsearch' ]"; then
30-
echo "* The '$GHE_REMOTE_DATA_USER_DIR/elasticsearch' directory doesn't exist." 1>&3
31-
exit 0
32-
fi
33-
34-
# Grab the elasticsearch.yml file which is root owned and mode -rw------- so
35-
# can't be read via rsync or cat. We use the root allowed grep -F as a cat
36-
# replacement.
37-
# XXX This should be fixed on the appliance side but will need to be maintained
38-
# for versions up to at least 11.10.343.
39-
echo "* Retrieving elasticsearch.yml config file ..." 1>&3
40-
ghe-ssh "$host" -- "sudo grep -F '' '$GHE_REMOTE_DATA_USER_DIR/elasticsearch/elasticsearch.yml'" \
41-
> "$GHE_SNAPSHOT_DIR/elasticsearch/elasticsearch.yml"
42-
chmod 0600 "$GHE_SNAPSHOT_DIR/elasticsearch/elasticsearch.yml"
43-
44-
# If we have a previous increment, avoid transferring existing files via rsync's
45-
# --link-dest support. This also decreases physical space usage considerably.
46-
if [ -d "$GHE_DATA_DIR/current/elasticsearch" ]; then
47-
link_dest="--link-dest=../../current/elasticsearch"
14+
ghe_remote_version_required "$GHE_HOSTNAME"
15+
16+
# Determine which version of ES we're backing up based on appliance version and
17+
# run the appropriate command.
18+
if [ "$GHE_VERSION_MAJOR" -eq 1 ]; then
19+
exec ghe-backup-es-v0.9-rsync "$@"
20+
else
21+
exec ghe-backup-es-v1.x-rsync "$@"
4822
fi
49-
50-
# Transfer ES indices from a GitHub instance to the current snapshot
51-
# directory, using a previous snapshot to avoid transferring files that have
52-
# already been transferred.
53-
echo "* Performing initial sync of ES indices ..." 1>&3
54-
ghe-rsync -avz \
55-
-e "ghe-ssh -p $(ssh_port_part "$host")" \
56-
--rsync-path='sudo -u git rsync' \
57-
$link_dest \
58-
--exclude='elasticsearch.yml' \
59-
"$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch/" \
60-
"$GHE_SNAPSHOT_DIR/elasticsearch" 1>&3
61-
62-
# Set up a trap to re-enable flushing on exit
63-
cleanup () {
64-
echo "* Enabling ES index flushing ..." 1>&3
65-
echo '{"index":{"translog.disable_flush":false}}' |
66-
ghe-ssh "$host" -- curl -s -XPUT "localhost:9200/_settings" -d @- >/dev/null
67-
}
68-
trap 'cleanup' EXIT
69-
trap 'exit $?' INT # ^C always terminate
70-
71-
# Disable ES flushing and force a flush right now
72-
echo "* Disabling ES index flushing ..." 1>&3
73-
echo '{"index":{"translog.disable_flush":true}}' |
74-
ghe-ssh "$host" -- curl -s -XPUT "localhost:9200/_settings" -d @- >/dev/null
75-
ghe-ssh "$host" -- curl -s -XPOST "localhost:9200/_flush" >/dev/null
76-
77-
# Transfer all ES indices again
78-
echo "* Performing follow-up sync of ES indices ..." 1>&3
79-
ghe-rsync -avz \
80-
-e "ghe-ssh -p $(ssh_port_part "$host")" \
81-
--rsync-path='sudo -u git rsync' \
82-
$link_dest \
83-
--exclude='elasticsearch.yml' \
84-
"$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch/" \
85-
"$GHE_SNAPSHOT_DIR/elasticsearch" 1>&3

‎libexec/ghe-backup-es-v0.9-rsync‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
#!/bin/sh
2+
#/ Usage: ghe-backup-es-v0.9-rsync
3+
#/ Take an online, incremental snapshot of Elasticsearch v0.9 indices. This
4+
#/ command is used only against GitHub Enterprise v11.10.x appliances.
5+
#/
6+
#/ Note: This command typically isn't called directly. It's invoked by
7+
#/ ghe-backup when the rsync strategy is used.
8+
set -e
9+
10+
# Bring in the backup configuration
11+
cd $(dirname "$0")/..
12+
. libexec/ghe-backup-config
13+
14+
# Set up remote host and root elastic backup directory based on config
15+
host="$GHE_HOSTNAME"
16+
17+
# Perform a host-check and establish GHE_REMOTE_XXX variables.
18+
ghe_remote_version_required "$host"
19+
20+
# Verify rsync is available.
21+
if ! rsync --version 1>/dev/null 2>&1; then
22+
echo "Error: rsync not found." 1>&2
23+
exit 1
24+
fi
25+
26+
# Make sure root backup dir exists if this is the first run
27+
mkdir -p "$GHE_SNAPSHOT_DIR/elasticsearch"
28+
29+
# Verify that the /data/elasticsearch directory exists.
30+
if ! ghe-ssh "$host" -- "[ -d '$GHE_REMOTE_DATA_USER_DIR/elasticsearch' ]"; then
31+
echo "* The '$GHE_REMOTE_DATA_USER_DIR/elasticsearch' directory doesn't exist." 1>&3
32+
exit 0
33+
fi
34+
35+
# Grab the elasticsearch.yml file which is root owned and mode -rw------- so
36+
# can't be read via rsync or cat. We use the root allowed grep -F as a cat
37+
# replacement.
38+
# XXX This should be fixed on the appliance side but will need to be maintained
39+
# for versions up to at least 11.10.343.
40+
echo "* Retrieving elasticsearch.yml config file ..." 1>&3
41+
ghe-ssh "$host" -- "sudo grep -F '' '$GHE_REMOTE_DATA_USER_DIR/elasticsearch/elasticsearch.yml'" \
42+
> "$GHE_SNAPSHOT_DIR/elasticsearch/elasticsearch.yml"
43+
chmod 0600 "$GHE_SNAPSHOT_DIR/elasticsearch/elasticsearch.yml"
44+
45+
# If we have a previous increment, avoid transferring existing files via rsync's
46+
# --link-dest support. This also decreases physical space usage considerably.
47+
if [ -d "$GHE_DATA_DIR/current/elasticsearch" ]; then
48+
link_dest="--link-dest=../../current/elasticsearch"
49+
fi
50+
51+
# Transfer ES indices from a GitHub instance to the current snapshot
52+
# directory, using a previous snapshot to avoid transferring files that have
53+
# already been transferred.
54+
echo "* Performing initial sync of ES indices ..." 1>&3
55+
ghe-rsync -avz \
56+
-e "ghe-ssh -p $(ssh_port_part "$host")" \
57+
--rsync-path='sudo -u git rsync' \
58+
$link_dest \
59+
--exclude='elasticsearch.yml' \
60+
"$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch/" \
61+
"$GHE_SNAPSHOT_DIR/elasticsearch" 1>&3
62+
63+
# Set up a trap to re-enable flushing on exit
64+
cleanup () {
65+
echo "* Enabling ES index flushing ..." 1>&3
66+
echo '{"index":{"translog.disable_flush":false}}' |
67+
ghe-ssh "$host" -- curl -s -XPUT "localhost:9200/_settings" -d @- >/dev/null
68+
}
69+
trap 'cleanup' EXIT
70+
trap 'exit $?' INT # ^C always terminate
71+
72+
# Disable ES flushing and force a flush right now
73+
echo "* Disabling ES index flushing ..." 1>&3
74+
echo '{"index":{"translog.disable_flush":true}}' |
75+
ghe-ssh "$host" -- curl -s -XPUT "localhost:9200/_settings" -d @- >/dev/null
76+
ghe-ssh "$host" -- curl -s -XPOST "localhost:9200/_flush" >/dev/null
77+
78+
# Transfer all ES indices again
79+
echo "* Performing follow-up sync of ES indices ..." 1>&3
80+
ghe-rsync -avz \
81+
-e "ghe-ssh -p $(ssh_port_part "$host")" \
82+
--rsync-path='sudo -u git rsync' \
83+
$link_dest \
84+
--exclude='elasticsearch.yml' \
85+
"$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch/" \
86+
"$GHE_SNAPSHOT_DIR/elasticsearch" 1>&3

‎libexec/ghe-backup-es-v1.x-rsync‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
#!/bin/sh
2+
#/ Usage: ghe-backup-es-v1.x-rsync
3+
#/ Take an online, incremental snapshot of Elasticsearch v1.x indices. This
4+
#/ command is used only against GitHub Enterprise v2.x appliances or greater.
5+
#/
6+
#/ Note: This command typically isn't called directly. It's invoked by
7+
#/ ghe-backup when the rsync strategy is used.
8+
set -e
9+
10+
# Bring in the backup configuration
11+
cd $(dirname "$0")/..
12+
. libexec/ghe-backup-config
13+
14+
# Set up remote host and root elastic backup directory based on config
15+
host="$GHE_HOSTNAME"
16+
17+
# Perform a host-check and establish GHE_REMOTE_XXX variables.
18+
ghe_remote_version_required "$host"
19+
20+
# Verify rsync is available.
21+
if ! rsync --version 1>/dev/null 2>&1; then
22+
echo "Error: rsync not found." 1>&2
23+
exit 1
24+
fi
25+
26+
# Make sure root backup dir exists if this is the first run
27+
mkdir -p "$GHE_SNAPSHOT_DIR/elasticsearch"
28+
29+
# Take a local ES snapshot on the appliance. This writes a new snapshot to
30+
# a /data/user/elasticsearch-snapshots directory. No data is transferred from
31+
# the appliance to the VM at this stage.
32+
ghe-ssh "$host" -- "ghe-es-snapshot" 1>&3
33+
34+
# If we have a previous increment, avoid transferring existing files via rsync's
35+
# --link-dest support. This also decreases physical space usage considerably.
36+
if [ -d "$GHE_DATA_DIR/current/elasticsearch" ]; then
37+
link_dest="--link-dest=../../current/elasticsearch"
38+
fi
39+
40+
# Transfer the newly created ES snapshot data from the GitHub appliance to the
41+
# current snapshot directory, using a previous snapshot to avoid transferring files
42+
# that have already been transferred.
43+
ghe-rsync -avz \
44+
-e "ghe-ssh -p $(ssh_port_part "$host")" \
45+
--rsync-path='sudo -u elasticsearch rsync' \
46+
$link_dest \
47+
--exclude='elasticsearch.yml' \
48+
"$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch-snapshots/" \
49+
"$GHE_SNAPSHOT_DIR/elasticsearch" 1>&3

‎libexec/ghe-restore-es-rsync‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,22 +23,42 @@ ghe_remote_version_required "$host"
2323
# us run this script directly.
2424
: ${GHE_RESTORE_SNAPSHOT:=current}
2525

26+
# The directory holding the snapshot to restore
27+
snapshot_dir="$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT"
28+
2629
# Use GNU tar on BSDs.
2730
TAR=tar
2831
if ! tar --version | grep -q GNU; then
2932
TAR=gtar
3033
fi
3134

3235
# Transfer all ES data from the latest snapshot to the GitHub instance.
33-
if [ ! -d "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/elasticsearch" ]; then
36+
if [ ! -d "$snapshot_dir/elasticsearch" ]; then
3437
echo "Warning: Elasticsearch backup missing. Skipping ..."
3538
exit 0
36-
elif [ "$GHE_VERSION_MAJOR" -gt 1 ]; then
39+
40+
# restoring v11.10.x ES snapshot into a v2.0 appliance
41+
elif [ "$GHE_VERSION_MAJOR" -gt 1 -a -f "$snapshot_dir/elasticsearch/elasticsearch.yml" ]; then
3742
cd "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/elasticsearch"
3843
ghe-ssh "$host" -- "sudo mkdir -p '$GHE_REMOTE_DATA_USER_DIR/elasticsearch-legacy'" 1>&3
3944

4045
$TAR -cf - --owner=root --group=root . |
4146
ghe-ssh "$host" -- "sudo tar -xf - -C '$GHE_REMOTE_DATA_USER_DIR/elasticsearch-legacy'" 1>&3
47+
48+
# restoring v2.0 ES snapshot into a v2.0 appliance
49+
elif [ "$GHE_VERSION_MAJOR" -gt 1 ]; then
50+
ghe-ssh "$host" -- "sudo mkdir -p '$GHE_REMOTE_DATA_USER_DIR/elasticsearch-snapshots'" 1>&3
51+
ghe-ssh "$host" -- "sudo chown elasticsearch:elasticsearch '$GHE_REMOTE_DATA_USER_DIR/elasticsearch-snapshots'" 1>&3
52+
53+
ghe-rsync -avz --delete \
54+
-e "ghe-ssh -p $(ssh_port_part "$host")" \
55+
--rsync-path="sudo -u elasticsearch rsync" \
56+
"$snapshot_dir/elasticsearch/" \
57+
"$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch-snapshots" 1>&3
58+
59+
ghe-ssh "$host" -- "ghe-es-restore"
60+
61+
# restoring v11.10.x ES snapshot into a v11.10.x appliance
4262
else
4363
cd "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT"
4464
$TAR -cf - --owner=root --group=root elasticsearch |

‎test/bin/ghe-es-snapshot‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/bin/sh
2+
# Usage: ghe-es-snapshot
3+
# Emulates the remote GitHub ghe-service-ensure-mysql command. Tests use this
4+
# to assert that the command was executed.
5+
set -e
6+
echo "ghe-es-snapshot OK"

‎test/test-ghe-backup.sh‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,16 @@ mkdir -p gh-enterprise-es/node/0
2121
touch gh-enterprise-es/node/0/stuff1
2222
touch gh-enterprise-es/node/0/stuff2
2323

24+
# Create some fake elasticsearch data in the remote snapshot data directory
25+
if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
26+
mkdir -p "$GHE_REMOTE_DATA_USER_DIR/elasticsearch-snapshots"
27+
cd "$GHE_REMOTE_DATA_USER_DIR/elasticsearch-snapshots"
28+
echo "fake snapshot file" > "snapshot-1"
29+
echo "fake metadata file" > "metadata-1"
30+
mkdir -p indices/repositories
31+
echo "fake document data" > indices/repositories/dumb-file
32+
fi
33+
2434
# Create some test repositories in the remote repositories dir
2535
mkdir "$GHE_REMOTE_DATA_USER_DIR/repositories"
2636
cd "$GHE_REMOTE_DATA_USER_DIR/repositories"
@@ -84,10 +94,13 @@ begin_test "ghe-backup first snapshot"
8494
# verify all pages data was transferred
8595
diff -ru "$GHE_REMOTE_DATA_USER_DIR/pages" "$GHE_DATA_DIR/current/pages"
8696

87-
# TODO ES backup not yet supported under 2.x VM
97+
# ES backup path is different under v11.10.x and v2.x appliances
8898
if [ "$GHE_VERSION_MAJOR" -eq 1 ]; then
89-
# verify all ES data was transferred
99+
# verify all ES data was transferred from live directory
90100
diff -ru "$GHE_REMOTE_DATA_USER_DIR/elasticsearch" "$GHE_DATA_DIR/current/elasticsearch"
101+
elif [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
102+
# verify all ES data was transferred from snapshot directory
103+
diff -ru "$GHE_REMOTE_DATA_USER_DIR/elasticsearch-snapshots" "$GHE_DATA_DIR/current/elasticsearch"
91104
fi
92105
)
93106
end_test
@@ -146,10 +159,13 @@ begin_test "ghe-backup subsequent snapshot"
146159
# verify all pages data was transferred
147160
diff -ru "$GHE_REMOTE_DATA_USER_DIR/pages" "$GHE_DATA_DIR/current/pages"
148161

149-
# TODO ES backup not yet supported under 2.x VM
162+
# ES backup path is different under v11.10.x and v2.x appliances
150163
if [ "$GHE_VERSION_MAJOR" -eq 1 ]; then
151-
# verify all ES data was transferred
164+
# verify all ES data was transferred from live directory
152165
diff -ru "$GHE_REMOTE_DATA_USER_DIR/elasticsearch" "$GHE_DATA_DIR/current/elasticsearch"
166+
elif [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
167+
# verify all ES data was transferred from snapshot directory
168+
diff -ru "$GHE_REMOTE_DATA_USER_DIR/elasticsearch-snapshots" "$GHE_DATA_DIR/current/elasticsearch"
153169
fi
154170
)
155171
end_test

0 commit comments

Comments
 (0)