Skip to content

Commit e543707

Browse files
rtomaykortomayko
committed
Split ES backup scripts into v0.9 and v1.x versions
1 parent 40c1e15 commit e543707

3 files changed

Lines changed: 143 additions & 71 deletions

File tree

‎libexec/ghe-backup-es-rsync‎

Lines changed: 8 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -10,76 +10,13 @@ set -e
1010
cd $(dirname "$0")/..
1111
. libexec/ghe-backup-config
1212

13-
# Set up remote host and root elastic backup directory based on config
14-
host="$GHE_HOSTNAME"
15-
1613
# Perform a host-check and establish GHE_REMOTE_XXX variables.
17-
ghe_remote_version_required "$host"
18-
19-
# Verify rsync is available.
20-
if ! rsync --version 1>/dev/null 2>&1; then
21-
echo "Error: rsync not found." 1>&2
22-
exit 1
23-
fi
24-
25-
# Make sure root backup dir exists if this is the first run
26-
mkdir -p "$GHE_SNAPSHOT_DIR/elasticsearch"
27-
28-
# Verify that the /data/elasticsearch directory exists.
29-
if ! ghe-ssh "$host" -- "[ -d '$GHE_REMOTE_DATA_USER_DIR/elasticsearch' ]"; then
30-
echo "* The '$GHE_REMOTE_DATA_USER_DIR/elasticsearch' directory doesn't exist." 1>&3
31-
exit 0
32-
fi
33-
34-
# Grab the elasticsearch.yml file which is root owned and mode -rw------- so
35-
# can't be read via rsync or cat. We use the root allowed grep -F as a cat
36-
# replacement.
37-
# XXX This should be fixed on the appliance side but will need to be maintained
38-
# for versions up to at least 11.10.343.
39-
echo "* Retrieving elasticsearch.yml config file ..." 1>&3
40-
ghe-ssh "$host" -- "sudo grep -F '' '$GHE_REMOTE_DATA_USER_DIR/elasticsearch/elasticsearch.yml'" \
41-
> "$GHE_SNAPSHOT_DIR/elasticsearch/elasticsearch.yml"
42-
chmod 0600 "$GHE_SNAPSHOT_DIR/elasticsearch/elasticsearch.yml"
43-
44-
# If we have a previous increment, avoid transferring existing files via rsync's
45-
# --link-dest support. This also decreases physical space usage considerably.
46-
if [ -d "$GHE_DATA_DIR/current/elasticsearch" ]; then
47-
link_dest="--link-dest=../../current/elasticsearch"
14+
ghe_remote_version_required "$GHE_HOSTNAME"
15+
16+
# Determine which version of ES we're backing up based on appliance version and
17+
# run the appropriate command.
18+
if [ "$GHE_VERSION_MAJOR" -eq 1 ]; then
19+
exec ghe-backup-es-v0.9-rsync "$@"
20+
else
21+
exec ghe-backup-es-v1.x-rsync "$@"
4822
fi
49-
50-
# Transfer ES indices from a GitHub instance to the current snapshot
51-
# directory, using a previous snapshot to avoid transferring files that have
52-
# already been transferred.
53-
echo "* Performing initial sync of ES indices ..." 1>&3
54-
ghe-rsync -avz \
55-
-e "ghe-ssh -p $(ssh_port_part "$host")" \
56-
--rsync-path='sudo -u git rsync' \
57-
$link_dest \
58-
--exclude='elasticsearch.yml' \
59-
"$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch/" \
60-
"$GHE_SNAPSHOT_DIR/elasticsearch" 1>&3
61-
62-
# Set up a trap to re-enable flushing on exit
63-
cleanup () {
64-
echo "* Enabling ES index flushing ..." 1>&3
65-
echo '{"index":{"translog.disable_flush":false}}' |
66-
ghe-ssh "$host" -- curl -s -XPUT "localhost:9200/_settings" -d @- >/dev/null
67-
}
68-
trap 'cleanup' EXIT
69-
trap 'exit $?' INT # ^C always terminate
70-
71-
# Disable ES flushing and force a flush right now
72-
echo "* Disabling ES index flushing ..." 1>&3
73-
echo '{"index":{"translog.disable_flush":true}}' |
74-
ghe-ssh "$host" -- curl -s -XPUT "localhost:9200/_settings" -d @- >/dev/null
75-
ghe-ssh "$host" -- curl -s -XPOST "localhost:9200/_flush" >/dev/null
76-
77-
# Transfer all ES indices again
78-
echo "* Performing follow-up sync of ES indices ..." 1>&3
79-
ghe-rsync -avz \
80-
-e "ghe-ssh -p $(ssh_port_part "$host")" \
81-
--rsync-path='sudo -u git rsync' \
82-
$link_dest \
83-
--exclude='elasticsearch.yml' \
84-
"$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch/" \
85-
"$GHE_SNAPSHOT_DIR/elasticsearch" 1>&3

‎libexec/ghe-backup-es-v0.9-rsync‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
#!/bin/sh
2+
#/ Usage: ghe-backup-es-v0.9-rsync
3+
#/ Take an online, incremental snapshot of Elasticsearch v0.9 indices. This
4+
#/ command is used only against GitHub Enterprise v11.10.x appliances.
5+
#/
6+
#/ Note: This command typically isn't called directly. It's invoked by
7+
#/ ghe-backup when the rsync strategy is used.
8+
set -e
9+
10+
# Bring in the backup configuration
11+
cd $(dirname "$0")/..
12+
. libexec/ghe-backup-config
13+
14+
# Set up remote host and root elastic backup directory based on config
15+
host="$GHE_HOSTNAME"
16+
17+
# Perform a host-check and establish GHE_REMOTE_XXX variables.
18+
ghe_remote_version_required "$host"
19+
20+
# Verify rsync is available.
21+
if ! rsync --version 1>/dev/null 2>&1; then
22+
echo "Error: rsync not found." 1>&2
23+
exit 1
24+
fi
25+
26+
# Make sure root backup dir exists if this is the first run
27+
mkdir -p "$GHE_SNAPSHOT_DIR/elasticsearch"
28+
29+
# Verify that the /data/elasticsearch directory exists.
30+
if ! ghe-ssh "$host" -- "[ -d '$GHE_REMOTE_DATA_USER_DIR/elasticsearch' ]"; then
31+
echo "* The '$GHE_REMOTE_DATA_USER_DIR/elasticsearch' directory doesn't exist." 1>&3
32+
exit 0
33+
fi
34+
35+
# Grab the elasticsearch.yml file which is root owned and mode -rw------- so
36+
# can't be read via rsync or cat. We use the root allowed grep -F as a cat
37+
# replacement.
38+
# XXX This should be fixed on the appliance side but will need to be maintained
39+
# for versions up to at least 11.10.343.
40+
echo "* Retrieving elasticsearch.yml config file ..." 1>&3
41+
ghe-ssh "$host" -- "sudo grep -F '' '$GHE_REMOTE_DATA_USER_DIR/elasticsearch/elasticsearch.yml'" \
42+
> "$GHE_SNAPSHOT_DIR/elasticsearch/elasticsearch.yml"
43+
chmod 0600 "$GHE_SNAPSHOT_DIR/elasticsearch/elasticsearch.yml"
44+
45+
# If we have a previous increment, avoid transferring existing files via rsync's
46+
# --link-dest support. This also decreases physical space usage considerably.
47+
if [ -d "$GHE_DATA_DIR/current/elasticsearch" ]; then
48+
link_dest="--link-dest=../../current/elasticsearch"
49+
fi
50+
51+
# Transfer ES indices from a GitHub instance to the current snapshot
52+
# directory, using a previous snapshot to avoid transferring files that have
53+
# already been transferred.
54+
echo "* Performing initial sync of ES indices ..." 1>&3
55+
ghe-rsync -avz \
56+
-e "ghe-ssh -p $(ssh_port_part "$host")" \
57+
--rsync-path='sudo -u git rsync' \
58+
$link_dest \
59+
--exclude='elasticsearch.yml' \
60+
"$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch/" \
61+
"$GHE_SNAPSHOT_DIR/elasticsearch" 1>&3
62+
63+
# Set up a trap to re-enable flushing on exit
64+
cleanup () {
65+
echo "* Enabling ES index flushing ..." 1>&3
66+
echo '{"index":{"translog.disable_flush":false}}' |
67+
ghe-ssh "$host" -- curl -s -XPUT "localhost:9200/_settings" -d @- >/dev/null
68+
}
69+
trap 'cleanup' EXIT
70+
trap 'exit $?' INT # ^C always terminate
71+
72+
# Disable ES flushing and force a flush right now
73+
echo "* Disabling ES index flushing ..." 1>&3
74+
echo '{"index":{"translog.disable_flush":true}}' |
75+
ghe-ssh "$host" -- curl -s -XPUT "localhost:9200/_settings" -d @- >/dev/null
76+
ghe-ssh "$host" -- curl -s -XPOST "localhost:9200/_flush" >/dev/null
77+
78+
# Transfer all ES indices again
79+
echo "* Performing follow-up sync of ES indices ..." 1>&3
80+
ghe-rsync -avz \
81+
-e "ghe-ssh -p $(ssh_port_part "$host")" \
82+
--rsync-path='sudo -u git rsync' \
83+
$link_dest \
84+
--exclude='elasticsearch.yml' \
85+
"$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch/" \
86+
"$GHE_SNAPSHOT_DIR/elasticsearch" 1>&3

‎libexec/ghe-backup-es-v1.x-rsync‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
#!/bin/sh
2+
#/ Usage: ghe-backup-es-v1.x-rsync
3+
#/ Take an online, incremental snapshot of Elasticsearch v1.x indices. This
4+
#/ command is used only against GitHub Enterprise v2.x appliances or greater.
5+
#/
6+
#/ Note: This command typically isn't called directly. It's invoked by
7+
#/ ghe-backup when the rsync strategy is used.
8+
set -e
9+
10+
# Bring in the backup configuration
11+
cd $(dirname "$0")/..
12+
. libexec/ghe-backup-config
13+
14+
# Set up remote host and root elastic backup directory based on config
15+
host="$GHE_HOSTNAME"
16+
17+
# Perform a host-check and establish GHE_REMOTE_XXX variables.
18+
ghe_remote_version_required "$host"
19+
20+
# Verify rsync is available.
21+
if ! rsync --version 1>/dev/null 2>&1; then
22+
echo "Error: rsync not found." 1>&2
23+
exit 1
24+
fi
25+
26+
# Make sure root backup dir exists if this is the first run
27+
mkdir -p "$GHE_SNAPSHOT_DIR/elasticsearch"
28+
29+
# Take a local ES snapshot on the appliance. This writes a new snapshot to
30+
# a /data/user/elasticsearch-snapshots directory. No data is transferred from
31+
# the appliance to the VM at this stage.
32+
ghe-ssh "$host" -- "ghe-es-snapshot" 1>&3
33+
34+
# If we have a previous increment, avoid transferring existing files via rsync's
35+
# --link-dest support. This also decreases physical space usage considerably.
36+
if [ -d "$GHE_DATA_DIR/current/elasticsearch" ]; then
37+
link_dest="--link-dest=../../current/elasticsearch"
38+
fi
39+
40+
# Transfer the newly created ES snapshot data from the GitHub appliance to the
41+
# current snapshot directory, using a previous snapshot to avoid transferring files
42+
# that have already been transferred.
43+
ghe-rsync -avz \
44+
-e "ghe-ssh -p $(ssh_port_part "$host")" \
45+
--rsync-path='sudo -u elasticsearch rsync' \
46+
$link_dest \
47+
--exclude='elasticsearch.yml' \
48+
"$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch-snapshots/" \
49+
"$GHE_SNAPSHOT_DIR/elasticsearch" 1>&3

0 commit comments

Comments
 (0)