Skip to content

Commit 464551c

Browse files
rtomaykortomayko
committed
Merge pull request #5 from github/special-user-data-dirs
Support for backing up alambic assets and hookshot deliveries on v2.0 appliances
2 parents 8d2c79f + 2004e24 commit 464551c

9 files changed

Lines changed: 189 additions & 50 deletions

‎bin/ghe-backup‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,16 @@ echo "Backing up GitHub Pages ..."
138138
ghe-backup-pages-${GHE_BACKUP_STRATEGY} ||
139139
failures="$failures pages"
140140

141+
if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
142+
echo "Backing up asset attachments ..."
143+
ghe-backup-userdata alambic_assets ||
144+
failures="$failures alambic_assets"
145+
146+
echo "Backing up hook deliveries ..."
147+
ghe-backup-userdata hookshot ||
148+
failures="$failures hookshot"
149+
fi
150+
141151
echo "Backing up Elasticsearch indices ..."
142152
ghe-backup-es-${GHE_BACKUP_STRATEGY} ||
143153
failures="$failures elasticsearch"

‎bin/ghe-restore‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,14 @@ ghe-restore-repositories-${GHE_BACKUP_STRATEGY} "$host" 1>&3
116116
echo "Restoring GitHub Pages ..."
117117
ghe-restore-pages-${GHE_BACKUP_STRATEGY} "$host" 1>&3
118118

119+
if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
120+
echo "Restoring asset attachments ..."
121+
ghe-restore-userdata alambic_assets "$host" 1>&3
122+
123+
echo "Restoring hook deliveries ..."
124+
ghe-restore-userdata hookshot "$host" 1>&3
125+
fi
126+
119127
echo "Restoring MySQL database ..."
120128
gzip -dc "$GHE_RESTORE_SNAPSHOT_PATH/mysql.sql.gz" | ghe-ssh "$host" -- 'ghe-import-mysql' 1>&3
121129

‎libexec/ghe-backup-pages-rsync‎

Lines changed: 2 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -10,34 +10,5 @@ set -e
1010
cd $(dirname "$0")/..
1111
. libexec/ghe-backup-config
1212

13-
# Verify rsync is available.
14-
if ! rsync --version 1>/dev/null 2>&1; then
15-
echo "Error: rsync not found." 1>&2
16-
exit 1
17-
fi
18-
19-
# Grab the host
20-
host="$GHE_HOSTNAME"
21-
22-
# Perform a host-check and establish GHE_REMOTE_XXX variables.
23-
ghe_remote_version_required "$host"
24-
25-
# Verify that the /data/pages directory exists. It won't if no pages sites have
26-
# been published under some versions.
27-
ghe-ssh "$host" -- "[ -d '$GHE_REMOTE_DATA_USER_DIR/pages' ]" || exit 0
28-
29-
# If we have a previous increment, avoid transferring existing files via rsync's
30-
# --link-dest support. This also decreases physical space usage considerably.
31-
if [ -d "$GHE_DATA_DIR/current/pages" ]; then
32-
link_dest="--link-dest=../../current/pages"
33-
fi
34-
35-
# Transfer Pages data from a GitHub instance to the current snapshot
36-
# directory, using a previous snapshot to avoid transferring files that have
37-
# already been transferred.
38-
ghe-rsync -avz \
39-
-e "ghe-ssh -p $(ssh_port_part "$host")" \
40-
--rsync-path='sudo -u git rsync' \
41-
$link_dest \
42-
"$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/pages/" \
43-
"$GHE_SNAPSHOT_DIR/pages" 1>&3
13+
# Use the common user data rsync backup utility.
14+
ghe-backup-userdata pages

‎libexec/ghe-backup-repositories-rsync‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,10 @@ RULES
168168
# __hookshot__, and __purgatory__ directories. The __nodeload_archives__,
169169
# __gitmon__, and __render__ directories are excludes since they act only as
170170
# caches.
171+
#
172+
# Under v2.x and greater, only the special __purgatory__ directory remains under
173+
# /data/repositories. All other special user data directories have been moved under
174+
# the /data/user directory.
171175
echo 1>&3
172176
echo "* Transferring special data directories ..." 1>&3
173177
rsync_repository_data <<RULES

‎libexec/ghe-backup-userdata‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
#!/bin/sh
2+
#/ Usage: ghe-backup-userdata <dirname>
3+
#/ Take an online, incremental snapshot of a user data directory. This is used
4+
#/ for a number of different simple datastores kept under /data/user on the
5+
#/ remote appliance, including: hookshot, alambic_assets, and pages data.
6+
set -e
7+
8+
# Bring in the backup configuration
9+
cd $(dirname "$0")/..
10+
. libexec/ghe-backup-config
11+
12+
# Verify rsync is available.
13+
if ! rsync --version 1>/dev/null 2>&1; then
14+
echo "Error: rsync not found." 1>&2
15+
exit 1
16+
fi
17+
18+
# Grab the host and /data/user directory name.
19+
host="$GHE_HOSTNAME"
20+
dirname="$1"
21+
22+
# Perform a host-check and establish GHE_REMOTE_XXX variables.
23+
ghe_remote_version_required "$host"
24+
25+
# Verify that the user data directory exists. Bail out if not, which may be due
26+
# to an older version of GHE or no data has been added to this directory yet.
27+
ghe-ssh "$host" -- "[ -d '$GHE_REMOTE_DATA_USER_DIR/$dirname' ]" || exit 0
28+
29+
# If we have a previous increment, avoid transferring existing files via rsync's
30+
# --link-dest support. This also decreases physical space usage considerably.
31+
if [ -d "$GHE_DATA_DIR/current/$dirname" ]; then
32+
link_dest="--link-dest=../../current/$dirname"
33+
fi
34+
35+
# Transfer all data from the user data directory using rsync.
36+
ghe-rsync -avz \
37+
-e "ghe-ssh -p $(ssh_port_part "$host")" \
38+
--rsync-path='sudo -u git rsync' \
39+
$link_dest \
40+
"$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/$dirname/" \
41+
"$GHE_SNAPSHOT_DIR/$dirname" 1>&3

‎libexec/ghe-restore-pages-rsync‎

Lines changed: 2 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -13,22 +13,5 @@ cd $(dirname "$0")/..
1313
# Show usage and bail with no arguments
1414
[ -z "$*" ] && print_usage
1515

16-
# Grab host arg
17-
host="$1"
18-
19-
# Perform a host-check and establish GHE_REMOTE_XXX variables.
20-
ghe_remote_version_required "$host"
21-
22-
# The snapshot to restore should be set by the ghe-restore command but this lets
23-
# us run this script directly.
24-
: ${GHE_RESTORE_SNAPSHOT:=current}
25-
26-
# Transfer all Pages data from the latest snapshot to the GitHub instance
27-
# in a single rsync invocation.
28-
if [ -d "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/pages/" ]; then
29-
ghe-rsync -avz --delete \
30-
-e "ghe-ssh -p $(ssh_port_part "$host")" \
31-
--rsync-path="sudo -u git rsync" \
32-
"$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/pages/" \
33-
"$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/pages" 1>&3
34-
fi
16+
# Restore all pages data via rsync
17+
ghe-restore-userdata pages "$1"

‎libexec/ghe-restore-userdata‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#!/bin/sh
2+
#/ Usage: ghe-restore-userdata <dirname> <host>
3+
#/ Restore a special user data directory via rsync. This is used
4+
#/ for a number of different simple datastores kept under /data/user on the
5+
#/ remote appliance, including: hookshot, alambic_assets, and pages data.
6+
set -e
7+
8+
# Bring in the backup configuration
9+
cd $(dirname "$0")/..
10+
. libexec/ghe-backup-config
11+
12+
# Show usage and bail with no arguments
13+
[ $# -lt 2 ] && print_usage
14+
15+
# Grab userdata directory name and host args
16+
dirname="$1"
17+
host="$2"
18+
19+
# Verify rsync is available.
20+
if ! rsync --version 1>/dev/null 2>&1; then
21+
echo "Error: rsync not found." 1>&2
22+
exit 1
23+
fi
24+
25+
# Perform a host-check and establish GHE_REMOTE_XXX variables.
26+
ghe_remote_version_required "$host"
27+
28+
# The snapshot to restore should be set by the ghe-restore command but this lets
29+
# us run this script directly.
30+
: ${GHE_RESTORE_SNAPSHOT:=current}
31+
32+
# Transfer data from the latest snapshot to the GitHub instance in a single
33+
# rsync invocation.
34+
if [ -d "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/$dirname" ]; then
35+
ghe-rsync -avz --delete \
36+
-e "ghe-ssh -p $(ssh_port_part "$host")" \
37+
--rsync-path="sudo -u git rsync" \
38+
"$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/$dirname/" \
39+
"$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/$dirname" 1>&3
40+
fi

‎test/test-ghe-backup.sh‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,23 @@ touch alice/index.html bob/index.html
1717
mkdir -p "$GHE_REMOTE_DATA_USER_DIR/common"
1818
echo "fake password hash data" > "$GHE_REMOTE_DATA_USER_DIR/common/manage-password"
1919

20+
# Create some fake hookshot data in the remote data directory
21+
if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
22+
mkdir -p "$GHE_REMOTE_DATA_USER_DIR/hookshot"
23+
cd "$GHE_REMOTE_DATA_USER_DIR/hookshot"
24+
mkdir -p repository-123 repository-456
25+
touch repository-123/test.bpack repository-456/test.bpack
26+
fi
27+
28+
# Create some fake alambic data in the remote data directory
29+
if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
30+
mkdir -p "$GHE_REMOTE_DATA_USER_DIR/alambic_assets/github-enterprise-assets/0000"
31+
touch "$GHE_REMOTE_DATA_USER_DIR/alambic_assets/github-enterprise-assets/0000/test.png"
32+
33+
mkdir -p "$GHE_REMOTE_DATA_USER_DIR/alambic_assets/github-enterprise-releases/0001"
34+
touch "$GHE_REMOTE_DATA_USER_DIR/alambic_assets/github-enterprise-releases/0001/1ed78298-522b-11e3-9dc0-22eed1f8132d"
35+
fi
36+
2037
# Create some fake elasticsearch data in the remote data directory
2138
mkdir -p "$GHE_REMOTE_DATA_USER_DIR/elasticsearch"
2239
cd "$GHE_REMOTE_DATA_USER_DIR/elasticsearch"
@@ -111,6 +128,14 @@ begin_test "ghe-backup first snapshot"
111128
if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
112129
[ "$(cat "$GHE_DATA_DIR/current/manage-password")" = "fake password hash data" ]
113130
fi
131+
132+
if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
133+
# verify all hookshot user data was transferred
134+
diff -ru "$GHE_REMOTE_DATA_USER_DIR/hookshot" "$GHE_DATA_DIR/current/hookshot"
135+
136+
# verify all alambic assets user data was transferred
137+
diff -ru "$GHE_REMOTE_DATA_USER_DIR/alambic_assets" "$GHE_DATA_DIR/current/alambic_assets"
138+
fi
114139
)
115140
end_test
116141

@@ -181,6 +206,14 @@ begin_test "ghe-backup subsequent snapshot"
181206
if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
182207
[ "$(cat "$GHE_DATA_DIR/current/manage-password")" = "fake password hash data" ]
183208
fi
209+
210+
if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
211+
# verify all hookshot user data was transferred
212+
diff -ru "$GHE_REMOTE_DATA_USER_DIR/hookshot" "$GHE_DATA_DIR/current/hookshot"
213+
214+
# verify all alambic assets user data was transferred
215+
diff -ru "$GHE_REMOTE_DATA_USER_DIR/alambic_assets" "$GHE_DATA_DIR/current/alambic_assets"
216+
fi
184217
)
185218
end_test
186219

‎test/test-ghe-restore.sh‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,23 @@ mkdir -p gh-enterprise-es/node/0
1818
touch gh-enterprise-es/node/0/stuff1
1919
touch gh-enterprise-es/node/0/stuff2
2020

21+
# Create some fake hookshot data in the remote data directory
22+
if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
23+
mkdir -p "$GHE_DATA_DIR/1/hookshot"
24+
cd "$GHE_DATA_DIR/1/hookshot"
25+
mkdir -p repository-123 repository-456
26+
touch repository-123/test.bpack repository-456/test.bpack
27+
fi
28+
29+
# Create some fake alambic data in the remote data directory
30+
if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
31+
mkdir -p "$GHE_DATA_DIR/1/alambic_assets/github-enterprise-assets/0000"
32+
touch "$GHE_DATA_DIR/1/alambic_assets/github-enterprise-assets/0000/test.png"
33+
34+
mkdir -p "$GHE_DATA_DIR/1/alambic_assets/github-enterprise-releases/0001"
35+
touch "$GHE_DATA_DIR/1/alambic_assets/github-enterprise-releases/0001/1ed78298-522b-11e3-9dc0-22eed1f8132d"
36+
fi
37+
2138
# Add some fake repositories to the snapshot
2239
mkdir -p "$GHE_DATA_DIR/1/repositories"
2340
cd "$GHE_DATA_DIR/1/repositories"
@@ -98,6 +115,14 @@ begin_test "ghe-restore into unconfigured vm"
98115

99116
# verify all pages data was transferred to the restore location
100117
diff -ru "$GHE_DATA_DIR/current/pages" "$GHE_REMOTE_DATA_USER_DIR/pages"
118+
119+
if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
120+
# verify all hookshot user data was transferred
121+
diff -ru "$GHE_DATA_DIR/current/hookshot" "$GHE_REMOTE_DATA_USER_DIR/hookshot"
122+
123+
# verify all alambic assets user data was transferred
124+
diff -ru "$GHE_DATA_DIR/current/alambic_assets" "$GHE_REMOTE_DATA_USER_DIR/alambic_assets"
125+
fi
101126
)
102127
end_test
103128

@@ -137,6 +162,14 @@ begin_test "ghe-restore into configured vm"
137162

138163
# verify all pages data was transferred to the restore location
139164
diff -ru "$GHE_DATA_DIR/current/pages" "$GHE_REMOTE_DATA_USER_DIR/pages"
165+
166+
if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
167+
# verify all hookshot user data was transferred
168+
diff -ru "$GHE_DATA_DIR/current/hookshot" "$GHE_REMOTE_DATA_USER_DIR/hookshot"
169+
170+
# verify all alambic assets user data was transferred
171+
diff -ru "$GHE_DATA_DIR/current/alambic_assets" "$GHE_REMOTE_DATA_USER_DIR/alambic_assets"
172+
fi
140173
)
141174
end_test
142175

@@ -175,6 +208,14 @@ begin_test "ghe-restore -c into configured vm"
175208

176209
# verify all pages data was transferred to the restore location
177210
diff -ru "$GHE_DATA_DIR/current/pages" "$GHE_REMOTE_DATA_USER_DIR/pages"
211+
212+
if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
213+
# verify all hookshot user data was transferred
214+
diff -ru "$GHE_DATA_DIR/current/hookshot" "$GHE_REMOTE_DATA_USER_DIR/hookshot"
215+
216+
# verify all alambic assets user data was transferred
217+
diff -ru "$GHE_DATA_DIR/current/alambic_assets" "$GHE_REMOTE_DATA_USER_DIR/alambic_assets"
218+
fi
178219
)
179220
end_test
180221

@@ -199,6 +240,14 @@ begin_test "ghe-restore with host arg"
199240

200241
# verify all pages data was transferred to the restore location
201242
diff -ru "$GHE_DATA_DIR/current/pages" "$GHE_REMOTE_DATA_USER_DIR/pages"
243+
244+
if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
245+
# verify all hookshot user data was transferred
246+
diff -ru "$GHE_DATA_DIR/current/hookshot" "$GHE_REMOTE_DATA_USER_DIR/hookshot"
247+
248+
# verify all alambic assets user data was transferred
249+
diff -ru "$GHE_DATA_DIR/current/alambic_assets" "$GHE_REMOTE_DATA_USER_DIR/alambic_assets"
250+
fi
202251
)
203252
end_test
204253

0 commit comments

Comments
 (0)