Merge pull request #52 from github/fix-vanished-files-warnings

Fix vanished file warnings causing ES transfer to fail

Author: rtomayko

libexec/ghe-backup-es-rsync

@@ -48,7 +48,7 @@ fi
 # directory, using a previous snapshot to avoid transferring files that have
 # already been transferred.
 echo "* Performing initial sync of ES indices ..." 1>&3
-rsync -avz \
+ghe-rsync -avz \
     -e "ghe-ssh -p $(ssh_port_part "$host")" \
     --rsync-path='sudo -u git rsync' \
     $link_dest \
@@ -73,7 +73,7 @@ ghe-ssh "$host" -- curl -s -XPOST "localhost:9200/_flush" >/dev/null
 
 # Transfer all ES indices again
 echo "* Performing follow-up sync of ES indices ..." 1>&3
-rsync -avz \
+ghe-rsync -avz \
     -e "ghe-ssh -p $(ssh_port_part "$host")" \
     --rsync-path='sudo -u git rsync' \
     $link_dest \

libexec/ghe-backup-pages-rsync

@@ -32,7 +32,7 @@ fi
 # Transfer Pages data from a GitHub instance to the current snapshot
 # directory, using a previous snapshot to avoid transferring files that have
 # already been transferred.
-rsync -avz \
+ghe-rsync -avz \
     -e "ghe-ssh -p $(ssh_port_part "$host")" \
     --rsync-path='sudo -u git rsync' \
     $link_dest \

libexec/ghe-backup-repositories-rsync

@@ -92,7 +92,7 @@ echo "
 # already been transferred. A set of rsync filter rules are provided on stdin
 # for each invocation.
 rsync_repository_data () {
-    rsync -av \
+    ghe-rsync -av \
         -e "ghe-ssh -p $(ssh_port_part "$host")" \
         $link_dest "$@" \
         --rsync-path='sudo -u git rsync' \

libexec/ghe-restore-pages-rsync

@@ -23,7 +23,7 @@ host="$1"
 # Transfer all Pages data from the latest snapshot to the GitHub instance
 # in a single rsync invocation.
 if [ -d "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/pages/" ]; then
-  rsync -avz --delete \
+  ghe-rsync -avz --delete \
       -e "ghe-ssh -p $(ssh_port_part "$host")" \
       --rsync-path="sudo -u git rsync" \
       "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/pages/" \

libexec/ghe-restore-repositories-rsync

@@ -22,7 +22,7 @@ host="$1"
 
 # Transfer all git repository data from the latest snapshot to the GitHub
 # instance in a single rsync invocation.
-rsync -av --delete \
+ghe-rsync -av --delete \
     -e "ghe-ssh -p $(ssh_port_part "$host")" \
     --rsync-path="sudo -u git rsync" \
     "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/repositories/" \

libexec/ghe-rsync

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+#/ Usage: ghe-rsync
+#/ Run rsync with silenced vanished file warnings (non-critical).
+#
+# Based on the rsync-no-vanished support script included with rsync:
+# https://bugzilla.samba.org/show_bug.cgi?id=10356
+
+set -o pipefail
+
+# Filter vanished file warnings from both stdout (rsync versions < 3.x) and
+# stderr (rsync versions >= 3.x). The complex redirections are necessary to
+# filter stderr while also keeping stdout and stderr separated.
+IGNOREOUT='^(file has vanished: |rsync warning: some files vanished before they could be transferred)'
+(rsync "${@}" 3>&1 1>&2 2>&3 3>&- |
+  (egrep -v "$IGNOREOUT" || true)) 3>&1 1>&2 2>&3 3>&- |
+  (egrep -v "$IGNOREOUT" || true)
+res=$?
+
+# rsync exits with 24 when vanished files are detected.
+if [ $res = 24 ]; then
+    res=0
+fi
+
+exit $res