Difference between revisions of "Rsync backup"

From Noah.org
Jump to navigationJump to search
 
Line 1: Line 1:
 +
[[Category:Engineering]]
  
 
This is the small script I use for online backups with rsync.
 
This is the small script I use for online backups with rsync.
This includes a one week rotating window. Prior days backups only
+
This includes a one week rotating window. Prior days include only
 
the files that changed, so each day does not take up the full
 
the files that changed, so each day does not take up the full
 
amount of disk. The prior days' backups will appear complete because
 
amount of disk. The prior days' backups will appear complete because
copied files are hard linked.
+
identical files are hard linked.
 
This isn't super fancy, but it gets the job done.
 
This isn't super fancy, but it gets the job done.
  
 
To use this script add something similar to the following line to /etc/crontab:
 
To use this script add something similar to the following line to /etc/crontab:
     01 4 * * * backup /usr/bin/rsync_backup /media/SharedDocs /home/backup
+
     01 4 * * * backup /usr/bin/rsync_backup /media/shareddocs /home/backup
  
 
<pre>
 
<pre>
Line 18: Line 19:
 
# prevents that from happening. All this permission changing it tedious, but it
 
# prevents that from happening. All this permission changing it tedious, but it
 
# eliminates any doubts. I could have done this with "chmod -R +X", but I wanted
 
# eliminates any doubts. I could have done this with "chmod -R +X", but I wanted
# to be explicit.
+
# to explicitly set the permission bits.
 
#
 
#
 
# Pass two arguments:
 
# Pass two arguments:
#    rsync_backup SOURCE_DIRECTORY BACKUP_ROOT
+
#    rsync_backup SOURCE_PATH BACKUP_PATH
 
#
 
#
 
# $Id$
 
# $Id$
  
SOURCE_DIRECTORY=$1
+
usage="usage: rsync_backup [-v] SOURCE_PATH BACKUP_PATH\n\
BACKUP_ROOT=$2
+
    SOURCE_PATH and BACKUP_PATH may be ssh-style remote paths; although,\n\
SOURCE_BASE=`basename $SOURCE_DIRECTORY`
+
    BACKUP_PATH is usually a local directory where you want the\n\
RSYNC_OPTS="-a --delete -q"
+
    backup set stored."
#RSYNC_OPTS="-a --delete -v --progress"
+
 
 +
while getopts ":vh" options; do
 +
    case $options in
 +
        v ) VERBOSE=1;;
 +
        h ) echo -e $usage
 +
            exit 1;;
 +
        \? ) echo -e $usage
 +
            exit 1;;
 +
        * ) echo -e $usage
 +
            exit 1;;
 +
    esac
 +
done
 +
shift $(($OPTIND - 1))
 +
SOURCE_PATH=$1
 +
BACKUP_PATH=$2
 +
if [[ ( -z $SOURCE_PATH ) || ( -z $BACKUP_PATH ) ]]; then
 +
    echo "Missing argument. Give source path and backup path in this format:"
 +
    echo "    $0 SOURCE_PATH BACKUP_PATH"
 +
    echo -e $usage
 +
    exit 1
 +
fi
 +
SOURCE_BASE=`basename $SOURCE_PATH`
 
PERMS_DIR=755
 
PERMS_DIR=755
 
PERMS_FILE=644
 
PERMS_FILE=644
 +
if [ $VERBOSE ]; then
 +
    RSYNC_OPTS="-a --delete -v"
 +
    date
 +
else
 +
    RSYNC_OPTS="-a --delete -q"
 +
fi
  
 
# Create the rotation directories if they don't exist.
 
# Create the rotation directories if they don't exist.
if [ ! -d $BACKUP_ROOT ] ; then
+
if [ ! -d $BACKUP_PATH ] ; then
     mkdir $BACKUP_ROOT
+
     mkdir $BACKUP_PATH
 
fi
 
fi
if [ ! -d $BACKUP_ROOT/$SOURCE_BASE.0 ] ; then
+
if [ ! -d $BACKUP_PATH/$SOURCE_BASE.0 ] ; then
     mkdir $BACKUP_ROOT/$SOURCE_BASE.0
+
     mkdir $BACKUP_PATH/$SOURCE_BASE.0
 
fi
 
fi
if [ ! -d $BACKUP_ROOT/$SOURCE_BASE.1 ] ; then
+
if [ ! -d $BACKUP_PATH/$SOURCE_BASE.1 ] ; then
     mkdir $BACKUP_ROOT/$SOURCE_BASE.1
+
     mkdir $BACKUP_PATH/$SOURCE_BASE.1
 
fi
 
fi
if [ ! -d $BACKUP_ROOT/$SOURCE_BASE.2 ] ; then
+
if [ ! -d $BACKUP_PATH/$SOURCE_BASE.2 ] ; then
     mkdir $BACKUP_ROOT/$SOURCE_BASE.2
+
     mkdir $BACKUP_PATH/$SOURCE_BASE.2
 
fi
 
fi
if [ ! -d $BACKUP_ROOT/$SOURCE_BASE.3 ] ; then
+
if [ ! -d $BACKUP_PATH/$SOURCE_BASE.3 ] ; then
     mkdir $BACKUP_ROOT/$SOURCE_BASE.3
+
     mkdir $BACKUP_PATH/$SOURCE_BASE.3
 
fi
 
fi
if [ ! -d $BACKUP_ROOT/$SOURCE_BASE.4 ] ; then
+
if [ ! -d $BACKUP_PATH/$SOURCE_BASE.4 ] ; then
     mkdir $BACKUP_ROOT/$SOURCE_BASE.4
+
     mkdir $BACKUP_PATH/$SOURCE_BASE.4
 
fi
 
fi
if [ ! -d $BACKUP_ROOT/$SOURCE_BASE.5 ] ; then
+
if [ ! -d $BACKUP_PATH/$SOURCE_BASE.5 ] ; then
     mkdir $BACKUP_ROOT/$SOURCE_BASE.5
+
     mkdir $BACKUP_PATH/$SOURCE_BASE.5
 
fi
 
fi
if [ ! -d $BACKUP_ROOT/$SOURCE_BASE.6 ] ; then
+
if [ ! -d $BACKUP_PATH/$SOURCE_BASE.6 ] ; then
     mkdir $BACKUP_ROOT/$SOURCE_BASE.6
+
     mkdir $BACKUP_PATH/$SOURCE_BASE.6
 
fi
 
fi
 +
 +
# TODO All these find operations to clean up permissions is going to add a lot
 +
# of overhead as the backup set gets bigger. At 40GB it's not a big deal. The
 +
# correct thing would be to have an exception based system where I correct
 +
# permissions when they cause a problem.
  
 
# Rotate backups.
 
# Rotate backups.
find $BACKUP_ROOT/$SOURCE_BASE.6 -type d -exec chmod $PERMS_DIR {} \;
+
find $BACKUP_PATH/$SOURCE_BASE.6 -type d -exec chmod $PERMS_DIR {} \;
find $BACKUP_ROOT/$SOURCE_BASE.6 -type f -exec chmod $PERMS_FILE {} \;
+
find $BACKUP_PATH/$SOURCE_BASE.6 -type f -exec chmod $PERMS_FILE {} \;
rm -rf $BACKUP_ROOT/$SOURCE_BASE.6
+
rm -rf $BACKUP_PATH/$SOURCE_BASE.6
mv    $BACKUP_ROOT/$SOURCE_BASE.5 $BACKUP_ROOT/$SOURCE_BASE.6
+
mv    $BACKUP_PATH/$SOURCE_BASE.5 $BACKUP_PATH/$SOURCE_BASE.6
mv    $BACKUP_ROOT/$SOURCE_BASE.4 $BACKUP_ROOT/$SOURCE_BASE.5
+
mv    $BACKUP_PATH/$SOURCE_BASE.4 $BACKUP_PATH/$SOURCE_BASE.5
mv    $BACKUP_ROOT/$SOURCE_BASE.3 $BACKUP_ROOT/$SOURCE_BASE.4
+
mv    $BACKUP_PATH/$SOURCE_BASE.3 $BACKUP_PATH/$SOURCE_BASE.4
mv    $BACKUP_ROOT/$SOURCE_BASE.2 $BACKUP_ROOT/$SOURCE_BASE.3
+
mv    $BACKUP_PATH/$SOURCE_BASE.2 $BACKUP_PATH/$SOURCE_BASE.3
mv    $BACKUP_ROOT/$SOURCE_BASE.1 $BACKUP_ROOT/$SOURCE_BASE.2
+
mv    $BACKUP_PATH/$SOURCE_BASE.1 $BACKUP_PATH/$SOURCE_BASE.2
cp -al $BACKUP_ROOT/$SOURCE_BASE.0 $BACKUP_ROOT/$SOURCE_BASE.1
+
cp -al $BACKUP_PATH/$SOURCE_BASE.0 $BACKUP_PATH/$SOURCE_BASE.1
  
 
# Backup.
 
# Backup.
find $BACKUP_ROOT/$SOURCE_BASE.0 -type d -exec chmod $PERMS_DIR {} \;
+
find $BACKUP_PATH/$SOURCE_BASE.0 -type d -exec chmod $PERMS_DIR {} \;
find $BACKUP_ROOT/$SOURCE_BASE.0 -type f -exec chmod $PERMS_FILE {} \;
+
find $BACKUP_PATH/$SOURCE_BASE.0 -type f -exec chmod $PERMS_FILE {} \;
rsync $RSYNC_OPTS $SOURCE_DIRECTORY/. $BACKUP_ROOT/$SOURCE_BASE.0/.
+
rsync $RSYNC_OPTS $SOURCE_PATH/. $BACKUP_PATH/$SOURCE_BASE.0/.
find $BACKUP_ROOT/$SOURCE_BASE.0 -type d -exec chmod $PERMS_DIR {} \;
+
find $BACKUP_PATH/$SOURCE_BASE.0 -type d -exec chmod $PERMS_DIR {} \;
find $BACKUP_ROOT/$SOURCE_BASE.0 -type f -exec chmod $PERMS_FILE {} \;
+
find $BACKUP_PATH/$SOURCE_BASE.0 -type f -exec chmod $PERMS_FILE {} \;
  
 
# Create a timestamp file to show when backup process was complete.
 
# Create a timestamp file to show when backup process was complete.
date > $BACKUP_ROOT/$SOURCE_BASE.0/BACKUP_TIMESTAMP
+
date > $BACKUP_PATH/$SOURCE_BASE.0/BACKUP_TIMESTAMP
  
 
# Create a shortcut to the latest backup.
 
# Create a shortcut to the latest backup.
ln -sf $BACKUP_ROOT/backup.0 $BACKUP_ROOT/latest
+
ln -sf $BACKUP_PATH/backup.0 $BACKUP_PATH/latest
 +
 
 +
if [ $VERBOSE ]; then
 +
    date
 +
fi
 
</pre>
 
</pre>

Revision as of 06:09, 20 April 2007


This is the small script I use for online backups with rsync. This includes a one week rotating window. Prior days include only the files that changed, so each day does not take up the full amount of disk. The prior days' backups will appear complete because identical files are hard linked. This isn't super fancy, but it gets the job done.

To use this script add something similar to the following line to /etc/crontab:

   01 4 * * * backup /usr/bin/rsync_backup /media/shareddocs /home/backup
#!/bin/sh
#
# This maintains a one week rotating backup. This will normalize permissions on
# all files and directories on backups. It has happened that someone removed
# owner write permissions on some files, thus breaking the backup process. This
# prevents that from happening. All this permission changing it tedious, but it
# eliminates any doubts. I could have done this with "chmod -R +X", but I wanted
# to explicitly set the permission bits.
#
# Pass two arguments:
#     rsync_backup SOURCE_PATH BACKUP_PATH
#
# $Id$

usage="usage: rsync_backup [-v] SOURCE_PATH BACKUP_PATH\n\
    SOURCE_PATH and BACKUP_PATH may be ssh-style remote paths; although,\n\
    BACKUP_PATH is usually a local directory where you want the\n\
    backup set stored."

while getopts ":vh" options; do
    case $options in
        v ) VERBOSE=1;;
        h ) echo -e $usage
            exit 1;;
        \? ) echo -e $usage
            exit 1;;
        * ) echo -e $usage
            exit 1;;
    esac
done
shift $(($OPTIND - 1))
SOURCE_PATH=$1
BACKUP_PATH=$2
if [[ ( -z $SOURCE_PATH ) || ( -z $BACKUP_PATH ) ]]; then
    echo "Missing argument. Give source path and backup path in this format:"
    echo "    $0 SOURCE_PATH BACKUP_PATH"
    echo -e $usage
    exit 1
fi
SOURCE_BASE=`basename $SOURCE_PATH`
PERMS_DIR=755
PERMS_FILE=644
if [ $VERBOSE ]; then
    RSYNC_OPTS="-a --delete -v"
    date
else
    RSYNC_OPTS="-a --delete -q"
fi

# Create the rotation directories if they don't exist.
if [ ! -d $BACKUP_PATH ] ; then
    mkdir $BACKUP_PATH
fi
if [ ! -d $BACKUP_PATH/$SOURCE_BASE.0 ] ; then
    mkdir $BACKUP_PATH/$SOURCE_BASE.0
fi
if [ ! -d $BACKUP_PATH/$SOURCE_BASE.1 ] ; then
    mkdir $BACKUP_PATH/$SOURCE_BASE.1
fi
if [ ! -d $BACKUP_PATH/$SOURCE_BASE.2 ] ; then
    mkdir $BACKUP_PATH/$SOURCE_BASE.2
fi
if [ ! -d $BACKUP_PATH/$SOURCE_BASE.3 ] ; then
    mkdir $BACKUP_PATH/$SOURCE_BASE.3
fi
if [ ! -d $BACKUP_PATH/$SOURCE_BASE.4 ] ; then
    mkdir $BACKUP_PATH/$SOURCE_BASE.4
fi
if [ ! -d $BACKUP_PATH/$SOURCE_BASE.5 ] ; then
    mkdir $BACKUP_PATH/$SOURCE_BASE.5
fi
if [ ! -d $BACKUP_PATH/$SOURCE_BASE.6 ] ; then
    mkdir $BACKUP_PATH/$SOURCE_BASE.6
fi

# TODO All these find operations to clean up permissions is going to add a lot
# of overhead as the backup set gets bigger. At 40GB it's not a big deal. The
# correct thing would be to have an exception based system where I correct
# permissions when they cause a problem.

# Rotate backups.
find $BACKUP_PATH/$SOURCE_BASE.6 -type d -exec chmod $PERMS_DIR {} \;
find $BACKUP_PATH/$SOURCE_BASE.6 -type f -exec chmod $PERMS_FILE {} \;
rm -rf $BACKUP_PATH/$SOURCE_BASE.6
mv     $BACKUP_PATH/$SOURCE_BASE.5 $BACKUP_PATH/$SOURCE_BASE.6
mv     $BACKUP_PATH/$SOURCE_BASE.4 $BACKUP_PATH/$SOURCE_BASE.5
mv     $BACKUP_PATH/$SOURCE_BASE.3 $BACKUP_PATH/$SOURCE_BASE.4
mv     $BACKUP_PATH/$SOURCE_BASE.2 $BACKUP_PATH/$SOURCE_BASE.3
mv     $BACKUP_PATH/$SOURCE_BASE.1 $BACKUP_PATH/$SOURCE_BASE.2
cp -al $BACKUP_PATH/$SOURCE_BASE.0 $BACKUP_PATH/$SOURCE_BASE.1

# Backup.
find $BACKUP_PATH/$SOURCE_BASE.0 -type d -exec chmod $PERMS_DIR {} \;
find $BACKUP_PATH/$SOURCE_BASE.0 -type f -exec chmod $PERMS_FILE {} \;
rsync $RSYNC_OPTS $SOURCE_PATH/. $BACKUP_PATH/$SOURCE_BASE.0/.
find $BACKUP_PATH/$SOURCE_BASE.0 -type d -exec chmod $PERMS_DIR {} \;
find $BACKUP_PATH/$SOURCE_BASE.0 -type f -exec chmod $PERMS_FILE {} \;

# Create a timestamp file to show when backup process was complete.
date > $BACKUP_PATH/$SOURCE_BASE.0/BACKUP_TIMESTAMP

# Create a shortcut to the latest backup.
ln -sf $BACKUP_PATH/backup.0 $BACKUP_PATH/latest

if [ $VERBOSE ]; then
    date
fi