#!/bin/bash # $Id: backup,v 2.20 2009-03-28 22:03:44 jlarsen Exp $ # Copyright 2004 through 2009 John R Larsen - john@larsen-family.us # Free for personal use. Contact author for commercial use. # # Description: bash script used in automated backups. Use "backup -hs" # for setup description and how the script works. # # Use a trap to handle termination trap 'echo "Exiting for SIGTERM or SIGINT"; exit' SIGTERM SIGINT #------------------------------------------------------------------------------- # get_version. Function that parses script version and date info from the RCS Id line. function get_version () { SCRIPT_VER="v`echo '$Id: backup,v 2.20 2009-03-28 22:03:44 jlarsen Exp $' | awk '{print $3}'`" SCRIPT_DATE="`echo '$Id: backup,v 2.20 2009-03-28 22:03:44 jlarsen Exp $' | awk '{print $4}'`" } # get_version #------------------------------------------------------------------------------ # display_help: Function to display the help screen function display_help () { local func_name=display_help if [ $(($D & 0x1)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> Inside display_help";fi cat << EOF ----- backup ($SCRIPT_VER $SCRIPT_DATE) -------------------------------------------- usage: backup {delta | daily | weekly | rsync} [ options ] delta Full backup on baseline day of month. Delta archives for all other days. daily Perform a daily backup. This freshens the archive with any changes. weekly Perform a weekly backup. This creates time stamped archive files. rsync Perform a rsync backup. This uses rsync to duplicate directory trees. Options: -bd DD Baseline Day for delta mode. Defaults to 01. Must include leading zero. -d 0xN Debug flags value. Use the "-hd" option to see how to use them. -dd N Delta Days. Number of days included in a delta archive. Default is 1. -e adr Email address (Default is none) (Separate multiple email addresses with commas and no spaces) -f file Name of File containing directories to backup (default: backup.dirlist) -fs n FailSafe recursion level (default: $FAIL_SAFE) -h Help screen -hd Help Debug. This describes how the builtin debugging works. -hs Help setup. This gives detailed operating and setup instructions. -test Do everything but zip the files. This shows what would have been done. You MUST modify these scripts for your situation! -mc Make template backup.conf file -md Make template backup.dirlist script -mdf Make template backup.df script -me Make template backup.exit_cmds script -mw Make template backup.webpage script EOF } # display_help #------------------------------------------------------------------------------ # display_debug_help: Function to display debug help. function display_debug_help () { local func_name=display_debug_help if [ $(($D & 0x1)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> Inside display_debug_help";fi cat << EOF | more ----- Debug Help ($SCRIPT_VER $SCRIPT_DATE) --------------------------------------- Debug is enabled in one of three ways in decending order of precedence: "-d 0xNNNN" on command line BACKUP_DEBUG_FLAGS environment variable "debug_flags" file in same directory as backup Each debug line is qualified with a construct like the following: if [ \$((\$D & 0x1)) -ne 0 ]; then echo "[\$LINENO]\$func_name:\$\$> Debug message or action";fi The expression on the left of -ne is a bash construct. The $((expr)) gets evaluated and returns a value. The expression I'm using is "$D & 0xNN". The & performs a bitwise AND and returns the result. If it is zero (no matching bits) then the debug line is skipped. If it is non zero then the debug line is performed. Each debug line can have multiple bits that turn it on or just a single bit. The pattern "0xNN" is the collection of bits that turn the debug on. The action of each bit is defined below. 00000000 - No debug 00000001 - Enable all the "Inside function name" debug lines 00000002 - process_command_line verbose output 00000004 - 00000008 - 00000010 - main: Display environment when starting up 00000020 - recurse_tree: Show pwd 00000040 - recurse_tree: Show pwd and ls -1A `pwd` 00000080 - read_file: Show contents of LINES array 80000000 - Don't output the "Debug flags changed" messages in "set_debug_level" EOF } # display_debug_help #------------------------------------------------------------------------------ # display_setup_help: Function to display the setup help screen function display_setup_help () { local func_name=display_setup_help if [ $(($D & 0x1)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> Inside display_setup_help";fi cat << EOF DESCRIPTION: This script performs a backup, either daily, weekly, or delta. Archives are zipfiles. A daily backup freshens the same archive every day. A weekly backup creates a new archive that has the date in the archive filename. A delta backup creates a full archive on the baseline day of the month and delta archives the other days. The file "backup.dirlist" contains a list of starting directories to backup. "backup" starts in a directory and recurses the directory structure from that point looking for files named "backup.list". When backup finds an instance of backup.list in a directory, it uses it as input to the zip program to create an archive. Once the archive is made, backup exits that directory and continues recursing through the rest of the directory structure. Once the entire directory structure has been searched backup moves on to the next starting directory defined in backup.dirlist. (rsync capability was added after v2.14. See rsync description below.) ARCHIVE NAMING: "backup" uses the full path to the directory where backup.list is found to create the archive name. It substitutes a period for slashes and a circumflex ^ for spaces. It is easy to tell from the archive filename where the archive started. For example, say that "backup" finds backup.list in the following directory and that the date is 1 Sep 2004: /mnt/cool-beast/c/Program Files/Microsoft Office/templates files "backup" creates daily archives with the filenames like the following: mnt.cool-beast.c.Program^Files.Microsoft^Office.templates^files.zip "backup" creates weekly and delta archives with filenames like the following: mnt.cool-beast.c.Program^Files.Microsoft^Office.templates^files__2004-Sep-01.zip Spaces in filenames cause all sorts of problems. A pox upon Microsoft. "backup" creates these filenames so that scp can be used to transfer files. "backup" can perform automated offsite backups. (See backup.exit_cmds below.) scp doesn't know how to handle filenames with spaces or parentheses in them. That's why this method of creating names was chosen. EMAIL: "backup" sends email to email addresses defined on the command line using the -e option. Multiple email addresses can be used by separating them with a comma and no space. The email reports all aspects of the backup including date, times, archive sizes, and beginning and ending disk usage. A single email is sent at the end of the entire backup operation. WEBPAGE UPDATE: "backup" can update a webpage. If the file "backup.webpage" exists in the same directory as the backup script, it is used to perform the update. Use "backup -mw" to create a template of backup.webpage that can be edited for your situation. Complete instructions are included in the file. If this file exists, it is used to updated the webpage just before each archive operation begins. That makes it possible to monitor progress on long archive operations by looking at the webpage. The webpage is essentially a duplicate of the contents of the email. DISK USAGE: "backup" reports beginning and ending disk usage. backup uses the "df" command unless it finds the file "backup.df" in the same directory as the backup script. In a situation with lots of auto mounted drives, the output of "df" can be very lengthy. You can use backup.df to perform df only on the drives that you're interested in. Use "backup -mdf" to create a template version of this file that you can edit for your situation. CRON OPERATION: It is intended that backup be run as a cron task. Typical crontab lines are shown below. The first line runs a daily backup Monday through Saturday starting at 3:01 AM. The second line runs a weekly backup Sunday morning: 01 03 * * 1-6 /root/backup/backup daily -f /root/backup/daily.dirlist -e user@domain.com 01 03 * * 0 /root/backup/backup weekly -f /root/backup/weekly.dirlist -e user@domain.com ERROR CHECKING: "backup" can be run in "test mode" by putting -test on the command line. In this mode it does everything except the actual time consuming archive operation. It also outputs a diagnostic message when it finds each control file. This makes it possible to verify that the backup configuration operates as expected. CONTROL FILES: Operation of "backup" is controlled by several files. These files are described below: backup.dirlist: This file defines the starting directories, archive storage directories, and commands to execute before starting the backup operation. The format of this file is described in the file itself. Use "backup -md" to create a template file that you can edit for your situation. Examples of "commands to execute" before a backup operation starts might be like starting a ClearCase view to make the starting directory visible, or mounting a drive. Spaces in directory names are permissible. There is no need to put double quotes around the names. "backup" takes care of that. backup.list: When backup finds this file in a directory, it stops recursing past that point, and uses its contents to define which files zip will archive. Each line of this file defines a directory or file that should be included in the archive. A single "." in the file directs zip to recursively archive everything in the current directory. This is the easiest way to archive a whole directory structure. If you only want to archive some of the files or directories in the current directory, then put each name on a separate line in the file. This is useful if you want to include files in a directory that isn't a subdirectory of the current directory. backup.enter_cmds: When backup finds this file it executes the commands in the file. The pwd used is the directory where the file is located. An example of using this file is to remove temporary files in a directory before archiving. For example, the Quicken program makes temporary files that start with tilda "~". These files mess up the zip program. Putting a command like "rm -f /home/jlarsen/quicken/\~*" in backup.enter would remove all the temp files starting with tilda. backup.exit_cmds: When backup finds this file it executes the commands in the file just before leaving the directory. When this script is called, $1 holds the path to the archive directory and $2 holds the name of the archive just made. If these values are null then no archive was made in the directory where backup.exit_cmds is located. This file can be used to do an automatic upload to an offsite location. Use "backup -me" to create a template file that can be modified for your situation. backup.no_recurse: When backup finds this file it exits the directory immediately doing nothing else. This is useful if there is a directory that you don't want backup to recurse into. For example, there are many hidden directories in a user's home directory (directories that start with a single dot ".") that have huge subdirectories. Putting backup.no_recurse keeps backup from wasting time and resources. backup.webpage: This file is described in "WEBPAGE UPDATE" above. backup.df: This file is described in "DISK USAGE" above. backup.email: This file is generated each time backup runs. It is overwritten each time. backup.html: This file is generated each time backup runs. It is the file that is copied over to the webserver. RSYNC DESCRIPTION: This section explains how backup works when "rsync" mode is used. The script works the same as far as emails and webpage updates are concerned. The backup.dirlist file is exactly the same except that it is possible to put a remote host in the backup directory line. In rsync mode the backup directory in the backup.dirlist file is considered the root. When backup finds the backup.rsync file in a directory it uses that as the starting point. The full path is maintained in the backup. For example, if you are running backup on a computer named "screamer" and you are backing up to a remote computer named "zippy", you could put a backup directory line in backup.dirlist of "zippy:/backups/screamer". When backup reads this line it would then look for a host entry of "zippy" in the backup.conf file. backup uses the settings for "zippy" to do the remote rsync backup. Now, if on screamer backup found a "backup.rsync" file in the /var/mail directory backup would use rsync to copy all the files in /var/mail over to zippy into the /backups/screamer/var/mail directory. In rsync mode backup.list, backup.enter_cmds, and backup.exit_cmds files are ignored. backup.no_recurse files behave the same way as described above. backup.conf: This file is used in rsync mode and contains ssh "host" definitions for remote sites where backups are stored. The format of this file is described in the file itself. Use "backup -mc" to create a template file that you can edit for your situation. backup.rsync: This file is looked for when backup is run in "rsync" mode. When found, backup stops recursing past that point. It uses the parameters in the backup.conf file to perform an rsync copy. The destination directory on the remote host comes from the backup.dirlist file. The local directory is replicated in the backup directory. CONFIGURING SSH CONNECTIONS: Use the "ssh-keygen" utility to generate an rsa public/private key pair. Name the key something like "id_rsa.client_name" so it is easily identifiable. Do not give the key a pass phrase. The ssh connections all require a public/private key. The ssh client's public RSA key must be copied to the ~/.ssh/authorized_keys file on the ssh server. This key must not have a passphrase as that would require entering a password, which can't be done in a cron job. The corresponding private key must be on the client in the location specified in the backup.conf files. Directory and file permissions are important. The RSA private key file must only be readable by the owner of the file, ie. "chmod 400 filename". The directory must only be writeable by the owner, ie. 700. Use chmod to set permissions accordingly. There are ssh host definition sections in backup.conf. Once the information is entered it is time to verify that ssh connections can be made. As an example say that there is a host definition "zippy". Verify that a connection can be made using the "zippy" host definition in the backup.conf file. On the command line use this command: ssh -F backup.conf zippy The above command verifies the settings in the backup.conf file for host zippy. You will be warned that the ssh server's key doesn't exist and you will be asked if you want to add it. Answer yes. Once this key is in the known_hosts file you won't be asked for it anymore. Type exit to return to the ssh client shell. Repeat the above configuration for all the host definitions in backup.conf. SUPPORTED PLATFORMS: This script has been used successfully on the following systems: Solaris 8 using bash v2.03.0(1) Solaris 9 using bash v2.05.0(1) Mandrake Linux 9.1 using bash v2.05b.0(1) cygwin 1.5.15 and later using bash v2.05b.0(1) and later Ubuntu 8.10 Linux 2.6.27-11-generic using bash v3.29(1) Copyright: 2004 through 2009 by John R Larsen Free for personal use. Contact the author for commercial use. http://larsen-family.us john@larsen-family.us EOF } # display_setup_help #------------------------------------------------------------------------------ # set_debug_level: Function that sets the $D variable based on a file, an #environment variable, or a command line variable. See display_debug_help. function set_debug_level () { local func_name=set_debug_level # Note: Command line -d overrides environment variable BACKUP_DEBUG_FLAGS if it exists, # which overrides debug_flags file if it exists. # Check one time if BACKUP_DEBUG_FLAGS env variable exists and use it if it does if [ "${ENV_VAR_TESTED:=FALSE}" = "FALSE" ]; then ENV_VAR_TESTED=TRUE D=${BACKUP_DEBUG_FLAGS:-0} if [ "$D" != "0" ] && [ $(($D & 0x80000000)) -eq 0 ]; then echo "[$LINENO]$$> ${func_name:-startup}: set_debug_level: Debug flags changed to: $D"; fi fi # Check the existence of a file named "debug_flags" and use it if conditions are right if [ -e debug_flags ]; then if [ "${DEBUG_FLAGS_FILE:=FIRST_READ}" = "FIRST_READ" ]; then # Getting here means the debug_flags file has never been read. DEBUG_FLAGS_FILE=`cat -s debug_flags` if [ "$D" = "0" ]; then # Getting here means $D was zero and the debug_flags contents should be used instead since env variable doesn't exist or has value of 0 D=$DEBUG_FLAGS_FILE if [ "$D" != "0" ] && [ $(($D & 0x80000000)) -eq 0 ]; then echo "[$LINENO]$$> ${func_name:-startup}: set_debug_level: Debug flags changed to: $D"; fi fi else # Read debug_flags file and update $D if the file has changed NEW_DEBUG_FLAGS_FILE=`cat -s debug_flags` if [ $NEW_DEBUG_FLAGS_FILE != $DEBUG_FLAGS_FILE ]; then DEBUG_FLAGS_FILE=$NEW_DEBUG_FLAGS_FILE D=$DEBUG_FLAGS_FILE if [ $(($D & 0x80000000)) -eq 0 ]; then echo "[$LINENO]$$> ${func_name:-startup}: set_debug_level: Debug flags changed to: $D" fi fi fi fi } # set_debug_level #------------------------------------------------------------------------------- # setup_env. Function that determines what OS the script is running on and # setups environment variables accordingly. function setup_env () { local func_name=setup_env if [ $(($D & 0x1)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> Inside setup_env";fi # Figure out which OS you're running on and setup program paths accordingly OS_TYPE=`uname -a | awk '{print $3}'` case $OS_TYPE in 5.*) # Solaris export DF="/usr/ucb/df" export HOST=`/usr/ucb/hostname` export LS=/bin/ls export MAIL=/usr/ucb/mail export SCP=/usr/local/bin/scp export SSH=/usr/local/bin/ssh export TEE=/usr/bin/tee export RSYNC_PGM=/usr/local/bin/rsync W=/usr/bin/w if [ -e /usr/bin/zip ]; then ZIP_PGM=/usr/bin/zip else echo "ERROR [$LINENO]$func_name> Missing zip." exit 1 fi ;; 2.4*) # Mandrake Linux 8.x and 9.x, Redhat Linux Work Station Enterprise 3 export DF="/bin/df -k" export HOST=`/bin/hostname` export LS=/bin/ls export MAIL=/bin/mail export SCP=/usr/bin/scp export SSH=/usr/bin/ssh export TEE=/usr/bin/tee export RSYNC_PGM=/usr/bin/rsync W=/usr/bin/w if [ -e /usr/bin/zip ]; then ZIP_PGM=/usr/bin/zip else echo "ERROR [$LINENO]$func_name> Missing zip." exit 1 fi ;; 2.6*) # Mandrake Linux 10.x, Ubuntu 8.10 export DF="/bin/df -k" export HOST=`/bin/hostname` export LS=/bin/ls export MAIL=/usr/bin/mail export SCP=/usr/bin/scp export SSH=/usr/bin/ssh export TEE=/usr/bin/tee export RSYNC_PGM=/usr/bin/rsync W=/usr/bin/w if [ -e /usr/bin/zip ]; then ZIP_PGM=/usr/bin/zip else echo "ERROR [$LINENO]$func_name> Missing zip." exit 1 fi ;; 1.5*) # cygwin export TEE=/usr/bin/tee # Base cygwin doesn't include many packages required for backup to work. # Check that these packages have been installed in their default locations. # Make sure the openssh package has been installed if [ -e /usr/bin/scp ]; then export SCP=/usr/bin/scp export SSH=/usr/bin/ssh else echo "ERROR [$LINENO]$func_name> Missing scp. Install the openssh package." exit 1 fi # Make sure the email package has been installed if [ -e /usr/bin/email ]; then MAIL=/usr/bin/email else echo "ERROR [$LINENO]$func_name> Missing email. Install the email package." exit 1 fi # Make sure the zip package has been installed if [ -e /usr/bin/zip ]; then ZIP_PGM=/usr/bin/zip else echo "ERROR [$LINENO]$func_name> Missing zip. Install the zip package." exit 1 fi # Make sure the rsync package has been installed if [ -e /usr/bin/rsync ]; then RSYNC_PGM=/usr/bin/rsync else echo "ERROR [$LINENO]$func_name> Missing rsync. Install the rsync package." exit 1 fi export DF="/usr/bin/df -k" export HOST=`/usr/bin/hostname` export LS=/usr/bin/ls W=/usr/bin/w ;; *) # Unknown OS echo "ERROR [$LINENO]$func_name> Unknown OS" exit 1 ;; esac # Set the default values of all environment variables here export WORKING_DIR=`pwd` export BACKUP_DATE=`date +%Y\-%b\-%d` export BACKUP_YYMMDDHHMM=`date +%y%m%d-%H%M` \ export YEAR=`date +%Y` export MONTH=`date +%m` export DAY=`date +%d` export BASELINE_DAY="01" export EMAIL_FILE=$WORKING_DIR/backup.email export HTML_FILE=$WORKING_DIR/backup.html export CALL_LEVEL=0 export BACKUP_TYPE=NOT_SET export BACKUP_DIR=~ export TEST_MODE="FALSE" BACKUP_LIST=backup.dirlist # The default for EMAIL_ADDRESS can be overwritten using the -e command line option EMAIL_ADDRESS="" # The maximum number or recursion levels is set by FAIL_SAFE; can be set using -fs option export FAIL_SAFE=50 # Set variable that tells zip what extensions to simply store with 0% compression export ZIPOPT="-n .gif:.zip:.wav:.mp3:.tar:.tgz:.jpg" # Set variable with options that the rsync program uses # rsync has a very large number of options. Use "man rsync" to understand those used here. #export RSYNC_OPT="-auvzR" export RSYNC_OPT="-rltDuvzRs --rsync-path=/usr/bin/rsync" # -r, --recursive recurse into directories # -l, --links copy symlinks as symlinks # -t, --times preserve modification times # -D same as --devices --specials # -u, --update skip files that are newer on the receiver # -v, --verbose increase verbosity # -z, --compress compress file data during the transfer # -R, --relative use relative path names # -s, --protect-args no space-splitting; wildcard chars only # --rsync-path put in path to rsync on remote host export RSYNC_FILTER="--filter='exclude Temporary Internet Files' --filter='exclude Application Data' --filter='exclude Application Data' --filter='exclude Cookies'" # Set the rsync I/O timeout value used TIMEOUT_VAL=60 # Set the number of rsync retries RETRY_LIMIT=3 YES=1 NO=0 TRUE=1 FALSE=0 # Need to figure out what yesterday's day, month, and year were for delta mode yesterday $DAY $MONTH $YEAR export ZIP_DAY=$OLD_DAY export ZIP_MONTH=$OLD_MONTH export ZIP_YEAR=$OLD_YEAR } # setup_env #------------------------------------------------------------------------------ # display_env # Function to display the values of environment variables function display_env () { local func_name=display_env if [ $(($D & 0x1)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> Inside display_env";fi echo "----- CURRENT ENVIRONMENT VARIABLE VALUES -----------------" echo "HOST: $HOST" echo "EMAIL_ADDRESS: $EMAIL_ADDRESS" echo "WORKING_DIR: $WORKING_DIR" echo "BACKUP_DATE: $BACKUP_DATE" echo "BACKUP_YYMMDDHHMM: $BACKUP_YYMMDDHHMM" echo "YEAR: $YEAR" echo "MONTH; $MONTH" echo "DAY: $DAY" echo "OLD_YEAR: $OLD_YEAR" echo "OLD_MONTH; $OLD_MONTH" echo "OLD_DAY: $OLD_DAY" echo "ZIP_YEAR: $ZIP_YEAR" echo "ZIP_MONTH; $ZIP_MONTH" echo "ZIP_DAY: $ZIP_DAY" echo "BASELINE_DAY: $BASELINE_DAY" echo "DELTA_DAYS: $DELTA_DAYS" echo "EMAIL_FILE: $EMAIL_FILE" echo "HTML_FILE: $HTML_FILE" echo "CALL_LEVEL: $CALL_LEVEL" echo "BACKUP_TYPE: $BACKUP_TYPE" echo "BACKUP_DIR: $BACKUP_DIR" echo "BACKUP_LIST: $BACKUP_LIST" echo "FAIL_SAFE: $FAIL_SAFE" echo "TEST_MODE: $TEST_MODE" echo "ZIPOPT: $ZIPOPT" echo "RSYNC_OPT: $RSYNC_OPT" echo "RSYNC_PGM: $RSYNC_PGM" echo "-----------------------------------------------------------" return 0 } # display_env #------------------------------------------------------------------------------ # yesterday. Function that provides a previous date in DD MM YYYY format. Inputs are: # $1: DD # $2: MM # $3: YYYY # $4: Number of days to go back (If NULL then only go back one day) # # The output of the function is returned in the following three variables: # OLD_DAY # OLD_MONTH # OLD_YEAR # # If no arguments are passed then OLD_DAY, OLD_MONTH, and OLD_YEAR will have yesterday's date. function yesterday () { DAY=${1:-`date +%d`} MONTH=${2:-`date +%m`} YEAR=${3:-`date +%Y`} NUM_DAYS=${4:-1} OLD_DAY=$DAY OLD_MONTH=$MONTH OLD_YEAR=$YEAR # Verify $DAY is in valid range and has correct format if [[ "$DAY" < "01" ]] || [[ "$DAY" > "31" ]] || [[ ${#DAY} -ne 2 ]]; then echo "ERROR[$LINENO]$func_name> DAY: $DAY has wrong format or is out of range" return 1 fi # Verify $MONTH is in valid range and has correct format if [[ "$MONTH" < "01" ]] || [[ "$MONTH" > "12" ]] || [[ ${#MONTH} -ne 2 ]]; then echo "ERROR[$LINENO]$func_name> MONTH: $MONTH has wrong format or is out of range" return 1 fi # Verify $YEAR is in valid range and has correct format if [[ "$YEAR" < "1970" ]] || [[ "$YEAR" > "9999" ]] || [[ ${#YEAR} -ne 4 ]]; then echo "ERROR[$LINENO]$func_name> YEAR: $YEAR has wrong format or is out of range" return 1 fi function _yesterday () { local _day=$1 local _month=$2 local _year=$3 if [ "$_day" = "01" ]; then case $_month in 01) OLD_DAY=31 OLD_MONTH=12 let OLD_YEAR=_year-1 ;; 02) OLD_DAY=31 OLD_MONTH=01 ;; 03) let LEAP=_year%4 if [[ $LEAP -eq 0 ]]; then OLD_DAY=29 else OLD_DAY=28 fi OLD_MONTH=02 ;; 04) OLD_DAY=31 OLD_MONTH=03 ;; 05) OLD_DAY=30 OLD_MONTH=04 ;; 06) OLD_DAY=31 OLD_MONTH=05 ;; 07) OLD_DAY=30 OLD_MONTH=06 ;; 08) OLD_DAY=31 OLD_MONTH=07 ;; 09) OLD_DAY=31 OLD_MONTH=08 ;; 10) OLD_DAY=30 OLD_MONTH=09 ;; 11) OLD_DAY=31 OLD_MONTH=10 ;; 12) OLD_DAY=30 OLD_MONTH=11 ;; esac else case $_day in 02) OLD_DAY=01 ;; 03) OLD_DAY=02 ;; 04) OLD_DAY=03 ;; 05) OLD_DAY=04 ;; 06) OLD_DAY=05 ;; 07) OLD_DAY=06 ;; 08) OLD_DAY=07 ;; 09) OLD_DAY=08 ;; 10) OLD_DAY=09 ;; *) let OLD_DAY=_day-1 ;; esac fi } # _yesterday # Backup the number of days requested local count=0 while [ $count -lt $NUM_DAYS ]; do _yesterday $OLD_DAY $OLD_MONTH $OLD_YEAR let count=count+1 done } # yesterday #------------------------------------------------------------------------------ # process_command_line function process_command_line () { local func_name=process_command_line if [ $(($D & 0x1)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> Inside process_command_line";fi while [ $# -ne 0 ] do case $1 in -bd) #baseline day - Day the full backup is made. Defaults to 01. if [ $(($D & 0x2)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> In case -bd"; fi BASELINE_DAY=$2 # Verify day is within range and has the correct format of DD including the leading zero if [[ "$BASELINE_DAY" < "01" ]] || [[ "$BASELINE_DAY" > "31" ]] || [[ ${#BASELINE_DAY} -ne 2 ]]; then echo "ERROR[$LINENO]$func_name> Baseline date: $2 has wrong format or is out of range" exit 1 fi shift ;; -d) #debug mode # Command line -d overrides environment variable BACKUP_DEBUG_FLAGS if it exists, which overrides debug_flags file if it exists D=$2 if [ "$D" != "0" ] && [ $(($D & 0x80000000)) -eq 0 ]; then echo "[$LINENO]$func_name:$$> Debug flags changed to: $D"; fi if [ $(($D & 0x200000)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> In case -d"; fi shift ;; daily) #performing daily backup if [ $(($D & 0x2)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> In case daily"; fi BACKUP_TYPE=DAILY ;; delta) #Delta mode if [ $(($D & 0x2)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> In case delta"; fi BACKUP_TYPE=DELTA ;; rsync) #rsync mode if [ $(($D & 0x2)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> In case rsync"; fi BACKUP_TYPE=RSYNC ;; -dd) #delta days - Number of days included in a delta backup. Default is 1. if [ $(($D & 0x2)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> In case -dd"; fi DELTA_DAYS=$2 # Verify the value is positive and not zero if [[ "$DELTA_DAYS" < "01" ]]; then echo "ERROR[$LINENO]$func_name> Delta days: $2 can't be less than 1" exit 1 fi # Recalculate the starting date for the delta archive yesterday $DAY $MONTH $YEAR $DELTA_DAYS export ZIP_DAY=$OLD_DAY export ZIP_MONTH=$OLD_MONTH export ZIP_YEAR=$OLD_YEAR shift ;; -e) #email address if [ $(($D & 0x2)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> In case -e"; fi EMAIL_ADDRESS=$2 shift ;; -f) # list of backups to perform if [ $(($D & 0x2)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> In case -f"; fi BACKUP_LIST=$2 shift ;; -fs) # fail safe level if [ $(($D & 0x2)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> In case -fs"; fi FAIL_SAFE=$2 shift ;; -h) #show help if [ $(($D & 0x2)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> In case -h"; fi display_help exit 0;; -hd) #show debug help if [ $(($D & 0x2)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> In case -hd"; fi display_debug_help exit 0;; -hs) #show setup help if [ $(($D & 0x2)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> In case -hs"; fi display_help >| /tmp/$$.tmp display_setup_help >> /tmp/$$.tmp cat /tmp/$$.tmp | more rm -f /tmp/$$.tmp exit 0;; -mc) #make template backup.conf file if [ $(($D & 0x2)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> In case -mc"; fi make_template_conf echo "Making template backup.conf file. You MUST modify this for your use." exit 0;; -md) #make template backup.dirlist script if [ $(($D & 0x2)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> In case -md"; fi make_template_dirlist echo "Making template backup.dirlist script. You MUST modify this for your use." exit 0;; -mdf) #make template backup.df script if [ $(($D & 0x2)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> In case -mdf"; fi make_template_df echo "Making template backup.df script. You MUST modify this for your use." exit 0;; -me) #make template backup.exit_cmds script if [ $(($D & 0x2)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> In case -me"; fi make_template_exit_cmds echo "Making template backup.exit_cmds script. You MUST modify this for your use." exit 0;; -mw) #make template backup.webpage script if [ $(($D & 0x2)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> In case -mw"; fi make_template_webpage echo "Making template backup.webpage script. You MUST modify this for your use." exit 0;; -test) # Don't zip files, but do everything else if [ $(($D & 0x2)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> In case -test"; fi TEST_MODE=TRUE echo "" echo "----- backup ($SCRIPT_VER $SCRIPT_DATE) --------------------------------------------" echo "Running in test mode. Archives aren't made." echo "" ;; weekly) #performing weekly backup (default) if [ $(($D & 0x2)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> In case weekly"; fi BACKUP_TYPE=WEEKLY ;; *) # Unrecognized option display_help echo "ERROR[$LINENO]$func_name> Unrecognized command line option: $1" exit 1 ;; esac # Don't shift command line if processing at last argument if [ $# -ne 0 ]; then shift fi done } # process_command_line #------------------------------------------------------------------------------ # web_page. Function that creates a web page and copies it somewhere periodically. # If the file backup.webpage exists then the web page is created and backup.webpage # is called with the name of the web page in $1. The backup.webpage file is unique # and must be self contained to copy the file where it needs to be. This can # be to another drive on the LAN or could be using scp. Use "backup -mw" to make # a template that can be modified for your needs. function web_page () { local func_name=web_page if [ $(($D & 0x1)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> Inside web_page";fi # Check if backup.webpage exists and exit if it doesn't if [ ! -e $WORKING_DIR/backup.webpage ]; then return; fi # Getting here means backup.webpage exists # Build the web page cat << EOF >| $HTML_FILE $BACKUP_TYPE backup report: $HOST

$BACKUP_TYPE backup report: $HOST

Time: `date +%Y\-%b\-%d\ %H\:%M\:%S\ %a`

EOF

cat $EMAIL_FILE >> $HTML_FILE
echo "
" >> $HTML_FILE echo " " >> $HTML_FILE # Use the backup.webpage script in the back ground to transfer the web page where it needs to go. $WORKING_DIR/backup.webpage $HTML_FILE & } # web_page #------------------------------------------------------------------------------ # make_template_df: Function to create a template backup.df script function make_template_df () { local func_name=make_template_df if [ $(($D & 0x1)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> Inside make_template_df";fi cat << EOF >| backup.df #!/bin/bash # Copyright 2004 through 2009 John R Larsen - john@larsen-family.us #----- backup ($SCRIPT_VER $SCRIPT_DATE) -------------------------------------------- # This is the backup.df script called by "backup" to update get a listing of disk usage. # You don't need this script if the normal output of df is acceptable. If this script # doesn't exist then "backup" will use df and capture its output. If you want to limit # the output of df to a few specific directories then use this script. # # The following environment variables are exported by backup and can be used in this script: # DF The full path to the df program on the host where the backup is running # \$DF EOF # Set execute permissions chmod 755 backup.df } # make_template_df #------------------------------------------------------------------------------ # make_template_exit_cmds: Function to create a template backup.exit_cmds script function make_template_exit_cmds () { local func_name=make_template_exit_cmds if [ $(($D & 0x1)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> Inside make_template_exit_cmds";fi cat << EOF >| backup.exit_cmds #!/bin/bash # Copyright 2004 through 2009 John R Larsen - john@larsen-family.us #----- backup ($SCRIPT_VER $SCRIPT_DATE) -------------------------------------------- # IMPORTANT: YOU MUST MODIFY THIS FILE FOR YOUR CONFIGURATION ########## START OF USER CONFIGURABLE VARIABLES ########################### # # Enter name of the remote host on the next line so it shows up in emails: REMOTE_HOST=remote_host_name # Full path to the destination directory on the remote host: DESTINATION_DIR=/path/to/destination/dir/on/remote/host # Full path to and name of the ssh configuration file: SSH_CONFIG_FILE=/path/to/ssh/config/file # Host name in the ssh config file SSH_HOST_NAME=name_of_ssh_host_in_config_file # Uncomment and use the next line to override backup's EMAIL_ADDRESS if desired: #EMAIL_ADDRESS= ########## END OF USER CONFIGURABLE VARIABLES ############################# # SCRIPT DESCRIPTION: # This is the backup.exit_cmds script called by "backup". It is executed in the # background so that "backup" continues in parallel. It contains any commands # that should be performed before "backup" leaves a directory. The example given here uses # this script to make an offsite copy of the archive just made. These are passed in: # \$1 - Path to the archive just made (Null if no archive made) # \$2 - Name of the archive file (Null if no archive made) # Abort if no filename passed in if [ "\$2" == "" ]; then echo "ERROR: No filename passed in. pwd = \`pwd\`" | \$TEE -a \$EMAIL_FILE exit 1 fi # The following environment variables are exported by backup and can be used in this script: # EMAIL_ADDRESS Email addresses to which email will be sent # EMAIL_FILE Full path to the email file that will be sent at the end of backup # HOST Name of the host performing the backup # LS Full path to the ls program # MAIL Full path to the email program # SCP Full path to the scp program on the host where "backup" is running # TEE Full path to the tee program # TEST_MODE Set to "TRUE" if operating in test mode # SCP copy. Use this to transfer a file to a remote machine using scp. # The "-F" option on the scp line tells scp to use the named ssh configuration file. # Below are the settings that might be in such a file. See manpage for ssh_config(5) # for complete descriptions: # Host host_entry # HostName host_name_or_IP_address # Port = 22 # UserKnownHostsFile = /path/to/file/known_hosts # User = login_name # IdentityFile = /path/to/private/rsa/key/file/id_rsa.name # Using the settings above and the private key (with no pass phrase) the transfer can be # done with no need to enter a password. The lines below are an example of performing # an scp transfer and sending an email informing of the success or failure of the # transfer. #------------------------------------------------------------------------------ # send_email: function to send an email after scp operation has completed # \$1 - SUCCESS or ERROR # \$2 - Name of file being copied # \$3 - Directory where file was located function send_email () { # Create the text of the email that will be sent echo "----- backup (\$SCRIPT_VER \$SCRIPT_DATE) --------------------------------------------" >| /tmp/\$\$.email if [ "\$1" == "SUCCESS" ]; then echo "SUCCESS: \$HOST to \$REMOTE_HOST copy succeeded" >> /tmp/\$\$.email else echo "ERROR: \$HOST to \$REMOTE_HOST copy failed" >> /tmp/\$\$.email fi echo "File: \$2" >> /tmp/\$\$.email echo "Source Dir: \$3" >> /tmp/\$\$.email echo "Destination Dir: \$DESTINATION_DIR" >> /tmp/\$\$.email echo "Transfer Ended: \$STOP_TIME" >> /tmp/\$\$.email echo "Transfer Started: \$START_TIME" >> /tmp/\$\$.email echo "Email Addresses: \$EMAIL_ADDRESS" >> /tmp/\$\$.email echo "This Script: \`pwd\`/backup.exit_cmds" >> /tmp/\$\$.email echo "File Stats:" >> /tmp/\$\$.email echo "\$SIZE_DATE_NAME" >> /tmp/\$\$.email # Send the email cat /tmp/\$\$.email | \$MAIL -s "backup: \$1: \$HOST to \$REMOTE_HOST" \$EMAIL_ADDRESS # Cleanup the temp file rm -f /tmp/\$\$.email } # send_email # Capture the start time of the transfer START_TIME=\`date +%Y\-%b\-%d\ %H\:%M\:%S\ %a\` if [ "\$EMAIL_ADDRESS" == "" ]; then # Try to put an entry into EMAIL_FILE. This might not make it if "backup" ends before this script starts up. echo "\$START_TIME: \$HOST to \$REMOTE_HOST copy of \$2 starting" >> \$EMAIL_FILE fi if [ "\$TEST_MODE" == "TRUE" ]; then # In test mode create a simple test file and send it over. # This will verify that scp works but won't take as much time. # Create a test file to send echo "----- backup (\$SCRIPT_VER \$SCRIPT_DATE) --------------------------------------------" >| \$\$.test echo "TESTING: \$HOST to \$REMOTE_HOST scp transfer" >> \$\$.test echo "File: \$2.test" >> \$\$.test echo "Source Dir: \$1" >> \$\$.test echo "Destination Dir: \$DESTINATION_DIR" >> \$\$.test echo "Transfer Started: \$START_TIME" >> \$\$.test echo "Email Addresses: \$EMAIL_ADDRESS" >> \$\$.test echo "This Script: \`pwd\`/backup.exit_cmds" >> \$\$.test # Uncomment first line to trouble shoot scp transfer problems #\$SCP -v -F "\$SSH_CONFIG_FILE" "\$\$.test" \$SSH_HOST_NAME:"\$DESTINATION_DIR/\$2.test" \$SCP -F "\$SSH_CONFIG_FILE" "\$\$.test" \$SSH_HOST_NAME:"\$DESTINATION_DIR/\$2.test" 2>/dev/null # Capture the result of the scp transfer to see if an error occurred RESULT=\$? # In test mode use passed in values as the file path and name SIZE_DATE_NAME=" test mode: \$1/\$2.test" # Clean up the temp file used rm -f \$\$.test else # Uncomment first line to trouble shoot scp transfer problems #\$SCP -v -F "\$SSH_CONFIG_FILE" "\$1/\$2" \$SSH_HOST_NAME:"\$DESTINATION_DIR/\$2" \$SCP -F "\$SSH_CONFIG_FILE" "\$1/\$2" \$SSH_HOST_NAME:"\$DESTINATION_DIR/\$2" 2>/dev/null # Capture the result of the scp transfer to see if an error occurred RESULT=\$? # The next line removes permissions field, the numeric field, and the owner field SIZE_DATE_NAME=\`\$LS -lo "\$1/\$2" | sed -e 's/[^ ]* *[^ ]* *[^ ]*//'\` fi # Capture the ending time of the transfer STOP_TIME=\`date +%Y\-%b\-%d\ %H\:%M\:%S\ %a\` # If EMAIL_ADDRESS isn't null then an email should be sent. Include success or failure status. if [ "\$EMAIL_ADDRESS" == "" ]; then if [ \$RESULT -eq 0 ]; then # Try to put an info line in EMAIL_FILE echo "SUCCESS: \$HOST to \$REMOTE_HOST copy of \$2 succeeded" >> \$EMAIL_FILE else # Try to put an info line in EMAIL_FILE echo "ERROR: \$HOST to \$REMOTE_HOST copy of \$2 failed" >> \$EMAIL_FILE fi else if [ \$RESULT -eq 0 ]; then send_email SUCCESS "\$2" "\$1" else send_email ERROR "\$2" "\$1" fi fi if [ "\$EMAIL_ADDRESS" == "" ]; then # Try to put an info line into EMAIL_FILE. This won't make it into the email or webpage if the "backup" # script ends before this script finishes its transfer. echo "\$STOP_TIME: \$HOST to \$REMOTE_HOST copy of \$2 ended" >> \$EMAIL_FILE fi EOF # Set execute permissions chmod 755 backup.exit_cmds } # make_template_exit_cmds #------------------------------------------------------------------------------ # make_template_dirlist: Function to create a template backup.dirlist script function make_template_dirlist () { local func_name=make_template_dirlist if [ $(($D & 0x1)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> Inside make_template_dirlist";fi cat << EOF >| backup.dirlist # Copyright 2004 through 2009 John R Larsen - john@larsen-family.us #----- backup ($SCRIPT_VER $SCRIPT_DATE) -------------------------------------------- # This is the backup.dirlist file that directs backup's operation. The format # of this file is important. Any line with a # as the first character is a # comment line and is ignored. Each entry has four lines, which are # described below: # # Line 1: startup commands # The first line contains commands that need to be executed before cd'ing to # the start directory. Put all commands on one line separating them with # semicolons. The starting directory might not be available without these # commands. For example, if the directory is in a ClearCase view, the view # might need to be started before the directory is available. If no commands # are needed, then leave this line blank. # # Line 2: starting directory # The second line is the starting directory. "backup" starts here and # recurses the entire directory structure looking for the file "backup.list". # If found, the contents of backup.list are used to tell "backup" what to # archive. (Use "backup -h" to read more about files backup.list, # backup.enter_cmds, backup.exit_cmds, and backup.no_recurse) # Spaces in directory names are permissible. There is no need to put # double quotes around the names. "backup" takes care of that. # # Line 3: archive directory # The third line is the directory where archives are stored. # Spaces in directory names are permissible. There is no need to put # double quotes around the names. "backup" takes care of that. A remote # host name can be used for rsync operation. # # Line 4: ending commands # The fourth line contains commands that need to be executed after the # backup completes. Put all commands on one line separating them with # semicolons. If no commands are needed, then leave this line blank. # # To make this file easier to read, use a dashed line to separate entries. # DON'T separate entry lines with comment lines. Keep all four lines # together. When the script finds the first non comment line it assumes the # next two lines complete the set of four lines. # # IMPORTANT: Make sure there are NO trailing spaces or slashes, else the # backup will fail. # # !!! TEST YOUR FILE !!! # Test your dirlist file using "backup daily -test -f filename" and carefully # look at the generated email output for correct path names. "backup" will # warn you of any trailing slashes or spaces. # # Below is a commented example of an entry: #---------------------------------------------------------------------------- # commands; to execute; before; the backup; begins # /path/to/directory/to/start/in # /path/to/directory/where/archives/are/stored # commands; to execute; after; the backup; ends EOF } # make_template_dirlist #------------------------------------------------------------------------------ # make_template_copy: Function to create a template backup.webpage script function make_template_webpage () { local func_name=make_template_webpage if [ $(($D & 0x1)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> Inside make_template_webpage";fi cat << EOF >| backup.webpage #!/bin/bash # Copyright 2004 through 2009 John R Larsen - john@larsen-family.us #----- backup ($SCRIPT_VER $SCRIPT_DATE) -------------------------------------------- # This is the backup.webpage script called by "backup" to update the website. The # name of the html file is passed in \$1 so that it can be renamed as needed. This is # required if multiple sites are reporting backups and all the web pages are in the same # directory on the webserver. # # The following environment variables are exported by backup and can be used in this script: # HOST Name of the host performing the backup # SCP The full path to the scp program on the host where the test is running # # The copying method will vary depending on how to access the web server. If it # is on the same LAN as the machine running the test, then a simple "cp" command # might work. If the webserver is on a remote machine then it may be necessary # to use "scp" to do the copy using already established public/private RSA key # access. Examples of both are given below. # LAN copy. This simple line copies the file to the correct webserver directory. #cp -f \$1 /var/www/html/backups/\${HOST}_backup-\$BACKUP_TYPE.html # SCP copy. Use this line if transferring the file to a remote webserver. The "-F" option # tells scp to use the named ssh configuration file. Below are the settings that might # be in such a file. See manpage for ssh_config(5) for complete descriptions: # # Host loader # HostName linux-beast # Port = 2222 # UserKnownHostsFile = /home/jlarsen/ssh_tunnel/linux-beast/known_hosts # User = jlarsen # IdentityFile = /home/jlarsen/ssh_tunnel/linux-beast/id_rsa.fhc-beast # # Using the settings above and the private key (with no pass phrase) the transfer can be # done by a cron job. #\$SCP -F /path/to/ssh/config/file \$1 loader:/path/to/webserver/directory/\${HOST}_backup-\$BACKUP_TYPE.html 2>/dev/null EOF # Set execute permissions chmod 755 backup.webpage } # make_template_copy #------------------------------------------------------------------------------ # make_template_conf: Function to create a template backup.conf file function make_template_conf () { local func_name=make_template_conf if [ $(($D & 0x1)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> Inside make_template_conf";fi cat << EOF >| backup.conf # Copyright 2004 through 2009 John R Larsen - john@larsen-family.us #----- backup ($SCRIPT_VER $SCRIPT_DATE) -------------------------------------------- # This is the backup.conf file used by "backup" in rsync mode. It contains "host" # definitions used for remote rsync operations. Each "host" specified in the # backup.dirlist file must have an entry in this file to work. Repeat all of the # entries for each "host" used in your backup.dirlist file. The directory where the # private key is located must be accessible only to the owner. Group and world # cannot have access. The key itself must also only be accessible to the owner. # Host = name_used_in_backup_dirlist HostName = fully_qualified_domain_name Port = 22 UserKnownHostsFile = /path/to/known_hosts User = username IdentityFile = /path/to/private/key/rsa.private EOF } # make_template_conf #------------------------------------------------------------------------------ # read_file: Function that reads a file line by line into array $LINES[]. # The number of lines in the file (and size of array) is put in $NUM_LINES. # $1 - File to input function read_file () { local func_name=read_file if [ $(($D & 0x1)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> Inside $func_name \$1: $1";fi # Remove any ^M in case the file is in DOS format which is evil sed -e 's/ //g' < $1 >| /tmp/read_file.$$ # Save the current value of the IFS variable old_ifs=$IFS IFS= # Use the "read" command to load up the array from the passed file NUM_LINES=0 while read LINES[$NUM_LINES] do let NUM_LINES=NUM_LINES+1 done Contents of \$LINES[]" local counter=0 while [ $counter -lt $NUM_LINES ]; do echo "LINES[$counter]: ${LINES[$counter]}" let counter=counter+1 done fi # Cleanup by removing the temporary read_file.$$ rm -f /tmp/read_file.$$ } # read_file #------------------------------------------------------------------------------ # recurse_tree: Function that recurses through the directory tree starting # with directory passed in $1. function recurse_tree () { local func_name=recurse_tree cd "$1" if [ $? -ne 0 ]; then # cd returns non zero value if permission denied to enter directory. This can happen # on a cygwin system where the user might not be running as administrator return 0 fi if [ $(($D & 0x1)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> Inside recurse_tree \$1: $1";fi #sleep 1 # Display recurse level and pwd if debug turned on if [ $(($D & 0x20)) -ne 0 ]; then echo "[$LINENO]$func_name> `date +%Y\-%b\-%d\ %H\:%M\:%S\ %a` CALL_LEVEL: $CALL_LEVEL pwd: `pwd`" fi # Show contents of directory if debug is turned on if [ $(($D & 0x40)) -ne 0 ]; then echo "[$LINENO]$func_name> ls `pwd`" $LS -AF "`pwd`" fi # If file backup.no_recurse exists in the current directory and CALL_LEVEL isn't zero # then return. CALL_LEVEL equal to zero means this is the top level directory. Not # checking for backup.no_recurse file allows having an entry in backup.dirlist that # starts recursing in an arbitrary directory even if backup.no_recurse is there. This # makes it possible to have a directory marked backup.no_recurse for a higher level # starting point but still use it as a starting directory for a subsequent entry in # a backup.dirlist file. if [ -e backup.no_recurse ] && [ $CALL_LEVEL -gt 0 ]; then if [ $TEST_MODE == "TRUE" ]; then echo "backup.no_recurse file found in: `pwd`" | $TEE -a $EMAIL_FILE fi return 0 fi # Check if recurse level has hit the fail safe value if [ $CALL_LEVEL -ge $FAIL_SAFE ]; then echo "ERROR [$LINENO]$func_name> Hit FAIL_SAFE level: $FAIL_SAFE in this directory: `pwd`" | $TEE -a $EMAIL_FILE return 1 fi # If file backup.enter_cmds exists in the current directory then source it if [ -e backup.enter_cmds ]; then # Strip off ^M in case file is in evil DOS format sed -e 's/ //g' < backup.enter_cmds >| enter_cmds mv enter_cmds backup.enter_cmds if [ $TEST_MODE == "TRUE" ]; then echo "backup.enter_cmds file found in: `pwd`" | $TEE -a $EMAIL_FILE fi source backup.enter_cmds >> $EMAIL_FILE fi # Check if file "backup.list" exists if [ -e backup.list ]; then if [ $TEST_MODE == "TRUE" ]; then echo "backup.list file found in: `pwd`" | $TEE -a $EMAIL_FILE fi # Add an entry to the email file echo "----------------------------------------------------------------" >> $EMAIL_FILE echo "Backup started at: `date +%Y\-%b\-%d\ %H\:%M\:%S\ %a`" >> $EMAIL_FILE echo "Backup file stored in: $BACKUP_DIR" >> $EMAIL_FILE echo "Starting directory: `pwd`" >> $EMAIL_FILE # Remove the first slash in the path, replace all the rest with dots, and replace spaces with circumflex ^ FILENAME=`pwd | sed -e 's/\///' -e 's/\//./g' -e 's/ /^/g'` # Check backup type case $BACKUP_TYPE in DAILY) FILENAME="${FILENAME}.zip" # Capture info about existing backup file for later comparison if [ -e "$BACKUP_DIR/$FILENAME" ]; then OLD_SIZE_DATE_NAME=`$LS -lo "$BACKUP_DIR/$FILENAME" | sed -e 's/[^ ]* *[^ ]* *[^ ]*//'` fi ;; WEEKLY) FILENAME="${FILENAME}__${BACKUP_DATE}.zip" echo "Created:" >> $EMAIL_FILE ;; DELTA) FILENAME="${FILENAME}__${BACKUP_DATE}.zip" ;; *) # Unknown backup type! echo "ERROR [$LINENO]$func_name> Unknown backup mode detected: $BACKUP_TYPE" | $TEE -a $EMAIL_FILE exit 1 ;; esac if [ $TEST_MODE == "TRUE" ]; then if [ "$BACKUP_TYPE" == "DELTA" ]; then SIZE_DATE_NAME=" test mode: $BACKUP_DIR/$YEAR/$MONTH/$DAY/$FILENAME" else SIZE_DATE_NAME=" test mode: $BACKUP_DIR/$FILENAME" fi else # Update the webpage before each backup so incremental progress can be detected during long backups web_page # Create the zipfile using backup.list to determine files included. Note that zip doesn't care if # backup.list is in unix or dos format, so there is no need to remove ^M from backup.list. if [ "$BACKUP_TYPE" == "DELTA" ]; then if [ "$DAY" == "$BASELINE_DAY" ]; then # Make a full backup on the first day of the month (or day specified by -bd option) cat backup.list | $ZIP_PGM -urqy "$BACKUP_DIR/$YEAR/$MONTH/$DAY/$FILENAME" -@ # The next line removes permissions field, the numeric field, and the owner field SIZE_DATE_NAME=`$LS -lo "$BACKUP_DIR/$YEAR/$MONTH/$DAY/$FILENAME" | sed -e 's/[^ ]* *[^ ]* *[^ ]*//'` else # Make a delta backup on the other days of the month cat backup.list | $ZIP_PGM -urqy -t ${ZIP_MONTH}${ZIP_DAY}${ZIP_YEAR} "$BACKUP_DIR/$YEAR/$MONTH/$DAY/$FILENAME" -@ if [ -e "$BACKUP_DIR/$YEAR/$MONTH/$DAY/$FILENAME" ]; then # The next line removes permissions field, the numeric field, and the owner field echo "Created:" >> $EMAIL_FILE SIZE_DATE_NAME=`$LS -lo "$BACKUP_DIR/$YEAR/$MONTH/$DAY/$FILENAME" | sed -e 's/[^ ]* *[^ ]* *[^ ]*//'` else SIZE_DATE_NAME=" No backup file created" fi fi else # daily or weekly mode cat backup.list | $ZIP_PGM -urqy "$BACKUP_DIR/$FILENAME" -@ SIZE_DATE_NAME=`$LS -lo "$BACKUP_DIR/$FILENAME" | sed -e 's/[^ ]* *[^ ]* *[^ ]*//'` if [ "$BACKUP_TYPE" == "DAILY" ]; then # Determine if the archive was updated or not and output to email accordingly if [[ "$OLD_SIZE_DATE_NAME" == "$SIZE_DATE_NAME" ]]; then echo "Archive not changed:" >> $EMAIL_FILE else echo "Updated:" >> $EMAIL_FILE fi fi fi fi echo "$SIZE_DATE_NAME" >> $EMAIL_FILE echo "Backup ended at: `date +%Y\-%b\-%d\ %H\:%M\:%S\ %a`" >> $EMAIL_FILE # If file backup.exit_cmds exists in the directory then source it if [ -e backup.exit_cmds ]; then # Strip off ^M in case file is in evil DOS format sed -e 's/ //g' < backup.exit_cmds >| exit_cmds mv exit_cmds backup.exit_cmds if [ $TEST_MODE == "TRUE" ]; then echo "backup.exit_cmds file found in: `pwd`" | $TEE -a $EMAIL_FILE if [ "$BACKUP_TYPE" == "DELTA" ]; then source backup.exit_cmds "$BACKUP_DIR/$YEAR/$MONTH/$DAY" "$FILENAME" | $TEE -a $EMAIL_FILE & else source backup.exit_cmds "$BACKUP_DIR" "$FILENAME" | $TEE -a $EMAIL_FILE & fi else if [ "$BACKUP_TYPE" == "DELTA" ]; then source backup.exit_cmds "$BACKUP_DIR/$YEAR/$MONTH/$DAY" "$FILENAME" >> $EMAIL_FILE & else source backup.exit_cmds "$BACKUP_DIR" "$FILENAME" >> $EMAIL_FILE & fi fi fi return 0 fi # Getting here means backup.list wasn't found and need to recurse all subdirectories # in this directory looking for instances of backup.list # Redirect directory listing to a file (echo "`$LS -1A .`" >/tmp/filelist) 2>/dev/null # Load array from the file read_file /tmp/filelist # Test each element of the current directory and recurse into subdirectories local count=0 while [ $count -lt $NUM_LINES ]; do if [ -d "${LINES[$count]}" ]; then let CALL_LEVEL=CALL_LEVEL+1 ( recurse_tree "${LINES[$count]}" ) let CALL_LEVEL=CALL_LEVEL-1 fi let count=count+1 done # If file backup.exit_cmds exists in the directory then source it. No backup operation # has been done at this point, so no arguments are passed to backup.exit_cmds. if [ -e backup.exit_cmds ]; then # Strip off ^M in case file is in evil DOS format sed -e 's/ //g' < backup.exit_cmds >| exit_cmds mv exit_cmds backup.exit_cmds if [ $TEST_MODE == "TRUE" ]; then echo "backup.exit_cmds file found in: `pwd`" | $TEE -a $EMAIL_FILE source backup.exit_cmds | $TEE -a $EMAIL_FILE & else source backup.exit_cmds >> $EMAIL_FILE & fi fi } # recurse_tree #------------------------------------------------------------------------------ # recurse_tree_rsync: Function that recurses through the directory tree starting # with directory passed in $1. This function is only used for rsync backups. function recurse_tree_rsync () { local func_name=recurse_tree_rsync cd "$1" if [ $? -ne 0 ]; then # cd returns non zero value if permission denied to enter directory. This can happen # on a cygwin system where the user might not be running as administrator return 0 fi if [ $(($D & 0x1)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> Inside recurse_tree_rsync \$1: $1";fi #sleep 1 # Display recurse level and pwd if debug turned on if [ $(($D & 0x20)) -ne 0 ]; then echo "[$LINENO]$func_name> `date +%Y\-%b\-%d\ %H\:%M\:%S\ %a` CALL_LEVEL: $CALL_LEVEL pwd: `pwd`" fi # Show contents of directory if debug is turned on if [ $(($D & 0x40)) -ne 0 ]; then echo "[$LINENO]$func_name> ls `pwd`" $LS -AF "`pwd`" fi # If file backup.no_recurse exists in the current directory and CALL_LEVEL isn't zero # then return. CALL_LEVEL equal to zero means this is the top level directory. Not # checking for backup.no_recurse file allows having an entry in backup.dirlist that # starts recursing in an arbitrary directory even if backup.no_recurse is there. This # makes it possible to have a directory marked backup.no_recurse for a higher level # starting point but still use it as a starting directory for a subsequent entry in # a backup.dirlist file. if [ -e backup.no_recurse ] && [ $CALL_LEVEL -gt 0 ]; then if [ $TEST_MODE == "TRUE" ]; then echo "backup.no_recurse file found in: `pwd`" | $TEE -a $EMAIL_FILE fi return 0 fi # Check if recurse level has hit the fail safe value if [ $CALL_LEVEL -ge $FAIL_SAFE ]; then echo "ERROR [$LINENO]$func_name> Hit FAIL_SAFE level: $FAIL_SAFE in this directory: `pwd`" | $TEE -a $EMAIL_FILE return 1 fi # Check if file "backup.rsync" exists if [ -e backup.rsync ]; then if [ $TEST_MODE == "TRUE" ]; then echo "backup.rsync file found in: `pwd`" | $TEE -a $EMAIL_FILE fi # Remove the first slash in the path, replace all the rest with dots, and replace spaces with circumflex ^ #DIRNAME=`pwd | sed -e 's/\///' -e 's/\//./g' -e 's/ /^/g'` #DIRNAME="${DIRNAME}.rsync" # Add an entry to the email file echo "----------------------------------------------------------------" >> $EMAIL_FILE echo "rsync backup started at: `date +%Y\-%b\-%d\ %H\:%M\:%S\ %a`" >> $EMAIL_FILE echo "Source directory: `pwd`" >> $EMAIL_FILE echo "Destination directory: $BACKUP_DIR`pwd`" >> $EMAIL_FILE if [ $TEST_MODE == "TRUE" ]; then echo " test mode: $BACKUP_DIR`pwd`" >> $EMAIL_FILE DRY_RUN=--dry-run else DRY_RUN="" # Update the webpage before each backup so incremental progress can be detected during long backups web_page fi # Check if $BACKUP_DIR has a colon in it. If it does that means the destination is a remote host echo $BACKUP_DIR | grep ":" if [ $? == 0 ]; then # The $BACKUP_DIR line from backup.dirlist has a host name that points to an entry in backup.conf. # This allows different remote destinations to be specified for each entry in backup.dirlist. # rsync is fussy about how its command line is made. Need to make a temporary # script to source so that it handles its options correctly. # Need to strip off the remote host and colon to obtain a directory name SHORT_BACKUP_DIR=`echo $BACKUP_DIR | sed -e 's/.*://'` cat << EOF >| $WORKING_DIR/rsync.sh #----- backup ($SCRIPT_VER $SCRIPT_DATE) -------------------------------------------- # This script auto generated by "backup" $RSYNC_PGM \ $RSYNC_OPT \ --no-g \ --chmod=ugo=rwX \ --delete \ -e "$SSH -F $WORKING_DIR/backup.conf" \ $RSYNC_FILTER \ $DRY_RUN \ --backup \ --backup-dir=${SHORT_BACKUP_DIR}.$BACKUP_YYMMDDHHMM \ --timeout=$TIMEOUT_VAL \ "`pwd`/" \ "$BACKUP_DIR" EOF else # No colon in $BACKUP_DIR means the rsync operation is to a local drive. # rsync is fussy about how its command line is made. Need to make a temporary # script to source so that it handles its options correctly. cat << EOF >| $WORKING_DIR/rsync.sh #----- backup ($SCRIPT_VER $SCRIPT_DATE) -------------------------------------------- # This script auto generated by "backup" $RSYNC_PGM \ $RSYNC_OPT \ --no-g \ --chmod=ugo=rwX \ --delete \ $RSYNC_FILTER \ $DRY_RUN \ --backup \ --backup-dir=${BACKUP_DIR}.$BACKUP_YYMMDDHHMM \ --timeout=$TIMEOUT_VAL \ "`pwd`/" \ "$BACKUP_DIR" EOF fi source $WORKING_DIR/rsync.sh >> $EMAIL_FILE echo "Rsync backup ended at: `date +%Y\-%b\-%d\ %H\:%M\:%S\ %a`" >> $EMAIL_FILE return 0 fi # Getting here means backup.rsync wasn't found and need to recurse all subdirectories # in this directory looking for instances of backup.rsync # Redirect directory listing to a file (echo "`$LS -1A .`" >/tmp/filelist) 2>/dev/null # Load array from the file read_file /tmp/filelist # Test each element of the current directory and recurse into subdirectories local count=0 while [ $count -lt $NUM_LINES ]; do if [ -d "${LINES[$count]}" ]; then let CALL_LEVEL=CALL_LEVEL+1 ( recurse_tree_rsync "${LINES[$count]}" ) let CALL_LEVEL=CALL_LEVEL-1 fi let count=count+1 done } # recurse_tree_rsync #------------------------------------------------------------------------------ # set_working_dir: Function that sets the WORKING_DIR based on $0 function set_working_dir () { # Configure working directory WORKING_DIR=`dirname $0` if [ "$WORKING_DIR" = "." ]; then WORKING_DIR=`pwd` fi cd $WORKING_DIR } # set_working_dir #------------------------------------------------------------------------------ # main function main () { local func_name=main if [ $(($D & 0x1)) -ne 0 ]; then echo "[$LINENO]$func_name:$$> Inside main";fi # If no arguments are passed then output the help screen and exit if [ $# -eq 0 ]; then display_help exit 1 fi setup_env process_command_line $* # Make sure backup type was on command line if [ $BACKUP_TYPE == "NOT_SET" ]; then display_help echo "ERROR [$LINENO]$func_name> Backup type (delta, daily or weekly) not on command line." exit 1 fi # Make sure $BACKUP_LIST exists if [ ! -e $BACKUP_LIST ]; then display_help echo "ERROR [$LINENO]$func_name> File $BACKUP_LIST doesn't exist" exit 1 fi # Show environment variable values if verbose level is high enough if [ $(($D & 0x10)) -ne 0 ]; then display_env fi # If running as root then try to remount all the drives to make sure they're all there UID_VALUE=`id | awk '{print $1}'` if [ "$UID_VALUE" = "uid=0(root)" ]; then umount -a 2>/dev/null mount -a 2>/dev/null fi # Capture current conditions echo "------------------------------- backup $SCRIPT_VER $SCRIPT_DATE --------" >| $EMAIL_FILE echo "$HOST backup log" >> $EMAIL_FILE echo "Email sent to: $EMAIL_ADDRESS " >> $EMAIL_FILE echo "Script started at: `date +%Y\-%b\-%d\ %H\:%M\:%S\ %a`" >> $EMAIL_FILE echo "----------------------------------------------------------------" >> $EMAIL_FILE echo "Starting uptime, load average, and users:" >> $EMAIL_FILE echo "`$W`" >> $EMAIL_FILE echo "----------------------------------------------------------------" >> $EMAIL_FILE echo "Starting disk usage:" >> $EMAIL_FILE if [ -e $WORKING_DIR/backup.df ]; then source $WORKING_DIR/backup.df >> $EMAIL_FILE cd $WORKING_DIR else echo "`$DF`" >> $EMAIL_FILE fi # Read in commands from $BACKUP_LIST read_file $BACKUP_LIST # Loop through and process all the entries in the $BACKUP_LIST index=0 while [ $index -lt $NUM_LINES ]; do if [ "${LINES[$index]:0:1}" == "#" ]; then # Skip comment lines let index=index+1 else # Load the three lines into correct variables # Execute the command line. Need to use a temp file and source it. echo "${LINES[$index]}" >| /tmp/$$.command chmod 700 /tmp/$$.command /tmp/$$.command $TEE -a $EMAIL_FILE rm -f /tmp/$$.command # Get the starting directory for the backup and cd to it let index=index+1 STARTING_DIR=${LINES[$index]} # Declare error if trailing space found in starting directory name length=${#STARTING_DIR} let length=length-1 if [ "${STARTING_DIR:$length:1}" == " " ]; then echo "ERROR in $BACKUP_LIST line $index: Trailing space found in starting directory name: \"$STARTING_DIR\"" | $TEE -a $EMAIL_FILE # Adjust index to point to next triplet let index=index+3 continue fi # Declare error if starting directory doesn't exist if [ ! -d "$STARTING_DIR" ]; then echo "ERROR in $BACKUP_LIST line $index: Specified starting directory doesn't exist: $STARTING_DIR" | $TEE -a $EMAIL_FILE let index=index+3 continue fi cd "$STARTING_DIR" let index=index+1 # Get the archive directory where the files are stored BACKUP_DIR=${LINES[$index]} # Declare error if trailing space found in archive directory name length=${#BACKUP_DIR} let length=length-1 if [ "${BACKUP_DIR:$length:1}" == " " ]; then echo "ERROR in $BACKUP_LIST line $index: Trailing space found in archive directory name: \"$BACKUP_DIR\"" | $TEE -a $EMAIL_FILE # Adjust index to point to next triplet let index=index+2 continue fi # Only check for archive directory if not doing an RSYNC backup if [ "$BACKUP_TYPE" != "RSYNC" ]; then # Declare error if archive directory doesn't exist if [ ! -d "$BACKUP_DIR" ]; then echo "ERROR in $BACKUP_LIST line $index: Specified archive directory doesn't exist: $BACKUP_DIR" | $TEE -a $EMAIL_FILE let index=index+2 # Skip to the next starting directory in $BACKUP_LIST continue fi fi let index=index+1 # delta mode requires subdirectories be added to $BACKUP_DIR based on day of the month # Files are stored in .../year/month/day. Create directories if they don't exist. if [ "$BACKUP_TYPE" == "DELTA" ]; then mkdir -p ${BACKUP_DIR}/${YEAR}/${MONTH}/${DAY} fi # Getting here means the starting and archive directories exist and the backup can proceed. # Start the backup. This must be done in a sub shell so that environment variables # aren't overwritten as the function recurses. if [ "$BACKUP_TYPE" == "RSYNC" ]; then # rsync backups behave differently and use their own recursive function ( recurse_tree_rsync "$STARTING_DIR" ) else # Use this function for all the other backup types ( recurse_tree "$STARTING_DIR" ) fi # Execute the end commands line. Need to use a temp file and source it. echo "${LINES[$index]}" >| /tmp/$$.command chmod 700 /tmp/$$.command /tmp/$$.command $TEE -a $EMAIL_FILE rm -f /tmp/$$.command let index=index+1 fi done echo "----------------------------------------------------------------" >> $EMAIL_FILE echo "Ending uptime, load average, and users:" >> $EMAIL_FILE echo "`$W`" >> $EMAIL_FILE echo "----------------------------------------------------------------" >> $EMAIL_FILE echo "Ending disk usage:" >> $EMAIL_FILE if [ -e $WORKING_DIR/backup.df ]; then source $WORKING_DIR/backup.df >> $EMAIL_FILE cd $WORKING_DIR else echo "`$DF`" >> $EMAIL_FILE fi echo "----------------------------------------------------------------" >> $EMAIL_FILE echo "Script ended at: `date +%Y\-%b\-%d\ %H\:%M\:%S\ %a`" >> $EMAIL_FILE echo "----------------------------------------------------------------" >> $EMAIL_FILE # Return to the starting directory cd $WORKING_DIR # Send the email if EMAIL_ADDRESS isn't null if [ "$EMAIL_ADDRESS" != "" ]; then # If an error occurred then put ERROR in the email subject line error_found=`cat $EMAIL_FILE | grep ERROR` if [ "$error_found" == "" ]; then cat $EMAIL_FILE | $MAIL -s "backup: $HOST" $EMAIL_ADDRESS else cat $EMAIL_FILE | $MAIL -s "backup: ERROR! $HOST" $EMAIL_ADDRESS fi fi # Copy the webpage over if enabled web_page } # main set_working_dir set_debug_level get_version main $*