#!/bin/bash
# OpaL data syncronization tool
#
#    Copyright (c) 2004-2006,2008-2011,2013,2016 Ola Lundqvist <ola@inguza.com>
#
#  This is free software; you can redistribute it and/or modify it under
#  the terms of the GNU General Public License as published by the Free
#  Software Foundation; either version 2, or (at your option) any later
#  version.
#
#  This is distributed in the hope that it will be useful, but WITHOUT
#  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
#  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
#  for more details.
#
#  You should have received a copy of the GNU General Public License with
#  this source package as the file COPYING.  If not, write to the
#  Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#  MA 02110-1301 USA.
#
#
# Depends on debianutils >= 1.6, because of tempfile
# Changelog:
#	2004-10-09	 Ola Lundqvist <ola@inguza.com>
#		Wrote it to fullfill my needs of a backup tool.
#	2004-10-10	 Ola Lundqvist <ola@inguza.com>
#		Modified it so that the output looks better and so that all errors are printed in a good way.
#	2004-10-11 (1.0) Ola Lundqvist <ola@inguza.com>
#		Incresed cutlength from 10 to 11.
#		Fixed so that size output is cutted at right decimal position.
#       2009-09-15 (1.1) Ola Lundqvist <ola@inguza.com>
#               Separates backup location from target location.
#       2010-10-23 (1.2) Ola Lundqvist <ola@inguza.com>
#               Added reduce0, reduce1, reduce2 and remove actions in addition
#               to the previous sync action.
#       2010-10-23 (1.2) Ola Lundqvist <ola@inguza.com>
#               Added clean action in addition to the previous actions.
#       2011-04-21 (1.4) Ola Lundqvist <ola@inguza.com>
#               Now always transfer using the -H option to preserve hard links.
#       2011-12-29 (1.4.1) Ola Lundqvist <ola@inguza.com>
#               Changed from /bin/sh to /bin/bash as this file is dependent
#               on bash specific things.
#       2013-06-08 (1.4.3) Ola Lundqvist <ola@inguza.com>
#               Make sure to not sync CTID in addition to VEID as it has
#               changed name in openvz.
#       2013-06-12 (1.5.0) Ola Lundqvist <ola@inguza.com>
#               Added the possibility to control whether the backup shall
#               be run or not based on a condition.
#       2016-07-06 (1.5.1) Ola Lundqist <ola@inguza.com>
#               More error conditions now give an error mail.
#       2016-07-21 (1.5.3) Ola Lundqvist <ola@inguza.com>
#               Make sure no error is printed in case there is no containers.
#       2016-11-09 (1.7.0) Ola Lundqvist <ola@inguza.com>
#               Changed CMAILDIR functionality. Now it creates a temporary
#               directory at beginning of execution. This means that it can not
#               be changed at config read (safely at least).
#       2017-08-21 (1.8.0) Ola Lundqvist <ola@inguza.com>
#               Introduced a active_lxcdirs function to support lxc instead of vz.
#               This features a check to make sure running containers are not overwritten.

FIRSTINVOCATION="no"
if [ -z "$CMAILDIR" ] ; then
    export CMAILDIR=$(/bin/mktemp -d --suffix=.opalsync.$$)
    FIRSTINVOCATION="yes"
fi

cmailcleanup() {
    if echo "$1" | grep "^/tmp" > /dev/null ; then
	rm -f "$1/"*
	rmdir "$1/"
    fi
}

active_lxcdirs() {
    HOST=$1
    LXCLOC=$2
    if [ -z "$LXCLOC" ] ; then
	LXCLOC=$(ssh $HOST /usr/bin/lxc-config lxc.lxcpath)
    fi
    if ! echo "$LXCLOC" | grep -q "/$" ; then
	LXCLOC="$LXCLOC/"
    fi
    if [ -n "$LXCLOC" ] ; then
	TFR=$(tempfile)
	ssh $HOST /usr/bin/lxc-ls --active --running --frozen --line > $TFR 2>&1
	/usr/bin/lxc-ls --active --running --frozen --line 2>&1 | while read C ; do
	    if grep "^$C$" $TFR > /dev/null ; then
		sed -i "/^$C$/d;" $TFR
		echo "WARNING! $C ignored as it is active locally." >&2
	    fi
	done
	cat $TFR | sed -e "s|^|$LXCLOC|;s|$|/rootfs/|;"
	rm -f $TFR
    fi
}

active_vzdirs() {
    HOST=$1
    VZLOC=$2
    if [ -n "$VZLOC" ] ; then
	if ! echo "$VZLOC" | grep -q "/$" ; then
	    VZLOC="$VZLOC/"
	fi
	ssh $HOST /usr/sbin/vzlist 2>&1 | grep "^[[:space:]]*[0-9]" | sed -e "s|^ *|$VZLOC|;s| .*||;"
    fi
}

method_rsyncoverssh() {
    # 1: Host (not used at all)
    # 2: Dir
    HOST="$1"
    D="$2"
    rsync -aH --numeric-ids --delete --force --stats \
	$BACKUPOPTION $EXCLUDEOPTION $EXTRAOPTIONS \
	$HOST:$D $DESTINATION$D > $TEMPSUMMARY 2> $TEMPERROR
}

method_rsyncdir() {
    # 1: Host (not used at all)
    # 2: Dir
    D="$2"
    rsync -aH --numeric-ids --delete --force --stats \
	$BACKUPOPTION $EXCLUDEOPTION $EXTRAOPTIONS \
	$D $DESTINATION$D > $TEMPSUMMARY 2> $TEMPERROR
}

method_rsyncoversmbfs() {
    # Not implemented yet
    D="$2"
    # rsync
}

printstripped() {
    echo -n $(echo "$1" | sed -e "s/\([[:alnum:][:punct:]][[:alnum:][:punct:]][[:alnum:][:punct:]][[:alnum:][:punct:]]\).*/\\1/;s/[[:punct:]]$//;")"$2"
}

printhumansize() {
    N=$(echo "$1" | sed -e 's/,//g;')
    if [ -z "$N" ] ; then
	N=0
    fi
    if [ $N -lt 0 ] ; then
	echo -n "-"
	N=$((-$N))
    fi
    if [ $N -gt $((1024 ** 4)) ] ; then
	printstripped $(($N / 1024 ** 4)).$(($N % 1024 ** 4)) "T"
    elif [ $N -gt $((1024 ** 3)) ] ; then
	printstripped $(($N / 1024 ** 3)).$(($N % 1024 ** 3 / 1000 ** 2)) "G"
    elif [ $N -gt $((1024 ** 2)) ] ; then
	printstripped $(($N / 1024 ** 2)).$(($N % 1024 ** 2 / 1000 ** 1)) "M"
    elif [ $N -gt $((1024 ** 1)) ] ; then
	printstripped $(($N / 1024 ** 1)).$(($N % 1024 ** 1)) "K"
    else
	echo -n "$N"
    fi
}

print_summary() {
    echo -n "$1 "
    echo -n "total ("
    echo -n "files: "
    printhumansize $(grep "Number of files:" $2 | sed -e 's/^[^:]*:[[:space:]]*//;s/[[:space:]].*$//;')
    echo -n ", size: "
    printhumansize $(grep "Total file size:" $2 | sed -e "s/^[^:]*:[[:space:]]*//;s/[[:space:]]*bytes//;")
    echo -n "B), transfer ("
    echo -n "files: "
    TMPFT=$(grep "Number of regular files transferred:" $2 | sed -e "s/^[^:]*:[[:space:]]*//;")
    if [ -z "$TMPFT" ] ; then
	# Backwards compatibility to old rsync version.
	TMPFT=$(grep "Number of files transferred:" $2 | sed -e "s/^[^:]*:[[:space:]]*//;")
    fi
    printhumansize $TMPFT
    echo -n ", size: "
    printhumansize $(grep "Total transferred file size:" $2 | sed -e "s/^[^:]*:[[:space:]]*//;s/[[:space:]]*bytes//;")
    echo "B)"
    if cat $3 | grep -qi "rsync error:" ; then
	NLINES=$(cat $3 | wc -l)
	if [ $NLINES -gt 11 ] ; then
	    # Too much output stripping
	    cat $3 | sed -e "5q" | sed -e "s/^/   /g;"
	    echo "     ...CUT $(($NLINES-10)) LINES OF OUTPUT..."
	    cat $3 | tail --lines 5 | sed -e "s/^/   /g;"
	else
	    cat $3 | sed -e "s/^/   /g;"
	fi
    fi
}

printout() {
    if [ -n "$MAILTO" ] ; then
	touch $CMAILDIR/$MAILTO
	echo "$*" >> $CMAILDIR/$MAILTO
    else
	echo "$*"
    fi
}

backupaction() {
    # Set defaults
    POINT="$1"
    HOST=""
    DIRS="/"
    BACKUP="%Y/%m/%d/%H:%M"
    BASEDIR=/srv/backup
    EXTRAOPTIONS=""
    BACKUPDIR=""
    BACKUPOPTION="--delete-after"
    EXCLUDEOPTION=""
    EXCLUDEFILES=""
    DESTINATION=""
    IGNORESYNCNORMAL=""
    IGNORESYNCERROR=""
    # Start of read config in order
    PART="/etc/opalsync"
    if [ -e "$PART/default" ] ; then
	. "$PART/default"
    fi
    if [ -e "$PART/default.exclude" ] ; then
	EXCLUDEOPTION="$EXCLUDEOPTION --exclude-from=$PART/default.exclude"
	EXCLUDEFILES="$EXCLUDEFILES $PART/default.exclude"
    fi
    for P in $(echo "$POINT" | sed -e "s|[^/][^/]*$||;" | tr "/" " ") ; do
	PART="$PART/$P"
	if [ -e "$PART/default" ] ; then
	    . "$PART/default"
	fi
	if [ -e "$PART/default.exclude" ] ; then
	    EXCLUDEOPTION="$EXCLUDEOPTION --exclude-from=$PART/default.exclude"
	    EXCLUDEFILES="$EXCLUDEFILES $PART/default.exclude"
	fi
    done
    . "/etc/opalsync/$POINT.conf"
    if [ -e "/etc/opalsync/$POINT.exclude" ] ; then
	EXCLUDEOPTION="$EXCLUDEOPTION --exclude-from=/etc/opalsync/$POINT.exclude"
	EXCLUDEFILES="$EXCLUDEFILES /etc/opalsync/$POINT.exclude"
    fi
    # End of read config in order
    if [ -z "$DESTINATION" ] ; then
	# Default to basedir
	DESTINATION=$BASEDIR/$POINT/current
    fi
    # Start check config
    if [ -z "$HOST" ] ; then
	pritout "$POINT: ERROR: Empty HOST variable defined in /etc/opalsync/$POINT.conf"
    elif [ -z "$METHOD" ] ; then
	printout "$POINT: ERROR: Empty METHOD variable defined in /etc/opalsync/$POINT.conf or defaults."
    elif [ -z "$DIRS" ] ; then
	printout "$POINT: WARNING: Empty DIRS variable defined in /etc/opalsync/$POINT.conf or defaults."
    # Check if this sync shall be ignored.
    elif [ -n "$IGNORESYNCNORMAL" ] && [ $IGNORESYNCNORMAL ] ; then
	printout "$POINT: Conditionally ignored."
    elif [ -n "$IGNORESYNCERROR" ] && [ $IGNORESYNCERROR ] ; then
	printout "$POINT: Error condition."
    else
    # End check config
	# Print start of POINT
	EXCLUDLINES=""
	if [ -n "$EXCLUDEFILES" ] ; then
	    EXCLUDELINES=$(cat $EXCLUDEFILES)
	fi
	EXCLUDEL=""
	for A in $EXCLUDELINES ; do
	    if [ -n "$EXCLUDEL" ] ; then
		EXCLUDEL="$EXCLUDEL "$A
	    else
		EXCLUDEL=$A
	    fi
	done
	printout "$POINT: Sync but exclude($EXCLUDEL)"
 
	for D in $DIRS ; do
	    if ! echo "$D" | grep -q "/$" ; then
		D="$D/"
	    fi
	    mkdir -p $DESTINATION$D
	    if [ -n "$BACKUP" ] ; then
		BACKUPDIR=$BASEDIR/$POINT/replaced/$(date +"$BACKUP")$D
		BACKUPOPTION="-b --backup-dir $BACKUPDIR"
		if [ ! -d "$BACKUPDIR" ] ; then
		    mkdir -p "$BACKUPDIR"
		fi
	    fi
	    TEMPSUMMARY=$(tempfile -p opal)
	    TEMPERROR=$(tempfile -p opal)
	    method_$METHOD "$HOST" "$D"
	    printout "$(print_summary $D $TEMPSUMMARY $TEMPERROR | sed -e 's/^/  /g;')"
	    rm -f $TEMPSUMMARY
	    rm -f $TEMPERROR
	done
    fi
    printout ""
}

reduceaction() {
    TMPPOINT=$1
    shift 1
    TMPBASEDIR=$1
    shift 1
    REDUCETODATE=$1
    shift 1
    REDUCEFROMDATE=$1
    shift 1
    REDUCE=$*
    for RED in $REDUCE ; do
	REDFROMDATE=$(date +"$REDUCEFROMDATE" -d "now-$RED")
	REDTODATE=$(date +"$REDUCETODATE" -d "now-$RED")
	FROMDIRS=$(echo $TMPBASEDIR/$TMPPOINT/$REDFROMDATE)
	TODIR=$TMPBASEDIR/$TMPPOINT/$REDTODATE
	#echo $FROMDIR $TODIR
	for FROMDIR in $FROMDIRS ; do
	    if [ -d "$FROMDIR" ] ; then
		mkdir -p $TODIR
		rsync -aH -H -A -X --update --numeric-ids \
		    $FROMDIR $TODIR
		if [ $? == 0 ] ; then
		    rm -Rf $FROMDIR
		    printout "$TMPPOINT: Reduced $REDFROMDATE to $REDTODATE"
		else
		    printout "$TMPPOINT: Error while reducing $REDFROMDATE to $REDTODATE"
		fi
	    fi
	done
    done
}

reduceaction0() {
    POINT="$1"
    # 30days 31days 32days ...
    REDUCE0=""
    REDUCEFROMDATE0="replaced/%Y/%m/%d/*:*/"
    REDUCETODATE0="reduced/%Y/%m/"
    BASEDIR=/srv/backup
    # Start of read config in order
    PART="/etc/opalsync"
    if [ -e "$PART/default" ] ; then
	. "$PART/default"
    fi
    for P in $(echo "$POINT" | sed -e "s|[^/][^/]*$||;" | tr "/" " ") ; do
	PART="$PART/$P"
	if [ -e "$PART/default" ] ; then
	    . "$PART/default"
	fi
    done
    . "/etc/opalsync/$POINT.conf"
    # End of read config in order
    reduceaction $POINT $BASEDIR $REDUCETODATE0 $REDUCEFROMDATE0 $REDUCE0
}

reduceaction1() {
    POINT="$1"
    # 8days 9days 10days ...
    REDUCE1=""
    REDUCEFROMDATE1="replaced/%Y/%m/%d/*:*/"
    REDUCETODATE1="reduced/%Y/%U/"
    BASEDIR=/srv/backup
    # Start of read config in order
    PART="/etc/opalsync"
    if [ -e "$PART/default" ] ; then
	. "$PART/default"
    fi
    for P in $(echo "$POINT" | sed -e "s|[^/][^/]*$||;" | tr "/" " ") ; do
	PART="$PART/$P"
	if [ -e "$PART/default" ] ; then
	    . "$PART/default"
	fi
    done
    . "/etc/opalsync/$POINT.conf"
    # End of read config in order
    reduceaction $POINT $BASEDIR $REDUCETODATE1 $REDUCEFROMDATE1 $REDUCE1
}

reduceaction2() {
    POINT="$1"
    # 366days 367days 358days ...
    REDUCE2=""
    REDUCEFROMDATE2="reduced/%Y/%m/"
    REDUCETODATE2="reduced/%Y/"
    BASEDIR=/srv/backup
    # Start of read config in order
    PART="/etc/opalsync"
    if [ -e "$PART/default" ] ; then
	. "$PART/default"
    fi
    for P in $(echo "$POINT" | sed -e "s|[^/][^/]*$||;" | tr "/" " ") ; do
	PART="$PART/$P"
	if [ -e "$PART/default" ] ; then
	    . "$PART/default"
	fi
    done
    . "/etc/opalsync/$POINT.conf"
    # End of read config in order
    reduceaction $POINT $BASEDIR $REDUCETODATE2 $REDUCEFROMDATE2 $REDUCE2
}

removeaction() {
    POINT="$1"
    # 30days 31days 32days ...
    REMOVEOLD=""
    REMOVEOLDDATE="replaced/%Y/%m/%d"
    BASEDIR=/srv/backup
    # Start of read config in order
    PART="/etc/opalsync"
    if [ -e "$PART/default" ] ; then
	. "$PART/default"
    fi
    for P in $(echo "$POINT" | sed -e "s|[^/][^/]*$||;" | tr "/" " ") ; do
	PART="$PART/$P"
	if [ -e "$PART/default" ] ; then
	    . "$PART/default"
	fi
    done
    . "/etc/opalsync/$POINT.conf"
    # End of read config in order
    printout "$POINT: remove to clean old data"
    for RM in $REMOVEOLD ; do
	RMDATE=$(date +"$REMOVEOLDDATE" -d "now-$RM")
	REMOVEOLDDIR=$BASEDIR/$POINT/$RMDATE
	if [ -d "$REMOVEOLDDIR" ] ; then
	    printout "$RMDATE"
	    rm -Rf "$REMOVEOLDDIR"
	fi
    done
    printout ""
}

cleansome() {
    TMPBASE=$1
    TMPTARG=$(echo "$2" | sed -e "s|\%[a-zA-Z]|*|g;")
    while echo $TMPTARG | grep "/" > /dev/null ; do
	rmdir --ignore-fail-on-non-empty -p $TMPBASE/$TMPTARG > /dev/null 2>&1
	TMPTARG=$(echo "$TMPTARG" | sed -e 's|/[^/]*$||')
    done
}

cleanaction() {
    POINT="$1"
    # 30days 31days 32days ...
    BACKUP="%Y/%m/%d/%H:%M"
    REDUCEFROMDATE0="replaced/%Y/%m/%d/*:*/"
    REDUCETODATE0="reduced/%Y/%m/"
    REDUCEFROMDATE1="replaced/%Y/%m/%d/*:*/"
    REDUCETODATE1="reduced/%Y/%m/"
    REDUCEFROMDATE2="replaced/%Y/%m/%d/*:*/"
    REDUCETODATE2="reduced/%Y/%m/"
    BASEDIR=/srv/backup
    # Start of read config in order
    PART="/etc/opalsync"
    if [ -e "$PART/default" ] ; then
	. "$PART/default"
    fi
    for P in $(echo "$POINT" | sed -e "s|[^/][^/]*$||;" | tr "/" " ") ; do
	PART="$PART/$P"
	if [ -e "$PART/default" ] ; then
	    . "$PART/default"
	fi
    done
    . "/etc/opalsync/$POINT.conf"
    # End of read config in order
    cleansome $BASEDIR/$POINT replaced/$BACKUP
    cleansome $BASEDIR/$POINT $REDUCEFROMDATE0
    cleansome $BASEDIR/$POINT $REDUCETODATE0
    cleansome $BASEDIR/$POINT $REDUCEFROMDATE1
    cleansome $BASEDIR/$POINT $REDUCETODATE1
    cleansome $BASEDIR/$POINT $REDUCEFROMDATE2
    cleansome $BASEDIR/$POINT $REDUCETODATE2
}

send_mails() {
    for TO in $* ; do
	TTO=$(basename $TO)
	SUBJECT="opalsync: $HOSTNAME sync ok"
	if grep -qi error $TO ; then
	    SUBJECT="opalsync: $HOSTNAME sync error"
	elif grep -qi warning $TO ; then
	    SUBJECT="opalsync: $HOSTNAME sync warning"
	fi
	mail -s "$SUBJECT" $TTO < $TO
	rm -f $TO
    done
}

ARGUMENT="$1"
case "$ARGUMENT" in
    foreach)
	shift 1
	for A in $* ; do
	    case "$A" in
		*)
		    POINTS=""
		    for D in $(find /etc/opalsync -name "*.conf") ; do
			POINTS="$POINTS $(echo $D | sed -e 's|/etc/opalsync/||;s/.conf$//;')"
		    done
		    $0 $A $POINTS
		    ;;
	    esac
	done
	;;
    sync)
	shift 1
	for TARGET in $* ; do
	    backupaction $TARGET
	done
	;;
    reduce0)
        shift 1
	for TARGET in $* ; do
	    reduceaction0 $TARGET
	done
	;;
    reduce1)
        shift 1
	for TARGET in $* ; do
	    reduceaction1 $TARGET
	done
	;;
    reduce2)
        shift 1
	for TARGET in $* ; do
	    reduceaction2 $TARGET
	done
	;;
    removeold)
        shift 1
	for TARGET in $* ; do
	    removeaction $TARGET
	done
	;;
    clean)
        shift 1
	for TARGET in $* ; do
	    cleanaction $TARGET
	done
	;;
    *)
	echo "ERROR: Unknown argument $ARGUMENT"
	echo "Syntax:"
	echo "opalsync foreach [sync] [reduce0] [reduce1] [reduce2] [clean]"
	echo " or"
	echo "opalsync sync <target1> [<target2>] [...]"
	echo " or"
	echo "opalsync reduce0 <target1> [<target2>] [...]"
	echo " or"
	echo "opalsync reduce1 <target1> [<target2>] [...]"
	echo " or"
	echo "opalsync reduce2 <target1> [<target2>] [...]"
	echo " or"
	echo "opalsync removeold <target1> [<target2>] [...]"
	echo " or"
	echo "opalsync clean <target1> [<target2>] [...]"
	if [ "$FIRSTINVOCATION" = "yes" ] ; then
	    cmailcleanup $CMAILDIR
	fi
	exit 1
	;;
esac
if [ "$FIRSTINVOCATION" = "yes" ] ; then
    send_mails $(find $CMAILDIR/ -type f -printf "%p\n")
    cmailcleanup $CMAILDIR
fi
