#!/bin/bash
set -x
# laden von Defaultwerten
. /etc/sysconfig/resticbackup
####### Konfigurationsparameter #############
#BACKUPCONFDIR=/root/TitaniaBackup
#BACKUPHOSTS=BackupHosts.example
#RESTIC=/usr/bin/restic
#RESTIC_REPO=rest:https://backer415.botter.cc:8444
#MAILFROMADDRESS=root@titania.botter.cc
#MAILTOADDRESS=root@botter.cc
#CACHEDIR=/backup/.cache
#############################################
#  Aufruf:
#  resticbackup [backuphostsdatei [host] ]
#
#  backuphostsdatei:
#
#  # Hostname
#  Hostname[:port] (PreCommand|-) KeepHourly KeepDaily KeepWeekly KeepMonthly KeepYearly [rdiff-backup_Parameter ...]
#
#  Keep[Hourly|Daily|Weekly|Monthly|Yearly]= <n|n/timespan|->
#    behalte n          Snapshots der entsprechenden Kategorie, 
#            n/timespan Snapshots der entsprechenden Kategorie in der angegebenen Zeitspanne, oder
#            -          keine 
#############################################
# source RESTIC_PASSWORD from config file
. $BACKUPCONFDIR/repopassword

if [ $1"_" != "_" ]; then
  if [ -f $BACKUPCONFDIR/$1 ]; then
    BACKUPHOSTS=$1
  fi 
fi 
# lese RESTIC_REPO aus BACKUPHOSTS Datei
if [ $(egrep ^RESTIC_REPO /root/TitaniaBackup/ResticHosts.titania) ]; then
  export $(egrep ^RESTIC_REPO $BACKUPCONFDIR/$BACKUPHOSTS)
fi
#
if [ $2"_" != "_" ]; then
  HOSTS="$(egrep -v "^$|^#|^RESTIC_REPO" $BACKUPCONFDIR/$BACKUPHOSTS | cut -f1 -d" " | egrep "^$2$")"
  if [ $HOSTS"_" = "_" ]; then
    echo Host $2 not in BackupHosts file.
    exit
  fi
else
  HOSTS="$(egrep -v "^$|^#|^RESTIC_REPO" $BACKUPCONFDIR/$BACKUPHOSTS | cut -f1 -d" " | tr "\n" "\|" | tr -d " " | sed -e 's/|$//' )"
fi
JOURNAL=$(mktemp /tmp/JOURNAL.XXXXXXXXXX)
echo "Working with BACKUPHOST $BACKUPHOSTS" >> $JOURNAL
LOCKFILE=$BACKUPCONFDIR/${BACKUPHOSTS}.lock
LOCKED=0
PRUNELOCK=$BACKUPCONFDIR/${BACKUPHOSTS}.lastprune
if [ -f $LOCKFILE ]; then
	echo Old lockfile exists. >> $JOURNAL
	PLOCK=$(cat $LOCKFILE)
	RET=0
	ps $PLOCK && RET=1
	if [ $RET = 1 ]; then
		echo Other backup job PID $PLOCK still running. Exit >> $JOURNAL
		LOCKED=1
	else
		echo No other backup job is running, removing stale lockfile. >> $JOURNAL
		rm $LOCKFILE
	fi
else
	echo Creating lockfile. >> $JOURNAL
	echo $$ > $LOCKFILE
fi
if [ $LOCKED = 0 ]; then
  RETURNOVERALL=0
  PREBACKUPTIMES=$(date +%s)
  PREBACKUPTIME=$(date)
  # loop over all host definitions in BACKUPHOSTS file
  for n in $(egrep -v "^$|^#" $BACKUPCONFDIR/$BACKUPHOSTS | egrep "($HOSTS)" | tr " " _ ); do
    # prepare variables from BACKUPHOSTS file for actual host
    STARTTIMES=$(date +%s)
    STARTTIME=$(date)
    OUTPUT=$(mktemp /tmp/RDB.XXXXXXXXXX)
    HOST=$(echo $n | cut -f1 -d_)
    if [ $HOST != ${HOST%%:*} ]; then
      PORT=${HOST##*:}
      HOST=${HOST%%:*}
    fi
    PRECOMMAND=$(echo $n | cut -f2 -d_)
    KEEPHOURLY=$(echo $n | cut -f3 -d_)
    KEEPDAILY=$(echo $n | cut -f4 -d_)
    KEEPWEEKLY=$(echo $n | cut -f5 -d_)
    KEEPMONTHLY=$(echo $n | cut -f6 -d_)
    KEEPYEARLY=$(echo $n | cut -f7 -d_)
    echo > $OUTPUT
    # do PreCommand, if configured
    echo Start time: $STARTTIME >> $OUTPUT
    if [ $PRECOMMAND != "-" ]; then
      echo "Running PreCommand" >> $OUTPUT
      echo "ssh $HOST $PRECOMMAND" >> $OUTPUT
      ssh $HOST $PRECOMMAND 2>&1 >> $OUTPUT
    else
      echo "skipping PreCommand" >> $OUTPUT
    fi
    PRECOMMANDTIMES=$(date +%s)
    PRECOMMANDTIME=$(date)
    printf "\nPreCommand time: %s\n" "$PRECOMMANDTIME" >> $OUTPUT
  
    # do backup via ssh to restic-server
    BACKUPOPTIONS=""
    [[ $PORT"_" != "_" ]] && SSHPORT=$PORT || SSHPORT=""
    BACKUPOPTIONS="$BACKUPOPTIONS export RESTIC_PASSWORD=\$LC_PAPER;" 
    BACKUPOPTIONS="$BACKUPOPTIONS $RESTIC -r $RESTIC_REPO/$HOST"
    BACKUPOPTIONS="$BACKUPOPTIONS backup --host=$HOST"
    BACKUPOPTIONS="$BACKUPOPTIONS /var/lib/rpm /var/adm/backup /usr/lib/sysimage/rpm /usr/lib/systemd"
    BACKUPOPTIONS="$BACKUPOPTIONS $(egrep "^$HOST(:$PORT)? " $BACKUPCONFDIR/$BACKUPHOSTS | cut -f8- -d" " | sed -e 's/--include=//g')"
    BACKUPOPTIONS="$BACKUPOPTIONS 2>&1 ; exit \$?"
    echo ssh root@$HOST $SSHPORT \"$BACKUPOPTIONS\" >> $OUTPUT
    LC_PAPER=$RESTIC_PASSWORD ssh root@$HOST $SSHPORT "$BACKUPOPTIONS" >> $OUTPUT 2>&1
    RETURNVALB=$?
    echo RETURNVALB $RETURNVALB
    if [ $RETURNVALB != 0 ]; then
      RETURNOVERALL=1
    fi
    sync
    BACKUPTIMES=$(date +%s)
    BACKUPTIME=$(date)
    echo >> $OUTPUT
  
    # mark restic snaphots in repo for deletion, according to KEEP* instructions from BACKUPHOSTS file
    FORGETOPTIONS="-r $RESTIC_REPO/$HOST --cache-dir=$CACHEDIR forget --host=$HOST --group-by=host"
    case $KEEPHOURLY in 
	    0|-)	;; 
	    *) 	if [[ $KEEPHOURLY == *"/"* ]]; then
    		    KEEPHOURLYWITHIN=${KEEPHOURLY##*/}
		    FORGETOPTIONS="$FORGETOPTIONS --keep-within-hourly $KEEPHOURLYWITHIN"
    		    KEEPHOURLY=${KEEPHOURLY%%/*}
  		  fi
  		  FORGETOPTIONS="$FORGETOPTIONS --keep-hourly $KEEPHOURLY";;
    esac
    case $KEEPDAILY in 
	    0|-)	;; 
	    *)	if [[ $KEEPDAILY == *"/"* ]]; then
                    KEEPDAILYWITHIN=${KEEPDAILY##*/}
                    FORGETOPTIONS="$FORGETOPTIONS --keep-within-daily $KEEPDAILYWITHIN"
                    KEEPDAILY=${KEEPDAILY%%/*}
                  fi
		  FORGETOPTIONS="$FORGETOPTIONS --keep-daily $KEEPDAILY";;
    esac
    case $KEEPWEEKLY in 
	    0|-)	;; 
	    *)	if [[ $KEEPWEEKLY == *"/"* ]]; then
                    KEEPWEEKLYWITHIN=${KEEPWEEKLY##*/}
                    FORGETOPTIONS="$FORGETOPTIONS --keep-within-weekly $KEEPWEEKLYWITHIN"
                    KEEPWEEKLY=${KEEPWEEKLY%%/*}
                  fi
		  FORGETOPTIONS="$FORGETOPTIONS --keep-weekly $KEEPWEEKLY";;
    esac
    case $KEEPMONTHLY in 
	    0|-)	;; 
	    *)	if [[ $KEEPMONTHLY == *"/"* ]]; then
                    KEEPMONTHLYWITHIN=${KEEPMONTHLY##*/}
                    FORGETOPTIONS="$FORGETOPTIONS --keep-within-monthly $KEEPMONTHLYWITHIN"
                    KEEPMONTHLY=${KEEPMONTHLY%%/*}
                  fi
		  FORGETOPTIONS="$FORGETOPTIONS --keep-monthly $KEEPMONTHLY";;
    esac
    case $KEEPYEARLY in 
	    0|-)	;; 
	    *)	if [[ $KEEPYEARLY == *"/"* ]]; then
                    KEEPYEARLYWITHIN=${KEEPYEARLY##*/}
                    FORGETOPTIONS="$FORGETOPTIONS --keep-within-yearly $KEEPYEARLYWITHIN"
                    KEEPYEARLY=${KEEPYEARLY%%/*}
                  fi
		  FORGETOPTIONS="$FORGETOPTIONS --keep-yearly $KEEPYEARLY";;
    esac
    echo "$RESTIC $FORGETOPTIONS" >> $OUTPUT
    $RESTIC $FORGETOPTIONS >> $OUTPUT 2>&1
    RETURNVALF=$?
    echo RETURNVALF $RETURNVALF
    if [ $RETURNVALF != 0 ]; then
      RETURNOVERALL=1
    fi
    FINALTIMES=$(date +%s)
    FINALTIME=$(date)
  
    # finalize host task by sending out email with output information from restic
    case $(( $RETURNVALB + $RETURNVALF )) in
	  0)	RETURNSTATE="OK     ";;
	  3)	RETURNSTATE="INCOMPLETE";;
	  *)	RETURNSTATE="FAILURE";;
    esac
    printf "\nStop time: %s\nTime PreCommand: %ss\nTime backup: %ss\nTime forget: %ss\nTime total: %ss\n" \
      "$BACKUPTIME" \
      $(($PRECOMMANDTIMES-$STARTTIMES)) \
      $(($BACKUPTIMES-$PRECOMMANDTIMES)) \
      $(($FINALTIMES-$BACKUPTIMES)) \
      $(($BACKUPTIMES-$STARTTIMES)) >> $OUTPUT
    if [ $RETURNSTATE \!= "OK" ]; then
      cat $OUTPUT | mail -n -r $MAILFROMADDRESS -s "backup log $RETURNSTATE $HOST $(($BACKUPTIMES-$STARTTIMES)) $STARTTIME" $MAILTOADDRESS
    fi
    rm $OUTPUT
    echo $RETURNSTATE $HOST $(($BACKUPTIMES-$STARTTIMES)) $STARTTIME >> $JOURNAL
    sync
  done
  
  # after looping over all hosts for backup und deletion marking of snapshots, prune marked snapshots from repo
  echo >> $JOURNAL
  PREPRUNETIMES=$(date +%s)
  PREPRUNETIME=$(date)
  echo $PREPRUNETIME >> $JOURNAL
  
  if [ -f $PRUNELOCK ]; then
	  LASTPRUNE=$(cat $PRUNELOCK)
  else
	  echo No LASTPRUNE file found, Setting to 19700101. >> $JOURNAL
	  LASTPRUNE=19700101
  fi
  DATENOW=$(date +%Y%m%d)
  if [ $DATENOW = $LASTPRUNE ]; then
	  echo "Pruning already done today ($DATENOW = $LASTPRUNE). Skipping pruning step." >> $JOURNAL
  else
	  echo Pruning. >> $JOURNAL
	  for HOST in $(egrep -v "^$|^#" $BACKUPCONFDIR/$BACKUPHOSTS | egrep "($HOSTS)" | cut -f1 -d" "); do
  		  PRUNEOPTIONS=""
  		  PRUNEOPTIONS="$PRUNEOPTIONS --cache-dir=$CACHEDIR -r $RESTIC_REPO/$HOST prune"
  		  echo $RESTIC $PRUNEOPTIONS >> $JOURNAL
  		  $RESTIC $PRUNEOPTIONS >> $JOURNAL 2>&1
  		  RETURNVAL=$?
  		  echo RETURNVAL $RETURNVAL
  		  if [ $RETURNVAL != 0 ]; then
    			  RETURNOVERALL=1
  		  fi
	  done
	  echo Pruning done. Creating LASTPRUNE file. >> $JOURNAL
	  echo $DATENOW > $PRUNELOCK
  fi
  PRUNETIMES=$(date +%s)
  PRUNETIME=$(date)
  echo $PRUNETIME >> $JOURNAL
  
  # finalize backup run by sending out overall notification
  case $RETURNOVERALL in
	  0)	RETURNSTATE="OK     ";;
	  *)	RETURNSTATE="FAILURE";;
  esac
  #sleep 10
  echo Removing lockfile. >> $JOURNAL
  rm $LOCKFILE >> $JOURNAL
else # LOCKED=1
  RETURNSTATE="FAILURE";
  # set time marks to avoid error in calculation below (subject of mail)
  PREPRUNETIMES=0
  PREBACKUPTIMES=0
  PRUNETIMES=0
  PREPRUNETIMES=0
fi
cat $JOURNAL | mail -n -r $MAILFROMADDRESS -s "Journal of backup activities $BACKUPHOSTS $RETURNSTATE $(($PREPRUNETIMES-$PREBACKUPTIMES)) $(($PRUNETIMES-$PREPRUNETIMES))" $MAILTOADDRESS
rm $JOURNAL
