ZFS sync script

From Lolly's Wiki
Jump to navigationJump to search

Sync

Like all of my scripts this script is coming without any guaranties!!! You can use it on your own risk!


About the script

  • It uses mbuffer. It is easy to compile.
  • It uses gawk.
  • The variable SECURE defines if you want to use ssh to encrypt your stream. Set it to yes or no.
  • To mark the datasets to copy from the backup host use this on the source:
# /usr/sbin/zfs set de.timmann:auto-backup=<backup host> <dataset>
  • Run the script on the destination/backup host.
  • If you don't want to use root as backup-user on source host do this to create a zfssync user (Solaris syntax):
# useradd -m zfssync
# passwd -N zfssync
# usermod -K type=normal zfssync
  • Make an ssh-key exchange to login without password for SRC_USER.

Good luck!

zfs_sync.sh

#!/bin/bash
 
# Written by Lars Timmann <L@rs.Timmann.de> 2013
 
# This script is a rotten bunch of code... rewrite it!

# Some defaults
BACKUP_PROPERTY="de.timmann:auto-backup"
BACKUP_SNAPSHOT_NAME="zfssync"
MBUFFER_PORT=10001
MBUFFER=/opt/mbuffer/bin/mbuffer;
SRC_USER=zfssync
INITIAL_COPIES=3
# Default yes means use SSH for encryption over the net. Every other value means just mbuffer.
SECURE="yes" 
LOCAL_SYNC="no" 
MBUFFER_PORT=10001
MBUFFER_OPTS="-v 0 --md5 -s 128k -m 256M"
BACKUP_PROPERTY="de.timmann:auto-backup"
 
ZFS=/usr/sbin/zfs
SSH="/usr/bin/ssh -xc blowfish"
AWK=/usr/bin/gawk
#AWK=/opt/sfw/bin/gawk
GREP=/usr/bin/grep
DATE=/usr/bin/date
MD5="/usr/bin/digest -a md5"
ROUTE=/usr/sbin/route
MBUFFER="/opt/mbuffer/bin/mbuffer"
 
MYHOST=$(/usr/bin/hostname)
MYNAME=$(/usr/bin/basename $0)

function usage () {
  if [ $# -gt 0 ]
  then
    if [ "_${1}_" != "_help_" ]
    then
      echo "Error: ${MYNAME} : $*"
    fi
  else
    echo "Error: ${MYNAME} : Check parameters"
  fi
  cat <<EOU
Usage: ${MYNAME} <params>
Where params is from this set of parameters:
-s|--src-ip <IP>        The host from where we want to sync
-d|--dst-ip <IP>        The IP on this host where the remote mbuffer should try to connect to
                        If omitted the IP to use is guessed via route get.
-u|--user <user>        The user on "--src-ip" which has rights to send a zfs.
                        It must be able to login via ssh with public key.
                        On Solaris it is the profile "ZFS File System Management"
                        Try this on the "--src-ip":
                        # roleadd \
                            -d /export/home/zfssync \
                            -c "User for zfs send/recv" \
                            -s /bin/bash \
                            -m \
                            -P "ZFS File System Management" \
                             zfssync
                        # rolemod -K type=normal zfssync
                        # passwd -N zfssync
        
                        And then put the ssh-public-key from this host into
                           /export/home/zfssync/.ssh/authorized_keys
                        on the "--src-ip".
                        Remember to set the permissions on .ssh to 700 and .ssh/authorized_keys to 600.
                        The Homedir of the user must not be world writeable.

-sp|--src-pool <zpool>  The zpool we want to sync from "--src-ip".
-dp|--dst-pool <zpool>  The zpool on this host where we want to sync to ${MYNAME}.
-mbp|--mbuffer-port <port>
                        If the default port 10001 is in use use another port.
-mb|--mbuffer-path <path>
                        Path of mbuffer binary including binary itself.
-mbbw|--mbuffer-bwlimit <rate>
                        Limit the read bandwith of mbuffer (mbuffer option -r) 
                        From mbuffer --help: limit read rate to <rate> B/s, where <rate> can be given in b,k,M,G
-bp|--backup-property <property>
                        This defaults to ${BACKUP_PROPERTY}.
                        You have to set this property on all ZFS datasets to ${MYHOST}.
                        # /usr/sbin/zfs set ${BACKUP_PROPERTY}=${MYHOST} <dataset>
                        This is inherited as usual.
-bsn|--backup-snap-name <snapshotname>
                        This is the name of the snapshot which we use to sync.
                        This defaults to ${BACKUP_SNAPSHOT_NAME}.
                        Never delete this snapshot manually or you will break the sync and restart
                        from the beginning.
-i|--insecure           Not for production environments! No ssh tunneling. No encryption over the net!
EOU
##-l|--local            Just do a local zfs send/recv...
  exit 1
}

while [ $# -gt 0 ]
do
  #if [ $# -ge 2 ]; then value=$2; fi
  case $1 in
  --help|-h)
    usage "help"
    ;;
  -l|--local)
    LOCAL_SYNC="yes"
    SRC_HOST="localhost"
    param="dummy"
    shift;
    ;;
  -i|--insecure|--fuck-off-security)
    SECURE="no"
    param="dummy"
    shift;
    ;;
  --?*=?*|-?*=?*)
    param=${1%=*}
    value=${1#*=}
    shift;
    ;;
  --?*=|-?*=)
    param=${1%=*}
    usage "${param} needs a vlaue!"
    ;;
  *)
    param=$1
    if [ $# -ge 2 -a "_${2%-*}_" != "__" ]
    then
      value=$2
      shift
    fi
    shift
    ;;
  esac

  
  case $param in
  -s|--src-ip)
    if [ -z $value ] ; then usage "Param ${param} needs a value" ; fi
    SRC_HOST=${value}
    ;;
  -d|--dst-ip)
    if [ -z $value ] ; then usage "Param ${param} needs a value" ; fi
    DST_HOST=${value};
    ;;
  -u|--user)
    if [ -z $value ] ; then usage "Param ${param} needs a value" ; fi
    SRC_USER=${value}
    ;;
  -sp|--src-pool)
    if [ -z $value ] ; then usage "Param ${param} needs a value" ; fi
    SRC_POOL=${value}
    ;;
  -bsn|--backup-snap-name)
    if [ -z $value ] ; then usage "Param ${param} needs a value" ; fi
    BACKUP_SNAPSHOT_NAME=${value}
    ;;
  -dp|--dst-pool)
    if [ -z $value ] ; then usage "Param ${param} needs a value" ; fi
    DST_POOL=${value}
    ;;
  -mbp|--mbuffer-port)
    if [ -z $value ] ; then usage "Param ${param} needs a value" ; fi
    MBUFFER_PORT=${value}
    ;;
  -mb|--mbuffer-path)
    if [ -z $value ] ; then usage "Param ${param} needs a value" ; fi
    MBUFFER=${value}
    ;;
  -mbbw|--mbuffer-bwlimit)
    if [ -z $value ] ; then usage "Param ${param} needs a value" ; fi
    MBUFFER_OPTS="${MBUFFER_OPTS} -r ${value}"
    ;;
  -bp|--backup-property)
    if [ -z $value ] ; then usage "Param ${param} needs a value" ; fi
    BACKUP_PROPERTY=${value}
    ;;
  dummy)
    ;;
  *)
    usage "Unknown parameter $1" 
  esac
done

if [ "_${LOCAL_SYNC}_" == "no" ]
then
  if [ -z ${SRC_HOST} ]; then usage "-s|--src-ip is missing" ; fi

  # Guess the right IP for communication with source host
  if [ -z ${DST_HOST} ]; then
    DST_HOST=$(${ROUTE} -vn get ${SRC_HOST} | ${AWK} '{ip=$2}END{print ip}')
    if [ -z ${DST_HOST} ]; then
      usage "-d|--dst-ip is missing"
    fi
  fi 
fi
if [ -z ${SRC_POOL} ]; then usage "-sp|--src-pool is missing" ; fi
if [ -z ${DST_POOL} ]; then usage "-dp|--dst-pool is missing" ; fi

 
 
 
 
SRC_DATASETS=/tmp/${MYNAME}_${DST_POOL/\//_}_src_ds.out
DST_DATASETS=/tmp/${MYNAME}_${DST_POOL/\//_}_dst_ds.out
LOCK_FILE=/var/run/${MYNAME}_${DST_POOL/\//_}.lck
TMP_FILE1=/tmp/${MYNAME}_${DST_POOL/\//_}.tmp1
TMP_FILE2=/tmp/${MYNAME}_${DST_POOL/\//_}.tmp2
 
START_TIME=$(${AWK} 'BEGIN{printf systime();}')
${AWK} -v time=${START_TIME} 'BEGIN{print "START:",strftime("%d.%m.%Y %H:%M.%S",time)}'
 
 
# Clean up on signal
# -------------------------
trap 'echo "\n--- Got signal: Exiting ...\n"; \
      date ; \
      sleep 3; kill -9 ${!} 2>/dev/null; \
      /usr/bin/rm -f ${LOCK_FILE}; \
      exit 1' 1 2 3 13 14 15 18
###########################
 
 
if [ -f ${LOCK_FILE} ] ; then
  echo "$0 is allready running as PID $(/usr/bin/cat ${LOCK_FILE}) look in ${LOCK_FILE}"
  exit 1
else
  echo $$ > ${LOCK_FILE}
fi
 

if [ "_${LOCAL_SYNC}_" == "_yes_" ]
then
  ${ZFS} list -rH -t filesystem,snapshot,volume -o name,type,${BACKUP_PROPERTY} -s creation ${SRC_POOL} > ${SRC_DATASETS} &
else
  ${SSH} ${SRC_USER:+"${SRC_USER}@"}${SRC_HOST} "${ZFS} list -rH -t filesystem,snapshot,volume -o name,type,${BACKUP_PROPERTY} -s creation ${SRC_POOL}" > ${SRC_DATASETS} &
fi

${ZFS} list -rH -t filesystem,snapshot,volume -o name,type -s creation ${DST_POOL} > ${DST_DATASETS} &
wait
 
function convert_to_poolname () {
  from_zfs=$1
  search=$2
  replace=$3
  echo ${from_zfs} | sed -e "s#^${search}#${replace}#g"
}
 
function is_available () {
  snapshot=$1
  list=$2
  ${AWK} -v snapshot=${snapshot} 'BEGIN{rc=1;}$1 == snapshot{print $1; rc=0;}END{exit rc;}' ${list}
  return $?
}
 
function expire_dst_pool_snapshots () {
  days_to_keep=$1
  min_to_keep=$2
  for expired_zfs in $(
  ${ZFS} list -o creation,name -S creation -t snapshot | \
  ${AWK} \
        -v days_to_keep=${days_to_keep} \
        -v min_to_keep=${min_to_keep} \
        -v DST_POOL="^${DST_POOL}" \
  '
  BEGIN{
    split("Jan:Feb:Mar:Apr:May:Jun:Jul:Aug:Sep:Oct:Nov:Dec",mon,":");
    for(m in mon){
      month[mon[m]]=m
    };
    expire_date=systime()-days_to_keep*60*60*24
  }
  $NF ~ DST_POOL {
    filesystem=$NF;
    gsub(/@.*$/,"",filesystem);
    split($4,time,":");
    filesystem_date=mktime(sprintf("%d %02d %02d %02d %02d 00", $5, month[$2], $3, time[1], time[2]));
    count[filesystem]++;
    if(filesystem_date < expire_date && count[filesystem] > min_to_keep )
    {
      print $NF;
    }
  }')
  do
    printf "$(${DATE}) Destroying snapshot ${expired_zfs}\n"
    ${ZFS} destroy ${expired_zfs}
  done
}
 
function get_src_list () {
  ${AWK} -v backup_server=${MYHOST} '
    ( $2=="filesystem" || $2=="volume" ) && $3==backup_server {
      path[$1]=1;
      for(name in path){
        # delete name from list, if name is substring of $1
        if( index($1,name)==1 && name != $1 && path[name]!=0 ){
          path[name]=0;
        }
      }
    }
    END{
      for(name in path){
        if(path[name]==1) print name
      }
    }
  ' ${SRC_DATASETS}
}
 
function first_snapshot () {
  ${AWK} -v zfs="${1}@" '
    $2=="snapshot" && $1 ~ zfs {
      first=$1;
      # und raus...
      nextfile;
    }
    END{
      print first;
    }
  ' $2
}
 
function last_snapshot () {
  ${AWK} -v zfs="^${1}" -F '[@ \t]' '
    $3 == "snapshot" && $1 ~ zfs {
      last=$1"@"$2;
    }
    END{
      printf last;
    }
  ' $2
}
 
function get_incremental_snapshot () {
  src_host=$1
  src_datasets=$2
  first=$3
  last=$4
  dst_pool=$5
  dst_datasets=$6
 
  if [ $# -lt 6 ] ; then
    echo "Called from line ${BASH_LINENO[$i]} with $# Arguments"
    end 1
  fi
 
  src_zfs=$(echo ${first} | ${AWK} -F'@' '{print $1}')
  first_snap=$(echo ${first} | ${AWK} -F'@' '{print FS""$2}')
 
  echo "Getting snapshot ${zfs}..."
 
  if [ "_${LOCAL_SYNC}_" == "_yes_" ]
  then
    ${ZFS} send -I ${first_snap} ${last} | ${ZFS} recv -vFd ${dst_pool}
  else 
    if [ "_${SECURE}_" == "_yes_" ]
    then
      # setup receiver
      ${MBUFFER} ${MBUFFER_OPTS} -l ${TMP_FILE1} -I 127.0.0.1:${MBUFFER_PORT} | \
        ${ZFS} recv -vFd ${dst_pool} 2>&1 &
   
      # start sender
      ${SSH} ${SRC_USER:+"${SRC_USER}@"}${SRC_HOST} \
        -R ${MBUFFER_PORT}:127.0.0.1:${MBUFFER_PORT} \
        "${ZFS} send -I ${first_snap} ${last} | ${MBUFFER} ${MBUFFER_OPTS} -O 127.0.0.1:${MBUFFER_PORT} 2>&1" >${TMP_FILE2} &
    else
      # setup receiver
      ${MBUFFER} ${MBUFFER_OPTS} -l ${TMP_FILE1} -I ${MBUFFER_PORT} | \
        ${ZFS} recv -vFd ${dst_pool} 2>&1 &
   
      # start sender
      ${SSH} ${SRC_USER:+"${SRC_USER}@"}${SRC_HOST} \
        "${ZFS} send -I ${first_snap} ${last} | ${MBUFFER} ${MBUFFER_OPTS} -O ${DST_HOST}:${MBUFFER_PORT} 2>&1" >${TMP_FILE2} &
    fi
    wait
    local_md5=$(grep md5 ${TMP_FILE1})
    remote_md5=$(grep md5 ${TMP_FILE2})
    local_summary=$(grep summary ${TMP_FILE1})
    remote_summary=$(grep summary ${TMP_FILE2})
    printf "remote %s\nlocal  %s\n" "${remote_md5}" "${local_md5}"
    printf "remote %s\nlocal  %s\n" "${remote_summary}" "${local_summary}"
    rm -f ${TMP_FILE1} ${TMP_FILE2}
  fi
}
 
function get_initial_snapshot () {
  src_host=$1
  src_datasets=$2
  zfs=$3
  dst_pool=$4
  dst_datasets=$5
  if [ -z "$(is_available ${zfs} ${dst_datasets})" ] ; then
    echo "Getting snapshot ${zfs}..."
    if [ "_${LOCAL_SYNC}_" == "_yes_" ]
    then
      ${ZFS} send -R ${zfs} | ${ZFS} recv -vFd ${dst_pool}
    else 
      if [ "_${SECURE}_" == "_yes_" ]
      then
        # setup receiver
        ${MBUFFER} ${MBUFFER_OPTS} -l ${TMP_FILE1} -I 127.0.0.1:${MBUFFER_PORT} | \
          ${ZFS} recv -vFd ${dst_pool} 2>&1 &
   
        # start sender
        ${SSH} ${SRC_USER:+"${SRC_USER}@"}${SRC_HOST} \
          -R ${MBUFFER_PORT}:127.0.0.1:${MBUFFER_PORT} \
          "${ZFS} send -R ${zfs} | ${MBUFFER} ${MBUFFER_OPTS} -O 127.0.0.1:${MBUFFER_PORT} 2>&1" >${TMP_FILE2} &
      else
        # setup receiver
        ${MBUFFER} ${MBUFFER_OPTS} -l ${TMP_FILE1} -I ${MBUFFER_PORT} | \
          ${ZFS} recv -vFd ${dst_pool} 2>&1 &
   
        # start sender
        ${SSH} ${SRC_USER:+"${SRC_USER}@"}${SRC_HOST} \
          "${ZFS} send -R ${zfs} | ${MBUFFER} ${MBUFFER_OPTS} -O ${DST_HOST}:${MBUFFER_PORT} 2>&1" >${TMP_FILE2} &
      fi
      wait
      local_md5=$(grep md5 ${TMP_FILE1})
      remote_md5=$(grep md5 ${TMP_FILE2})
      local_summary=$(grep summary ${TMP_FILE1})
      remote_summary=$(grep summary ${TMP_FILE2})
      printf "remote %s\nlocal  %s\n" "${remote_md5}" "${local_md5}"
      printf "remote %s\nlocal  %s\n" "${remote_summary}" "${local_summary}"
      rm -f ${TMP_FILE1} ${TMP_FILE2}
    fi
  fi
}
 
function timestamp () {
  echo $(${DATE} '+%Y%m%d-%H:%M:%S')
}
 
function expire_backup_snapshots () {
  src_host=$1
  src_datasets=$2
  dst_datasets=$3
  src_last_to_keep=$4
  dst_pool=$5
 
  src_zfs=$(echo ${src_last_to_keep} | ${AWK} -F'@' '{print $1}')
  dst_zfs=$(convert_to_poolname ${src_zfs} ${SRC_POOL} ${dst_pool})
  dst_last_to_keep=$(convert_to_poolname ${src_last_to_keep} ${SRC_POOL} ${dst_pool})
 
  echo "Deleting old backup snapshots before ${dst_last_to_keep}"
  if ( ${ZFS} list -o name ${dst_last_to_keep} >/dev/null 2>&1 ) ; then
    for src_backup_snapshot in $(${AWK} -v src_backup="${src_zfs}@${BACKUP_SNAPSHOT_NAME}" -v src_last_to_keep="${src_last_to_keep}" '
        $1 == src_last_to_keep {
          exit 0;
        }
        $1 ~ src_backup {
          print $1;
        }
        ' ${src_datasets}) 
    do
      printf "\tDeleting on src ${src_backup_snapshot} ..."
      if [ "_${LOCAL_SYNC}_" == "_yes_" ]
      then
        ${ZFS} destroy ${src_backup_snapshot}
        status=$?
      else 
        ${SSH} ${SRC_USER:+"${SRC_USER}@"}${SRC_HOST} "${ZFS} destroy ${src_backup_snapshot}"
        status=$?
      fi
      if [ ${status} -eq 0 ] ; then
        echo "done"
      else
        echo "failed"
      fi
    done
    for dst_backup_snapshot in $(${AWK} -v dst_backup="${dst_zfs}@${BACKUP_SNAPSHOT_NAME}" -v dst_last_to_keep=${dst_last_to_keep} '
        $1 == dst_last_to_keep {
          exit 0;
        }
        $1 ~ dst_backup {
          print $1;
        }
        ' ${dst_datasets}) 
    do
      printf "\tDeleting on destination ${dst_backup_snapshot} ..."
      if ( ${ZFS} destroy ${dst_backup_snapshot} ) ; then
        echo "done"
      else
        echo "failed"
      fi
    done
  else
    echo "Strange we do not have the copy of ${dst_last_to_keep} => STOP!"
  fi
}
 
function end () {
  /usr/bin/rm -f ${LOCK_FILE}
  exit $1
}
 
for src_zfs in $(get_src_list) ; do
  echo "Evaluating ${src_zfs}"
  dst_zfs=$(convert_to_poolname ${src_zfs} ${SRC_POOL} ${DST_POOL})
  last_src=$(last_snapshot ${src_zfs} ${SRC_DATASETS})
  last_dst=$(last_snapshot ${dst_zfs} ${DST_DATASETS})
  last_backup_src=$(${AWK} -v zfs="${src_zfs}@${BACKUP_SNAPSHOT_NAME}" '$1 ~ zfs{last=$1}END{printf last}' ${SRC_DATASETS})
  last_backup_dst=$(${AWK} -v zfs="${dst_zfs}@${BACKUP_SNAPSHOT_NAME}" '$1 ~ zfs{last=$1}END{printf last}' ${DST_DATASETS})
  last_dst_on_src=$(convert_to_poolname ${last_dst} ${DST_POOL} ${SRC_POOL})
  this_backup_src=${src_zfs}@${BACKUP_SNAPSHOT_NAME}_$(timestamp)
 
  # Create snapshot for incremental backups
  if [ "_${LOCAL_SYNC}_" == "_yes_" ]
  then
    ${ZFS} snapshot ${this_backup_src}
  else
    ${SSH} ${SRC_USER:+"${SRC_USER}@"}${SRC_HOST} "${ZFS} snapshot ${this_backup_src}"
  fi
  if [ -z "${last_src}" ] ; then
    last_src=${this_backup_src}
  fi
  if [ -n "$(is_available ${dst_zfs} ${DST_DATASETS})" -a -z "${last_dst}" ] ; then
    echo "zfs is on dst, but no snapshots. Getting ${last_src}..."
    get_initial_snapshot ${SRC_HOST} ${SRC_DATASETS} ${last_src} ${DST_POOL} ${DST_DATASETS}
  # Look for last backup snapshot on destination
  elif [ -n "${last_backup_dst}" ] ; then
    # Name of last backup snapshot on src
    last_dst_backup_on_src=$(convert_to_poolname ${last_backup_dst} ${DST_POOL} ${SRC_POOL})
 
    # If converted name is not empty and snapshot is in the list of src snapshots
    # then get all snapshots from last backup until now
    if [ -n "${last_dst_backup_on_src}" ] ; then
      if [ -n "$(is_available ${last_dst_backup_on_src} ${SRC_DATASETS})" ] ; then
        # Get the snapshot of this backup
        printf "%s\tsnapshot\n" ${this_backup_src} >> ${SRC_DATASETS}
        get_incremental_snapshot ${SRC_HOST} ${SRC_DATASETS} ${last_dst_backup_on_src} ${this_backup_src} ${DST_POOL} ${DST_DATASETS} && \
          expire_backup_snapshots ${SRC_HOST} ${SRC_DATASETS} ${DST_DATASETS} ${this_backup_src} ${DST_POOL}
      fi
    fi
  elif [ -n "$(is_available ${dst_zfs} ${DST_DATASETS})" ] ; then
    # No last backup snapshot on dst but we have snapshots
    if [ -n "$(is_available ${last_dst_on_src} ${SRC_DATASETS})" ] ; then
      echo "Try to backup from ${last_dst_on_src} to ${this_backup_src}"
      first=${last_dst_on_src}
      last=${last_src}
      get_incremental_snapshot ${SRC_HOST} ${SRC_DATASETS} ${first} ${last} ${DST_POOL} ${DST_DATASETS} && \
        expire_backup_snapshots ${SRC_HOST} ${SRC_DATASETS} ${DST_DATASETS} ${this_backup_src} ${DST_POOL}
      # Get the snapshot of this backup
      printf "%s\tsnapshot\n" ${this_backup_src} >> ${SRC_DATASETS}
      get_incremental_snapshot ${SRC_HOST} ${SRC_DATASETS} ${last} ${this_backup_src} ${DST_POOL} ${DST_DATASETS} && \
        expire_backup_snapshots ${SRC_HOST} ${SRC_DATASETS} ${DST_DATASETS} ${this_backup_src} ${DST_POOL}
    else
      echo "OK I tried hard... now it is your job..."
    fi
  else
    # No existing copies for this zfs. Get the last <INITIAL_COPIES> copies
    first=$(${AWK} -v zfs=${src_zfs} -v intitial_copies=$((${INITIAL_COPIES}-1)) '
      $1 ~ zfs && $2=="snapshot" {
        last[++count]=$1;
      }
      END {
        if(count>intitial_copies){
          print last[count-intitial_copies]
        }else{
          print last[1]
        }
      }' ${SRC_DATASETS})
    last=$( ${AWK} -v zfs=${src_zfs} '$1 ~ zfs && $2=="snapshot"{last=$1}END{printf last}' ${SRC_DATASETS} )
    get_initial_snapshot ${SRC_HOST} ${SRC_DATASETS} ${first} ${DST_POOL} ${DST_DATASETS}
    get_incremental_snapshot ${SRC_HOST} ${SRC_DATASETS} ${first} ${last} ${DST_POOL} ${DST_DATASETS} && \
      expire_backup_snapshots ${SRC_HOST} ${SRC_DATASETS} ${DST_DATASETS} ${this_backup_src} ${DST_POOL}
    # Get the snapshot of this backup
    printf "%s\tsnapshot\n" ${this_backup_src} >> ${SRC_DATASETS}
    get_incremental_snapshot ${SRC_HOST} ${SRC_DATASETS} ${last} ${this_backup_src} ${DST_POOL} ${DST_DATASETS} && \
      expire_backup_snapshots ${SRC_HOST} ${SRC_DATASETS} ${DST_DATASETS} ${this_backup_src} ${DST_POOL}
  fi  
 
  echo
  echo --------------------------------------------------------------------------------
  date
  echo
done
 
# expire_dst_pool_snapshots days_to_keep min_to_keep
expire_dst_pool_snapshots 34 70
 
END_TIME=$(${AWK} 'BEGIN{printf systime();}')
${AWK} -v time=${END_TIME} 'BEGIN{print "END  :",strftime("%d.%m.%Y %H:%M.%S",time)}'
${AWK} -v start=${START_TIME} -v end=${END_TIME} 'BEGIN{print "DURATION:",strftime("%H:%M.%S",end-start-3600*strftime("%H",0))}'
end 0