ZFS sync script
From Lolly's Wiki
Like all of my scripts this script is coming without any guaranties!!! You can use it on your own risk!
About the script
- It uses mbuffer. It is easy to compile.
- It uses gawk.
- The variable SECURE defines if you want to use ssh to encrypt your stream. Set it to yes or no.
- To mark the datasets to copy from the backup host use this on the source:
# /usr/sbin/zfs set de.timmann:auto-backup=<backup host> <dataset>
- Run the script on the destination/backup host.
- If you don't want to use root as backup-user on source host do this to create a zfssync user (Solaris syntax):
# useradd -m zfssync
# passwd -N zfssync
# usermod -K type=normal zfssync
- Make an ssh-key exchange to login without password for SRC_USER.
Good luck!
zfs_sync.sh
#!/bin/bash
# Written by Lars Timmann <L@rs.Timmann.de> 2013
# This script is a rotten bunch of code... rewrite it!
# Some defaults
BACKUP_PROPERTY="de.timmann:auto-backup"
BACKUP_SNAPSHOT_NAME="zfssync"
MBUFFER_PORT=10001
MBUFFER=/opt/mbuffer/bin/mbuffer;
SRC_USER=zfssync
INITIAL_COPIES=3
# Default yes means use SSH for encryption over the net. Every other value means just mbuffer.
SECURE="yes"
LOCAL_SYNC="no"
MBUFFER_PORT=10001
MBUFFER_OPTS="-v 0 --md5 -s 128k -m 256M"
BACKUP_PROPERTY="de.timmann:auto-backup"
ZFS=/usr/sbin/zfs
SSH="/usr/bin/ssh -xc blowfish"
AWK=/usr/bin/gawk
#AWK=/opt/sfw/bin/gawk
GREP=/usr/bin/grep
DATE=/usr/bin/date
MD5="/usr/bin/digest -a md5"
ROUTE=/usr/sbin/route
MBUFFER="/opt/mbuffer/bin/mbuffer"
MYHOST=$(/usr/bin/hostname)
MYNAME=$(/usr/bin/basename $0)
function usage () {
if [ $# -gt 0 ]
then
if [ "_${1}_" != "_help_" ]
then
echo "Error: ${MYNAME} : $*"
fi
else
echo "Error: ${MYNAME} : Check parameters"
fi
cat <<EOU
Usage: ${MYNAME} <params>
Where params is from this set of parameters:
-s|--src-ip <IP> The host from where we want to sync
-d|--dst-ip <IP> The IP on this host where the remote mbuffer should try to connect to
If omitted the IP to use is guessed via route get.
-u|--user <user> The user on "--src-ip" which has rights to send a zfs.
It must be able to login via ssh with public key.
On Solaris it is the profile "ZFS File System Management"
Try this on the "--src-ip":
# roleadd \
-d /export/home/zfssync \
-c "User for zfs send/recv" \
-s /bin/bash \
-m \
-P "ZFS File System Management" \
zfssync
# rolemod -K type=normal zfssync
# passwd -N zfssync
And then put the ssh-public-key from this host into
/export/home/zfssync/.ssh/authorized_keys
on the "--src-ip".
Remember to set the permissions on .ssh to 700 and .ssh/authorized_keys to 600.
The Homedir of the user must not be world writeable.
-sp|--src-pool <zpool> The zpool we want to sync from "--src-ip".
-dp|--dst-pool <zpool> The zpool on this host where we want to sync to ${MYNAME}.
-mbp|--mbuffer-port <port>
If the default port 10001 is in use use another port.
-mb|--mbuffer-path <path>
Path of mbuffer binary including binary itself.
-mbbw|--mbuffer-bwlimit <rate>
Limit the read bandwith of mbuffer (mbuffer option -r)
From mbuffer --help: limit read rate to <rate> B/s, where <rate> can be given in b,k,M,G
-bp|--backup-property <property>
This defaults to ${BACKUP_PROPERTY}.
You have to set this property on all ZFS datasets to ${MYHOST}.
# /usr/sbin/zfs set ${BACKUP_PROPERTY}=${MYHOST} <dataset>
This is inherited as usual.
-bsn|--backup-snap-name <snapshotname>
This is the name of the snapshot which we use to sync.
This defaults to ${BACKUP_SNAPSHOT_NAME}.
Never delete this snapshot manually or you will break the sync and restart
from the beginning.
-i|--insecure Not for production environments! No ssh tunneling. No encryption over the net!
EOU
##-l|--local Just do a local zfs send/recv...
exit 1
}
while [ $# -gt 0 ]
do
#if [ $# -ge 2 ]; then value=$2; fi
case $1 in
--help|-h)
usage "help"
;;
-l|--local)
LOCAL_SYNC="yes"
SRC_HOST="localhost"
param="dummy"
shift;
;;
-i|--insecure|--fuck-off-security)
SECURE="no"
param="dummy"
shift;
;;
--?*=?*|-?*=?*)
param=${1%=*}
value=${1#*=}
shift;
;;
--?*=|-?*=)
param=${1%=*}
usage "${param} needs a vlaue!"
;;
*)
param=$1
if [ $# -ge 2 -a "_${2%-*}_" != "__" ]
then
value=$2
shift
fi
shift
;;
esac
case $param in
-s|--src-ip)
if [ -z $value ] ; then usage "Param ${param} needs a value" ; fi
SRC_HOST=${value}
;;
-d|--dst-ip)
if [ -z $value ] ; then usage "Param ${param} needs a value" ; fi
DST_HOST=${value};
;;
-u|--user)
if [ -z $value ] ; then usage "Param ${param} needs a value" ; fi
SRC_USER=${value}
;;
-sp|--src-pool)
if [ -z $value ] ; then usage "Param ${param} needs a value" ; fi
SRC_POOL=${value}
;;
-bsn|--backup-snap-name)
if [ -z $value ] ; then usage "Param ${param} needs a value" ; fi
BACKUP_SNAPSHOT_NAME=${value}
;;
-dp|--dst-pool)
if [ -z $value ] ; then usage "Param ${param} needs a value" ; fi
DST_POOL=${value}
;;
-mbp|--mbuffer-port)
if [ -z $value ] ; then usage "Param ${param} needs a value" ; fi
MBUFFER_PORT=${value}
;;
-mb|--mbuffer-path)
if [ -z $value ] ; then usage "Param ${param} needs a value" ; fi
MBUFFER=${value}
;;
-mbbw|--mbuffer-bwlimit)
if [ -z $value ] ; then usage "Param ${param} needs a value" ; fi
MBUFFER_OPTS="${MBUFFER_OPTS} -r ${value}"
;;
-bp|--backup-property)
if [ -z $value ] ; then usage "Param ${param} needs a value" ; fi
BACKUP_PROPERTY=${value}
;;
dummy)
;;
*)
usage "Unknown parameter $1"
esac
done
if [ "_${LOCAL_SYNC}_" == "no" ]
then
if [ -z ${SRC_HOST} ]; then usage "-s|--src-ip is missing" ; fi
# Guess the right IP for communication with source host
if [ -z ${DST_HOST} ]; then
DST_HOST=$(${ROUTE} -vn get ${SRC_HOST} | ${AWK} '{ip=$2}END{print ip}')
if [ -z ${DST_HOST} ]; then
usage "-d|--dst-ip is missing"
fi
fi
fi
if [ -z ${SRC_POOL} ]; then usage "-sp|--src-pool is missing" ; fi
if [ -z ${DST_POOL} ]; then usage "-dp|--dst-pool is missing" ; fi
SRC_DATASETS=/tmp/${MYNAME}_${DST_POOL/\//_}_src_ds.out
DST_DATASETS=/tmp/${MYNAME}_${DST_POOL/\//_}_dst_ds.out
LOCK_FILE=/var/run/${MYNAME}_${DST_POOL/\//_}.lck
TMP_FILE1=/tmp/${MYNAME}_${DST_POOL/\//_}.tmp1
TMP_FILE2=/tmp/${MYNAME}_${DST_POOL/\//_}.tmp2
START_TIME=$(${AWK} 'BEGIN{printf systime();}')
${AWK} -v time=${START_TIME} 'BEGIN{print "START:",strftime("%d.%m.%Y %H:%M.%S",time)}'
# Clean up on signal
# -------------------------
trap 'echo "\n--- Got signal: Exiting ...\n"; \
date ; \
sleep 3; kill -9 ${!} 2>/dev/null; \
/usr/bin/rm -f ${LOCK_FILE}; \
exit 1' 1 2 3 13 14 15 18
###########################
if [ -f ${LOCK_FILE} ] ; then
echo "$0 is allready running as PID $(/usr/bin/cat ${LOCK_FILE}) look in ${LOCK_FILE}"
exit 1
else
echo $$ > ${LOCK_FILE}
fi
if [ "_${LOCAL_SYNC}_" == "_yes_" ]
then
${ZFS} list -rH -t filesystem,snapshot,volume -o name,type,${BACKUP_PROPERTY} -s creation ${SRC_POOL} > ${SRC_DATASETS} &
else
${SSH} ${SRC_USER:+"${SRC_USER}@"}${SRC_HOST} "${ZFS} list -rH -t filesystem,snapshot,volume -o name,type,${BACKUP_PROPERTY} -s creation ${SRC_POOL}" > ${SRC_DATASETS} &
fi
${ZFS} list -rH -t filesystem,snapshot,volume -o name,type -s creation ${DST_POOL} > ${DST_DATASETS} &
wait
function convert_to_poolname () {
from_zfs=$1
search=$2
replace=$3
echo ${from_zfs} | sed -e "s#^${search}#${replace}#g"
}
function is_available () {
snapshot=$1
list=$2
${AWK} -v snapshot=${snapshot} 'BEGIN{rc=1;}$1 == snapshot{print $1; rc=0;}END{exit rc;}' ${list}
return $?
}
function expire_dst_pool_snapshots () {
days_to_keep=$1
min_to_keep=$2
for expired_zfs in $(
${ZFS} list -o creation,name -S creation -t snapshot | \
${AWK} \
-v days_to_keep=${days_to_keep} \
-v min_to_keep=${min_to_keep} \
-v DST_POOL="^${DST_POOL}" \
'
BEGIN{
split("Jan:Feb:Mar:Apr:May:Jun:Jul:Aug:Sep:Oct:Nov:Dec",mon,":");
for(m in mon){
month[mon[m]]=m
};
expire_date=systime()-days_to_keep*60*60*24
}
$NF ~ DST_POOL {
filesystem=$NF;
gsub(/@.*$/,"",filesystem);
split($4,time,":");
filesystem_date=mktime(sprintf("%d %02d %02d %02d %02d 00", $5, month[$2], $3, time[1], time[2]));
count[filesystem]++;
if(filesystem_date < expire_date && count[filesystem] > min_to_keep )
{
print $NF;
}
}')
do
printf "$(${DATE}) Destroying snapshot ${expired_zfs}\n"
${ZFS} destroy ${expired_zfs}
done
}
function get_src_list () {
${AWK} -v backup_server=${MYHOST} '
( $2=="filesystem" || $2=="volume" ) && $3==backup_server {
path[$1]=1;
for(name in path){
# delete name from list, if name is substring of $1
if( index($1,name)==1 && name != $1 && path[name]!=0 ){
path[name]=0;
}
}
}
END{
for(name in path){
if(path[name]==1) print name
}
}
' ${SRC_DATASETS}
}
function first_snapshot () {
${AWK} -v zfs="${1}@" '
$2=="snapshot" && $1 ~ zfs {
first=$1;
# und raus...
nextfile;
}
END{
print first;
}
' $2
}
function last_snapshot () {
${AWK} -v zfs="^${1}" -F '[@ \t]' '
$3 == "snapshot" && $1 ~ zfs {
last=$1"@"$2;
}
END{
printf last;
}
' $2
}
function get_incremental_snapshot () {
src_host=$1
src_datasets=$2
first=$3
last=$4
dst_pool=$5
dst_datasets=$6
if [ $# -lt 6 ] ; then
echo "Called from line ${BASH_LINENO[$i]} with $# Arguments"
end 1
fi
src_zfs=$(echo ${first} | ${AWK} -F'@' '{print $1}')
first_snap=$(echo ${first} | ${AWK} -F'@' '{print FS""$2}')
echo "Getting snapshot ${zfs}..."
if [ "_${LOCAL_SYNC}_" == "_yes_" ]
then
${ZFS} send -I ${first_snap} ${last} | ${ZFS} recv -vFd ${dst_pool}
else
if [ "_${SECURE}_" == "_yes_" ]
then
# setup receiver
${MBUFFER} ${MBUFFER_OPTS} -l ${TMP_FILE1} -I 127.0.0.1:${MBUFFER_PORT} | \
${ZFS} recv -vFd ${dst_pool} 2>&1 &
# start sender
${SSH} ${SRC_USER:+"${SRC_USER}@"}${SRC_HOST} \
-R ${MBUFFER_PORT}:127.0.0.1:${MBUFFER_PORT} \
"${ZFS} send -I ${first_snap} ${last} | ${MBUFFER} ${MBUFFER_OPTS} -O 127.0.0.1:${MBUFFER_PORT} 2>&1" >${TMP_FILE2} &
else
# setup receiver
${MBUFFER} ${MBUFFER_OPTS} -l ${TMP_FILE1} -I ${MBUFFER_PORT} | \
${ZFS} recv -vFd ${dst_pool} 2>&1 &
# start sender
${SSH} ${SRC_USER:+"${SRC_USER}@"}${SRC_HOST} \
"${ZFS} send -I ${first_snap} ${last} | ${MBUFFER} ${MBUFFER_OPTS} -O ${DST_HOST}:${MBUFFER_PORT} 2>&1" >${TMP_FILE2} &
fi
wait
local_md5=$(grep md5 ${TMP_FILE1})
remote_md5=$(grep md5 ${TMP_FILE2})
local_summary=$(grep summary ${TMP_FILE1})
remote_summary=$(grep summary ${TMP_FILE2})
printf "remote %s\nlocal %s\n" "${remote_md5}" "${local_md5}"
printf "remote %s\nlocal %s\n" "${remote_summary}" "${local_summary}"
rm -f ${TMP_FILE1} ${TMP_FILE2}
fi
}
function get_initial_snapshot () {
src_host=$1
src_datasets=$2
zfs=$3
dst_pool=$4
dst_datasets=$5
if [ -z "$(is_available ${zfs} ${dst_datasets})" ] ; then
echo "Getting snapshot ${zfs}..."
if [ "_${LOCAL_SYNC}_" == "_yes_" ]
then
${ZFS} send -R ${zfs} | ${ZFS} recv -vFd ${dst_pool}
else
if [ "_${SECURE}_" == "_yes_" ]
then
# setup receiver
${MBUFFER} ${MBUFFER_OPTS} -l ${TMP_FILE1} -I 127.0.0.1:${MBUFFER_PORT} | \
${ZFS} recv -vFd ${dst_pool} 2>&1 &
# start sender
${SSH} ${SRC_USER:+"${SRC_USER}@"}${SRC_HOST} \
-R ${MBUFFER_PORT}:127.0.0.1:${MBUFFER_PORT} \
"${ZFS} send -R ${zfs} | ${MBUFFER} ${MBUFFER_OPTS} -O 127.0.0.1:${MBUFFER_PORT} 2>&1" >${TMP_FILE2} &
else
# setup receiver
${MBUFFER} ${MBUFFER_OPTS} -l ${TMP_FILE1} -I ${MBUFFER_PORT} | \
${ZFS} recv -vFd ${dst_pool} 2>&1 &
# start sender
${SSH} ${SRC_USER:+"${SRC_USER}@"}${SRC_HOST} \
"${ZFS} send -R ${zfs} | ${MBUFFER} ${MBUFFER_OPTS} -O ${DST_HOST}:${MBUFFER_PORT} 2>&1" >${TMP_FILE2} &
fi
wait
local_md5=$(grep md5 ${TMP_FILE1})
remote_md5=$(grep md5 ${TMP_FILE2})
local_summary=$(grep summary ${TMP_FILE1})
remote_summary=$(grep summary ${TMP_FILE2})
printf "remote %s\nlocal %s\n" "${remote_md5}" "${local_md5}"
printf "remote %s\nlocal %s\n" "${remote_summary}" "${local_summary}"
rm -f ${TMP_FILE1} ${TMP_FILE2}
fi
fi
}
function timestamp () {
echo $(${DATE} '+%Y%m%d-%H:%M:%S')
}
function expire_backup_snapshots () {
src_host=$1
src_datasets=$2
dst_datasets=$3
src_last_to_keep=$4
dst_pool=$5
src_zfs=$(echo ${src_last_to_keep} | ${AWK} -F'@' '{print $1}')
dst_zfs=$(convert_to_poolname ${src_zfs} ${SRC_POOL} ${dst_pool})
dst_last_to_keep=$(convert_to_poolname ${src_last_to_keep} ${SRC_POOL} ${dst_pool})
echo "Deleting old backup snapshots before ${dst_last_to_keep}"
if ( ${ZFS} list -o name ${dst_last_to_keep} >/dev/null 2>&1 ) ; then
for src_backup_snapshot in $(${AWK} -v src_backup="${src_zfs}@${BACKUP_SNAPSHOT_NAME}" -v src_last_to_keep="${src_last_to_keep}" '
$1 == src_last_to_keep {
exit 0;
}
$1 ~ src_backup {
print $1;
}
' ${src_datasets})
do
printf "\tDeleting on src ${src_backup_snapshot} ..."
if [ "_${LOCAL_SYNC}_" == "_yes_" ]
then
${ZFS} destroy ${src_backup_snapshot}
status=$?
else
${SSH} ${SRC_USER:+"${SRC_USER}@"}${SRC_HOST} "${ZFS} destroy ${src_backup_snapshot}"
status=$?
fi
if [ ${status} -eq 0 ] ; then
echo "done"
else
echo "failed"
fi
done
for dst_backup_snapshot in $(${AWK} -v dst_backup="${dst_zfs}@${BACKUP_SNAPSHOT_NAME}" -v dst_last_to_keep=${dst_last_to_keep} '
$1 == dst_last_to_keep {
exit 0;
}
$1 ~ dst_backup {
print $1;
}
' ${dst_datasets})
do
printf "\tDeleting on destination ${dst_backup_snapshot} ..."
if ( ${ZFS} destroy ${dst_backup_snapshot} ) ; then
echo "done"
else
echo "failed"
fi
done
else
echo "Strange we do not have the copy of ${dst_last_to_keep} => STOP!"
fi
}
function end () {
/usr/bin/rm -f ${LOCK_FILE}
exit $1
}
for src_zfs in $(get_src_list) ; do
echo "Evaluating ${src_zfs}"
dst_zfs=$(convert_to_poolname ${src_zfs} ${SRC_POOL} ${DST_POOL})
last_src=$(last_snapshot ${src_zfs} ${SRC_DATASETS})
last_dst=$(last_snapshot ${dst_zfs} ${DST_DATASETS})
last_backup_src=$(${AWK} -v zfs="${src_zfs}@${BACKUP_SNAPSHOT_NAME}" '$1 ~ zfs{last=$1}END{printf last}' ${SRC_DATASETS})
last_backup_dst=$(${AWK} -v zfs="${dst_zfs}@${BACKUP_SNAPSHOT_NAME}" '$1 ~ zfs{last=$1}END{printf last}' ${DST_DATASETS})
last_dst_on_src=$(convert_to_poolname ${last_dst} ${DST_POOL} ${SRC_POOL})
this_backup_src=${src_zfs}@${BACKUP_SNAPSHOT_NAME}_$(timestamp)
# Create snapshot for incremental backups
if [ "_${LOCAL_SYNC}_" == "_yes_" ]
then
${ZFS} snapshot ${this_backup_src}
else
${SSH} ${SRC_USER:+"${SRC_USER}@"}${SRC_HOST} "${ZFS} snapshot ${this_backup_src}"
fi
if [ -z "${last_src}" ] ; then
last_src=${this_backup_src}
fi
if [ -n "$(is_available ${dst_zfs} ${DST_DATASETS})" -a -z "${last_dst}" ] ; then
echo "zfs is on dst, but no snapshots. Getting ${last_src}..."
get_initial_snapshot ${SRC_HOST} ${SRC_DATASETS} ${last_src} ${DST_POOL} ${DST_DATASETS}
# Look for last backup snapshot on destination
elif [ -n "${last_backup_dst}" ] ; then
# Name of last backup snapshot on src
last_dst_backup_on_src=$(convert_to_poolname ${last_backup_dst} ${DST_POOL} ${SRC_POOL})
# If converted name is not empty and snapshot is in the list of src snapshots
# then get all snapshots from last backup until now
if [ -n "${last_dst_backup_on_src}" ] ; then
if [ -n "$(is_available ${last_dst_backup_on_src} ${SRC_DATASETS})" ] ; then
# Get the snapshot of this backup
printf "%s\tsnapshot\n" ${this_backup_src} >> ${SRC_DATASETS}
get_incremental_snapshot ${SRC_HOST} ${SRC_DATASETS} ${last_dst_backup_on_src} ${this_backup_src} ${DST_POOL} ${DST_DATASETS} && \
expire_backup_snapshots ${SRC_HOST} ${SRC_DATASETS} ${DST_DATASETS} ${this_backup_src} ${DST_POOL}
fi
fi
elif [ -n "$(is_available ${dst_zfs} ${DST_DATASETS})" ] ; then
# No last backup snapshot on dst but we have snapshots
if [ -n "$(is_available ${last_dst_on_src} ${SRC_DATASETS})" ] ; then
echo "Try to backup from ${last_dst_on_src} to ${this_backup_src}"
first=${last_dst_on_src}
last=${last_src}
get_incremental_snapshot ${SRC_HOST} ${SRC_DATASETS} ${first} ${last} ${DST_POOL} ${DST_DATASETS} && \
expire_backup_snapshots ${SRC_HOST} ${SRC_DATASETS} ${DST_DATASETS} ${this_backup_src} ${DST_POOL}
# Get the snapshot of this backup
printf "%s\tsnapshot\n" ${this_backup_src} >> ${SRC_DATASETS}
get_incremental_snapshot ${SRC_HOST} ${SRC_DATASETS} ${last} ${this_backup_src} ${DST_POOL} ${DST_DATASETS} && \
expire_backup_snapshots ${SRC_HOST} ${SRC_DATASETS} ${DST_DATASETS} ${this_backup_src} ${DST_POOL}
else
echo "OK I tried hard... now it is your job..."
fi
else
# No existing copies for this zfs. Get the last <INITIAL_COPIES> copies
first=$(${AWK} -v zfs=${src_zfs} -v intitial_copies=$((${INITIAL_COPIES}-1)) '
$1 ~ zfs && $2=="snapshot" {
last[++count]=$1;
}
END {
if(count>intitial_copies){
print last[count-intitial_copies]
}else{
print last[1]
}
}' ${SRC_DATASETS})
last=$( ${AWK} -v zfs=${src_zfs} '$1 ~ zfs && $2=="snapshot"{last=$1}END{printf last}' ${SRC_DATASETS} )
get_initial_snapshot ${SRC_HOST} ${SRC_DATASETS} ${first} ${DST_POOL} ${DST_DATASETS}
get_incremental_snapshot ${SRC_HOST} ${SRC_DATASETS} ${first} ${last} ${DST_POOL} ${DST_DATASETS} && \
expire_backup_snapshots ${SRC_HOST} ${SRC_DATASETS} ${DST_DATASETS} ${this_backup_src} ${DST_POOL}
# Get the snapshot of this backup
printf "%s\tsnapshot\n" ${this_backup_src} >> ${SRC_DATASETS}
get_incremental_snapshot ${SRC_HOST} ${SRC_DATASETS} ${last} ${this_backup_src} ${DST_POOL} ${DST_DATASETS} && \
expire_backup_snapshots ${SRC_HOST} ${SRC_DATASETS} ${DST_DATASETS} ${this_backup_src} ${DST_POOL}
fi
echo
echo --------------------------------------------------------------------------------
date
echo
done
# expire_dst_pool_snapshots days_to_keep min_to_keep
expire_dst_pool_snapshots 34 70
END_TIME=$(${AWK} 'BEGIN{printf systime();}')
${AWK} -v time=${END_TIME} 'BEGIN{print "END :",strftime("%d.%m.%Y %H:%M.%S",time)}'
${AWK} -v start=${START_TIME} -v end=${END_TIME} 'BEGIN{print "DURATION:",strftime("%H:%M.%S",end-start-3600*strftime("%H",0))}'
end 0
