ZFS sync script
From Lolly's Wiki
Jump to navigationJump to search
Like all of my scripts this script is coming without any guaranties!!!
You can use it on your own risk!
About the script
- It uses mbuffer. It is easy to compile.
- It uses gawk.
- The variable SECURE defines if you want to use ssh to encrypt your stream. Set it to yes or no.
- To mark the datasets to copy from the backup host use this on the source:
# /usr/sbin/zfs set de.timmann:auto-backup=<backup host> <dataset>
- Run the script on the destination/backup host.
- If you don't want to use root as backup-user on source host do this to create a zfssync user (Solaris syntax):
# useradd -m zfssync
# passwd -N zfssync
# usermod -K type=normal zfssync
- Make an ssh-key exchange to login without password for SRC_USER.
Good luck!
zfs_sync.sh
#!/bin/bash
# Written by Lars Timmann <L@rs.Timmann.de> 2013
# This script is a rotten bunch of code... rewrite it!
# Some defaults
BACKUP_PROPERTY="de.timmann:auto-backup"
BACKUP_SNAPSHOT_NAME="zfssync"
MBUFFER_PORT=10001
MBUFFER=/opt/mbuffer/bin/mbuffer;
SRC_USER=zfssync
INITIAL_COPIES=3
# Default yes means use SSH for encryption over the net. Every other value means just mbuffer.
SECURE="yes"
LOCAL_SYNC="no"
MBUFFER_PORT=10001
MBUFFER_OPTS="-v 0 --md5 -s 128k -m 256M"
BACKUP_PROPERTY="de.timmann:auto-backup"
ZFS=/usr/sbin/zfs
SSH="/usr/bin/ssh -xc blowfish"
AWK=/usr/bin/gawk
#AWK=/opt/sfw/bin/gawk
GREP=/usr/bin/grep
DATE=/usr/bin/date
MD5="/usr/bin/digest -a md5"
ROUTE=/usr/sbin/route
MBUFFER="/opt/mbuffer/bin/mbuffer"
MYHOST=$(/usr/bin/hostname)
MYNAME=$(/usr/bin/basename $0)
function usage () {
if [ $# -gt 0 ]
then
if [ "_${1}_" != "_help_" ]
then
echo "Error: ${MYNAME} : $*"
fi
else
echo "Error: ${MYNAME} : Check parameters"
fi
cat <<EOU
Usage: ${MYNAME} <params>
Where params is from this set of parameters:
-s|--src-ip <IP> The host from where we want to sync
-d|--dst-ip <IP> The IP on this host where the remote mbuffer should try to connect to
If omitted the IP to use is guessed via route get.
-u|--user <user> The user on "--src-ip" which has rights to send a zfs.
It must be able to login via ssh with public key.
On Solaris it is the profile "ZFS File System Management"
Try this on the "--src-ip":
# roleadd \
-d /export/home/zfssync \
-c "User for zfs send/recv" \
-s /bin/bash \
-m \
-P "ZFS File System Management" \
zfssync
# rolemod -K type=normal zfssync
# passwd -N zfssync
And then put the ssh-public-key from this host into
/export/home/zfssync/.ssh/authorized_keys
on the "--src-ip".
Remember to set the permissions on .ssh to 700 and .ssh/authorized_keys to 600.
The Homedir of the user must not be world writeable.
-sp|--src-pool <zpool> The zpool we want to sync from "--src-ip".
-dp|--dst-pool <zpool> The zpool on this host where we want to sync to ${MYNAME}.
-mbp|--mbuffer-port <port>
If the default port 10001 is in use use another port.
-mb|--mbuffer-path <path>
Path of mbuffer binary including binary itself.
-mbbw|--mbuffer-bwlimit <rate>
Limit the read bandwith of mbuffer (mbuffer option -r)
From mbuffer --help: limit read rate to <rate> B/s, where <rate> can be given in b,k,M,G
-bp|--backup-property <property>
This defaults to ${BACKUP_PROPERTY}.
You have to set this property on all ZFS datasets to ${MYHOST}.
# /usr/sbin/zfs set ${BACKUP_PROPERTY}=${MYHOST} <dataset>
This is inherited as usual.
-bsn|--backup-snap-name <snapshotname>
This is the name of the snapshot which we use to sync.
This defaults to ${BACKUP_SNAPSHOT_NAME}.
Never delete this snapshot manually or you will break the sync and restart
from the beginning.
-i|--insecure Not for production environments! No ssh tunneling. No encryption over the net!
EOU
##-l|--local Just do a local zfs send/recv...
exit 1
}
while [ $# -gt 0 ]
do
#if [ $# -ge 2 ]; then value=$2; fi
case $1 in
--help|-h)
usage "help"
;;
-l|--local)
LOCAL_SYNC="yes"
SRC_HOST="localhost"
param="dummy"
shift;
;;
-i|--insecure|--fuck-off-security)
SECURE="no"
param="dummy"
shift;
;;
--?*=?*|-?*=?*)
param=${1%=*}
value=${1#*=}
shift;
;;
--?*=|-?*=)
param=${1%=*}
usage "${param} needs a vlaue!"
;;
*)
param=$1
if [ $# -ge 2 -a "_${2%-*}_" != "__" ]
then
value=$2
shift
fi
shift
;;
esac
case $param in
-s|--src-ip)
if [ -z $value ] ; then usage "Param ${param} needs a value" ; fi
SRC_HOST=${value}
;;
-d|--dst-ip)
if [ -z $value ] ; then usage "Param ${param} needs a value" ; fi
DST_HOST=${value};
;;
-u|--user)
if [ -z $value ] ; then usage "Param ${param} needs a value" ; fi
SRC_USER=${value}
;;
-sp|--src-pool)
if [ -z $value ] ; then usage "Param ${param} needs a value" ; fi
SRC_POOL=${value}
;;
-bsn|--backup-snap-name)
if [ -z $value ] ; then usage "Param ${param} needs a value" ; fi
BACKUP_SNAPSHOT_NAME=${value}
;;
-dp|--dst-pool)
if [ -z $value ] ; then usage "Param ${param} needs a value" ; fi
DST_POOL=${value}
;;
-mbp|--mbuffer-port)
if [ -z $value ] ; then usage "Param ${param} needs a value" ; fi
MBUFFER_PORT=${value}
;;
-mb|--mbuffer-path)
if [ -z $value ] ; then usage "Param ${param} needs a value" ; fi
MBUFFER=${value}
;;
-mbbw|--mbuffer-bwlimit)
if [ -z $value ] ; then usage "Param ${param} needs a value" ; fi
MBUFFER_OPTS="${MBUFFER_OPTS} -r ${value}"
;;
-bp|--backup-property)
if [ -z $value ] ; then usage "Param ${param} needs a value" ; fi
BACKUP_PROPERTY=${value}
;;
dummy)
;;
*)
usage "Unknown parameter $1"
esac
done
if [ "_${LOCAL_SYNC}_" == "no" ]
then
if [ -z ${SRC_HOST} ]; then usage "-s|--src-ip is missing" ; fi
# Guess the right IP for communication with source host
if [ -z ${DST_HOST} ]; then
DST_HOST=$(${ROUTE} -vn get ${SRC_HOST} | ${AWK} '{ip=$2}END{print ip}')
if [ -z ${DST_HOST} ]; then
usage "-d|--dst-ip is missing"
fi
fi
fi
if [ -z ${SRC_POOL} ]; then usage "-sp|--src-pool is missing" ; fi
if [ -z ${DST_POOL} ]; then usage "-dp|--dst-pool is missing" ; fi
SRC_DATASETS=/tmp/${MYNAME}_${DST_POOL/\//_}_src_ds.out
DST_DATASETS=/tmp/${MYNAME}_${DST_POOL/\//_}_dst_ds.out
LOCK_FILE=/var/run/${MYNAME}_${DST_POOL/\//_}.lck
TMP_FILE1=/tmp/${MYNAME}_${DST_POOL/\//_}.tmp1
TMP_FILE2=/tmp/${MYNAME}_${DST_POOL/\//_}.tmp2
START_TIME=$(${AWK} 'BEGIN{printf systime();}')
${AWK} -v time=${START_TIME} 'BEGIN{print "START:",strftime("%d.%m.%Y %H:%M.%S",time)}'
# Clean up on signal
# -------------------------
trap 'echo "\n--- Got signal: Exiting ...\n"; \
date ; \
sleep 3; kill -9 ${!} 2>/dev/null; \
/usr/bin/rm -f ${LOCK_FILE}; \
exit 1' 1 2 3 13 14 15 18
###########################
if [ -f ${LOCK_FILE} ] ; then
echo "$0 is allready running as PID $(/usr/bin/cat ${LOCK_FILE}) look in ${LOCK_FILE}"
exit 1
else
echo $$ > ${LOCK_FILE}
fi
if [ "_${LOCAL_SYNC}_" == "_yes_" ]
then
${ZFS} list -rH -t filesystem,snapshot,volume -o name,type,${BACKUP_PROPERTY} -s creation ${SRC_POOL} > ${SRC_DATASETS} &
else
${SSH} ${SRC_USER:+"${SRC_USER}@"}${SRC_HOST} "${ZFS} list -rH -t filesystem,snapshot,volume -o name,type,${BACKUP_PROPERTY} -s creation ${SRC_POOL}" > ${SRC_DATASETS} &
fi
${ZFS} list -rH -t filesystem,snapshot,volume -o name,type -s creation ${DST_POOL} > ${DST_DATASETS} &
wait
function convert_to_poolname () {
from_zfs=$1
search=$2
replace=$3
echo ${from_zfs} | sed -e "s#^${search}#${replace}#g"
}
function is_available () {
snapshot=$1
list=$2
${AWK} -v snapshot=${snapshot} 'BEGIN{rc=1;}$1 == snapshot{print $1; rc=0;}END{exit rc;}' ${list}
return $?
}
function expire_dst_pool_snapshots () {
days_to_keep=$1
min_to_keep=$2
for expired_zfs in $(
${ZFS} list -o creation,name -S creation -t snapshot | \
${AWK} \
-v days_to_keep=${days_to_keep} \
-v min_to_keep=${min_to_keep} \
-v DST_POOL="^${DST_POOL}" \
'
BEGIN{
split("Jan:Feb:Mar:Apr:May:Jun:Jul:Aug:Sep:Oct:Nov:Dec",mon,":");
for(m in mon){
month[mon[m]]=m
};
expire_date=systime()-days_to_keep*60*60*24
}
$NF ~ DST_POOL {
filesystem=$NF;
gsub(/@.*$/,"",filesystem);
split($4,time,":");
filesystem_date=mktime(sprintf("%d %02d %02d %02d %02d 00", $5, month[$2], $3, time[1], time[2]));
count[filesystem]++;
if(filesystem_date < expire_date && count[filesystem] > min_to_keep )
{
print $NF;
}
}')
do
printf "$(${DATE}) Destroying snapshot ${expired_zfs}\n"
${ZFS} destroy ${expired_zfs}
done
}
function get_src_list () {
${AWK} -v backup_server=${MYHOST} '
( $2=="filesystem" || $2=="volume" ) && $3==backup_server {
path[$1]=1;
for(name in path){
# delete name from list, if name is substring of $1
if( index($1,name)==1 && name != $1 && path[name]!=0 ){
path[name]=0;
}
}
}
END{
for(name in path){
if(path[name]==1) print name
}
}
' ${SRC_DATASETS}
}
function first_snapshot () {
${AWK} -v zfs="${1}@" '
$2=="snapshot" && $1 ~ zfs {
first=$1;
# und raus...
nextfile;
}
END{
print first;
}
' $2
}
function last_snapshot () {
${AWK} -v zfs="^${1}" -F '[@ \t]' '
$3 == "snapshot" && $1 ~ zfs {
last=$1"@"$2;
}
END{
printf last;
}
' $2
}
function get_incremental_snapshot () {
src_host=$1
src_datasets=$2
first=$3
last=$4
dst_pool=$5
dst_datasets=$6
if [ $# -lt 6 ] ; then
echo "Called from line ${BASH_LINENO[$i]} with $# Arguments"
end 1
fi
src_zfs=$(echo ${first} | ${AWK} -F'@' '{print $1}')
first_snap=$(echo ${first} | ${AWK} -F'@' '{print FS""$2}')
echo "Getting snapshot ${zfs}..."
if [ "_${LOCAL_SYNC}_" == "_yes_" ]
then
${ZFS} send -I ${first_snap} ${last} | ${ZFS} recv -vFd ${dst_pool}
else
if [ "_${SECURE}_" == "_yes_" ]
then
# setup receiver
${MBUFFER} ${MBUFFER_OPTS} -l ${TMP_FILE1} -I 127.0.0.1:${MBUFFER_PORT} | \
${ZFS} recv -vFd ${dst_pool} 2>&1 &
# start sender
${SSH} ${SRC_USER:+"${SRC_USER}@"}${SRC_HOST} \
-R ${MBUFFER_PORT}:127.0.0.1:${MBUFFER_PORT} \
"${ZFS} send -I ${first_snap} ${last} | ${MBUFFER} ${MBUFFER_OPTS} -O 127.0.0.1:${MBUFFER_PORT} 2>&1" >${TMP_FILE2} &
else
# setup receiver
${MBUFFER} ${MBUFFER_OPTS} -l ${TMP_FILE1} -I ${MBUFFER_PORT} | \
${ZFS} recv -vFd ${dst_pool} 2>&1 &
# start sender
${SSH} ${SRC_USER:+"${SRC_USER}@"}${SRC_HOST} \
"${ZFS} send -I ${first_snap} ${last} | ${MBUFFER} ${MBUFFER_OPTS} -O ${DST_HOST}:${MBUFFER_PORT} 2>&1" >${TMP_FILE2} &
fi
wait
local_md5=$(grep md5 ${TMP_FILE1})
remote_md5=$(grep md5 ${TMP_FILE2})
local_summary=$(grep summary ${TMP_FILE1})
remote_summary=$(grep summary ${TMP_FILE2})
printf "remote %s\nlocal %s\n" "${remote_md5}" "${local_md5}"
printf "remote %s\nlocal %s\n" "${remote_summary}" "${local_summary}"
rm -f ${TMP_FILE1} ${TMP_FILE2}
fi
}
function get_initial_snapshot () {
src_host=$1
src_datasets=$2
zfs=$3
dst_pool=$4
dst_datasets=$5
if [ -z "$(is_available ${zfs} ${dst_datasets})" ] ; then
echo "Getting snapshot ${zfs}..."
if [ "_${LOCAL_SYNC}_" == "_yes_" ]
then
${ZFS} send -R ${zfs} | ${ZFS} recv -vFd ${dst_pool}
else
if [ "_${SECURE}_" == "_yes_" ]
then
# setup receiver
${MBUFFER} ${MBUFFER_OPTS} -l ${TMP_FILE1} -I 127.0.0.1:${MBUFFER_PORT} | \
${ZFS} recv -vFd ${dst_pool} 2>&1 &
# start sender
${SSH} ${SRC_USER:+"${SRC_USER}@"}${SRC_HOST} \
-R ${MBUFFER_PORT}:127.0.0.1:${MBUFFER_PORT} \
"${ZFS} send -R ${zfs} | ${MBUFFER} ${MBUFFER_OPTS} -O 127.0.0.1:${MBUFFER_PORT} 2>&1" >${TMP_FILE2} &
else
# setup receiver
${MBUFFER} ${MBUFFER_OPTS} -l ${TMP_FILE1} -I ${MBUFFER_PORT} | \
${ZFS} recv -vFd ${dst_pool} 2>&1 &
# start sender
${SSH} ${SRC_USER:+"${SRC_USER}@"}${SRC_HOST} \
"${ZFS} send -R ${zfs} | ${MBUFFER} ${MBUFFER_OPTS} -O ${DST_HOST}:${MBUFFER_PORT} 2>&1" >${TMP_FILE2} &
fi
wait
local_md5=$(grep md5 ${TMP_FILE1})
remote_md5=$(grep md5 ${TMP_FILE2})
local_summary=$(grep summary ${TMP_FILE1})
remote_summary=$(grep summary ${TMP_FILE2})
printf "remote %s\nlocal %s\n" "${remote_md5}" "${local_md5}"
printf "remote %s\nlocal %s\n" "${remote_summary}" "${local_summary}"
rm -f ${TMP_FILE1} ${TMP_FILE2}
fi
fi
}
function timestamp () {
echo $(${DATE} '+%Y%m%d-%H:%M:%S')
}
function expire_backup_snapshots () {
src_host=$1
src_datasets=$2
dst_datasets=$3
src_last_to_keep=$4
dst_pool=$5
src_zfs=$(echo ${src_last_to_keep} | ${AWK} -F'@' '{print $1}')
dst_zfs=$(convert_to_poolname ${src_zfs} ${SRC_POOL} ${dst_pool})
dst_last_to_keep=$(convert_to_poolname ${src_last_to_keep} ${SRC_POOL} ${dst_pool})
echo "Deleting old backup snapshots before ${dst_last_to_keep}"
if ( ${ZFS} list -o name ${dst_last_to_keep} >/dev/null 2>&1 ) ; then
for src_backup_snapshot in $(${AWK} -v src_backup="${src_zfs}@${BACKUP_SNAPSHOT_NAME}" -v src_last_to_keep="${src_last_to_keep}" '
$1 == src_last_to_keep {
exit 0;
}
$1 ~ src_backup {
print $1;
}
' ${src_datasets})
do
printf "\tDeleting on src ${src_backup_snapshot} ..."
if [ "_${LOCAL_SYNC}_" == "_yes_" ]
then
${ZFS} destroy ${src_backup_snapshot}
status=$?
else
${SSH} ${SRC_USER:+"${SRC_USER}@"}${SRC_HOST} "${ZFS} destroy ${src_backup_snapshot}"
status=$?
fi
if [ ${status} -eq 0 ] ; then
echo "done"
else
echo "failed"
fi
done
for dst_backup_snapshot in $(${AWK} -v dst_backup="${dst_zfs}@${BACKUP_SNAPSHOT_NAME}" -v dst_last_to_keep=${dst_last_to_keep} '
$1 == dst_last_to_keep {
exit 0;
}
$1 ~ dst_backup {
print $1;
}
' ${dst_datasets})
do
printf "\tDeleting on destination ${dst_backup_snapshot} ..."
if ( ${ZFS} destroy ${dst_backup_snapshot} ) ; then
echo "done"
else
echo "failed"
fi
done
else
echo "Strange we do not have the copy of ${dst_last_to_keep} => STOP!"
fi
}
function end () {
/usr/bin/rm -f ${LOCK_FILE}
exit $1
}
for src_zfs in $(get_src_list) ; do
echo "Evaluating ${src_zfs}"
dst_zfs=$(convert_to_poolname ${src_zfs} ${SRC_POOL} ${DST_POOL})
last_src=$(last_snapshot ${src_zfs} ${SRC_DATASETS})
last_dst=$(last_snapshot ${dst_zfs} ${DST_DATASETS})
last_backup_src=$(${AWK} -v zfs="${src_zfs}@${BACKUP_SNAPSHOT_NAME}" '$1 ~ zfs{last=$1}END{printf last}' ${SRC_DATASETS})
last_backup_dst=$(${AWK} -v zfs="${dst_zfs}@${BACKUP_SNAPSHOT_NAME}" '$1 ~ zfs{last=$1}END{printf last}' ${DST_DATASETS})
last_dst_on_src=$(convert_to_poolname ${last_dst} ${DST_POOL} ${SRC_POOL})
this_backup_src=${src_zfs}@${BACKUP_SNAPSHOT_NAME}_$(timestamp)
# Create snapshot for incremental backups
if [ "_${LOCAL_SYNC}_" == "_yes_" ]
then
${ZFS} snapshot ${this_backup_src}
else
${SSH} ${SRC_USER:+"${SRC_USER}@"}${SRC_HOST} "${ZFS} snapshot ${this_backup_src}"
fi
if [ -z "${last_src}" ] ; then
last_src=${this_backup_src}
fi
if [ -n "$(is_available ${dst_zfs} ${DST_DATASETS})" -a -z "${last_dst}" ] ; then
echo "zfs is on dst, but no snapshots. Getting ${last_src}..."
get_initial_snapshot ${SRC_HOST} ${SRC_DATASETS} ${last_src} ${DST_POOL} ${DST_DATASETS}
# Look for last backup snapshot on destination
elif [ -n "${last_backup_dst}" ] ; then
# Name of last backup snapshot on src
last_dst_backup_on_src=$(convert_to_poolname ${last_backup_dst} ${DST_POOL} ${SRC_POOL})
# If converted name is not empty and snapshot is in the list of src snapshots
# then get all snapshots from last backup until now
if [ -n "${last_dst_backup_on_src}" ] ; then
if [ -n "$(is_available ${last_dst_backup_on_src} ${SRC_DATASETS})" ] ; then
# Get the snapshot of this backup
printf "%s\tsnapshot\n" ${this_backup_src} >> ${SRC_DATASETS}
get_incremental_snapshot ${SRC_HOST} ${SRC_DATASETS} ${last_dst_backup_on_src} ${this_backup_src} ${DST_POOL} ${DST_DATASETS} && \
expire_backup_snapshots ${SRC_HOST} ${SRC_DATASETS} ${DST_DATASETS} ${this_backup_src} ${DST_POOL}
fi
fi
elif [ -n "$(is_available ${dst_zfs} ${DST_DATASETS})" ] ; then
# No last backup snapshot on dst but we have snapshots
if [ -n "$(is_available ${last_dst_on_src} ${SRC_DATASETS})" ] ; then
echo "Try to backup from ${last_dst_on_src} to ${this_backup_src}"
first=${last_dst_on_src}
last=${last_src}
get_incremental_snapshot ${SRC_HOST} ${SRC_DATASETS} ${first} ${last} ${DST_POOL} ${DST_DATASETS} && \
expire_backup_snapshots ${SRC_HOST} ${SRC_DATASETS} ${DST_DATASETS} ${this_backup_src} ${DST_POOL}
# Get the snapshot of this backup
printf "%s\tsnapshot\n" ${this_backup_src} >> ${SRC_DATASETS}
get_incremental_snapshot ${SRC_HOST} ${SRC_DATASETS} ${last} ${this_backup_src} ${DST_POOL} ${DST_DATASETS} && \
expire_backup_snapshots ${SRC_HOST} ${SRC_DATASETS} ${DST_DATASETS} ${this_backup_src} ${DST_POOL}
else
echo "OK I tried hard... now it is your job..."
fi
else
# No existing copies for this zfs. Get the last <INITIAL_COPIES> copies
first=$(${AWK} -v zfs=${src_zfs} -v intitial_copies=$((${INITIAL_COPIES}-1)) '
$1 ~ zfs && $2=="snapshot" {
last[++count]=$1;
}
END {
if(count>intitial_copies){
print last[count-intitial_copies]
}else{
print last[1]
}
}' ${SRC_DATASETS})
last=$( ${AWK} -v zfs=${src_zfs} '$1 ~ zfs && $2=="snapshot"{last=$1}END{printf last}' ${SRC_DATASETS} )
get_initial_snapshot ${SRC_HOST} ${SRC_DATASETS} ${first} ${DST_POOL} ${DST_DATASETS}
get_incremental_snapshot ${SRC_HOST} ${SRC_DATASETS} ${first} ${last} ${DST_POOL} ${DST_DATASETS} && \
expire_backup_snapshots ${SRC_HOST} ${SRC_DATASETS} ${DST_DATASETS} ${this_backup_src} ${DST_POOL}
# Get the snapshot of this backup
printf "%s\tsnapshot\n" ${this_backup_src} >> ${SRC_DATASETS}
get_incremental_snapshot ${SRC_HOST} ${SRC_DATASETS} ${last} ${this_backup_src} ${DST_POOL} ${DST_DATASETS} && \
expire_backup_snapshots ${SRC_HOST} ${SRC_DATASETS} ${DST_DATASETS} ${this_backup_src} ${DST_POOL}
fi
echo
echo --------------------------------------------------------------------------------
date
echo
done
# expire_dst_pool_snapshots days_to_keep min_to_keep
expire_dst_pool_snapshots 34 70
END_TIME=$(${AWK} 'BEGIN{printf systime();}')
${AWK} -v time=${END_TIME} 'BEGIN{print "END :",strftime("%d.%m.%Y %H:%M.%S",time)}'
${AWK} -v start=${START_TIME} -v end=${END_TIME} 'BEGIN{print "DURATION:",strftime("%H:%M.%S",end-start-3600*strftime("%H",0))}'
end 0
