diff --git a/bwdatar b/bwdatar new file mode 100755 index 0000000..1f90dba --- /dev/null +++ b/bwdatar @@ -0,0 +1,139 @@ +#!/bin/bash +# Compare contents of a directory with the contents of the corresponding +# archive on HPSS. +# Requirements: +# - lftp must be available +# - ~/.netrc must be configured +# - ~/.bwda must be configured (bwda_acc, bwda_url, ldir_base, rdir_base) +# - directory structure needs to be the same on both ends +source $HOME/.bwda + +# Usage function +usage(){ + echo "Usage: $(basename $0) [opt] [dir]" + echo " dir path to local directory on LSDF" + echo " --include-size-mismatch include files which mismatch in size to sync list" + echo " -n | --dry-run print commands to stdout instead of executing them" + echo " -d | --tempdir dir Path to the temporary directory where tar-file is created (default: \$PWD)" + echo " -h | --help display this message" +} + +# Get command line argument +flag_sizemismatch=false +flag_dryrun=false +tempdir=$PWD + +POSITIONAL=() +while [[ $# -gt 0 ]] +do +key="$1" +case $key in + --include-size-mismatch) + flag_sizemismatch=true + shift + ;; + -n|--dry-run) + flag_dryrun=true + shift + ;; + -d|--tempdir) + tempdir="$2" + shift + shift + ;; + -h|--help) + usage + exit 0 + shift + ;; + *) + POSITIONAL+=("$1") + shift + ;; +esac +done +set -- "${POSITIONAL[@]}" # restore positional parameters + +if [ $# -ne 1 ]; then + usage + exit -1 +fi + +ldir_target="$(realpath $1)" +if [ ! -d ${ldir_target} ]; then + >&2 echo "Not a directory: $1" + exit -3 +fi + +# Get current path relative to base +if [[ $ldir_target == "${ldir_base}"* ]]; then + dir_target=${ldir_target#"$ldir_base"} +else + echo "Target directory is not located on LSDF!" + echo "Is the base directory setting correct?" + echo "ldir_base: $ldir_base" + exit -2 +fi + +# Construct name of tar file +tar_exec_dir="$(dirname $ldir_target)" +tar_target_dir="$(basename $ldir_target)" +tar_filename="${tar_target_dir}.tar" + +# Construct paths of tar file +rdir_target="$(dirname ${rdir_base}/${dir_target})" +rfile_target="${rdir_target}/${tar_filename}" + +tempdir=$(realpath ${tempdir}) +ltmpfile="${tempdir}/${tar_filename}" + +# Receive list of files on SFTP server, omit directories +rfilelist=$(lftp sftp://${bwda_acc}@${bwda_url} -e "ls -l ${rdir_target}; bye" | grep -v '^d') +rfilename=($(echo "$rfilelist" | awk '{print $9}')) +rfilesize=($(echo "$rfilelist" | awk '{print $5}')) + +# Check if remote tar file already exists +ipos=-1 +for ir in ${!rfilename[@]}; do + if [[ ${rfilename[${ir}]} == ${tar_filename} ]]; then + ipos=$ir + break + fi +done + +# If file already exists, check if filesize matches +if [[ $ipos -ge 0 ]]; then + filesize=$(cd ${tar_exec_dir}; \ + tar cf /dev/null --totals ${tar_target_dir} 2>&1 | \ + awk -F: '{print $2}' | awk '{print $1}') + if [[ ${rfilesize[${ipos}]} != ${filesize} ]]; then + >&2 echo "Filesize mismatch: ${rfilename[${ipos}]}, HPSS=${rfilesize[${ipos}]}, estimate=${filesize}" + if [[ ! "$flag_sizemismatch" == true ]]; then + exit 10 + fi + else + echo "Nothing to sync." + exit 0 + fi +fi + +# Write a lftp batch script for syncing +>&2 echo "Uploading ${rdir_target}/${tar_filename}" +cmd="open sftp://${bwda_acc}@${bwda_url}\n" +cmd+="set cmd:parallel 1\n" +cmd+="cd ${rdir_target}\n" +cmd+="lcd ${tempdir}\n" +cmd+="put -c ${tar_filename}\n" +cmd+="bye\n" + +# Execute +if [[ "$flag_dryrun" == true ]]; then + printf "$cmd" +else + (cd ${tar_exec_dir}; tar cf ${ltmpfile} ${tar_target_dir}) + tmpfile=$(mktemp) + printf "$cmd" > $tmpfile + lftp -f $tmpfile + rm $tmpfile + rm ${ltmpfile} +fi