push directories as tar archives

bwdatar

@@ -0,0 +1,139 @@
+#!/bin/bash
+# Compare contents of a directory with the contents of the corresponding
+# archive on HPSS.
+# Requirements:
+# - lftp must be available
+# - ~/.netrc must be configured
+# - ~/.bwda must be configured (bwda_acc, bwda_url, ldir_base, rdir_base)
+# - directory structure needs to be the same on both ends
+source $HOME/.bwda
+
+# Usage function
+usage(){
+  echo "Usage: $(basename $0) [opt] [dir]"
+  echo " dir                      path to local directory on LSDF"
+  echo " --include-size-mismatch  include files which mismatch in size to sync list"
+  echo " -n | --dry-run           print commands to stdout instead of executing them"
+  echo " -d | --tempdir dir       Path to the temporary directory where tar-file is created (default: \$PWD)"
+  echo " -h | --help              display this message"
+}
+
+# Get command line argument
+flag_sizemismatch=false
+flag_dryrun=false
+tempdir=$PWD
+
+POSITIONAL=()
+while [[ $# -gt 0 ]]
+do
+key="$1"
+case $key in
+    --include-size-mismatch)
+    flag_sizemismatch=true
+    shift
+    ;;
+    -n|--dry-run)
+    flag_dryrun=true
+    shift
+    ;;
+    -d|--tempdir)
+    tempdir="$2"
+    shift
+    shift
+    ;;
+    -h|--help)
+    usage
+    exit 0
+    shift
+    ;;
+    *)
+    POSITIONAL+=("$1")
+    shift
+    ;;
+esac
+done
+set -- "${POSITIONAL[@]}" # restore positional parameters
+
+if [ $# -ne 1 ]; then
+  usage
+  exit -1
+fi
+
+ldir_target="$(realpath $1)"
+if [ ! -d ${ldir_target} ]; then
+  >&2 echo "Not a directory: $1"
+  exit -3
+fi
+
+# Get current path relative to base
+if [[ $ldir_target == "${ldir_base}"* ]]; then
+  dir_target=${ldir_target#"$ldir_base"}
+else
+  echo "Target directory is not located on LSDF!"
+  echo "Is the base directory setting correct?"
+  echo "ldir_base: $ldir_base"
+  exit -2
+fi
+
+# Construct name of tar file
+tar_exec_dir="$(dirname $ldir_target)"
+tar_target_dir="$(basename $ldir_target)"
+tar_filename="${tar_target_dir}.tar"
+
+# Construct paths of tar file
+rdir_target="$(dirname ${rdir_base}/${dir_target})"
+rfile_target="${rdir_target}/${tar_filename}"
+
+tempdir=$(realpath ${tempdir})
+ltmpfile="${tempdir}/${tar_filename}"
+
+# Receive list of files on SFTP server, omit directories
+rfilelist=$(lftp sftp://${bwda_acc}@${bwda_url} -e "ls -l ${rdir_target}; bye" | grep -v '^d')
+rfilename=($(echo "$rfilelist" | awk '{print $9}'))
+rfilesize=($(echo "$rfilelist" | awk '{print $5}'))
+
+# Check if remote tar file already exists
+ipos=-1
+for ir in ${!rfilename[@]}; do
+  if [[ ${rfilename[${ir}]} == ${tar_filename} ]]; then
+      ipos=$ir
+      break
+  fi
+done
+
+# If file already exists, check if filesize matches
+if [[ $ipos -ge 0 ]]; then
+  filesize=$(cd ${tar_exec_dir}; \
+             tar cf /dev/null --totals ${tar_target_dir} 2>&1 | \
+             awk -F: '{print $2}' | awk '{print $1}')
+  if [[ ${rfilesize[${ipos}]} != ${filesize} ]]; then 
+    >&2 echo "Filesize mismatch: ${rfilename[${ipos}]}, HPSS=${rfilesize[${ipos}]}, estimate=${filesize}"
+    if [[ ! "$flag_sizemismatch" == true ]]; then
+        exit 10
+    fi
+  else
+    echo "Nothing to sync."
+    exit 0
+  fi
+fi
+
+# Write a lftp batch script for syncing
+>&2 echo "Uploading ${rdir_target}/${tar_filename}"
+cmd="open sftp://${bwda_acc}@${bwda_url}\n"
+cmd+="set cmd:parallel 1\n"
+cmd+="cd ${rdir_target}\n"
+cmd+="lcd ${tempdir}\n"
+cmd+="put -c ${tar_filename}\n"
+cmd+="bye\n"
+
+# Execute
+if [[ "$flag_dryrun" == true ]]; then
+  printf "$cmd"
+else
+  (cd ${tar_exec_dir}; tar cf ${ltmpfile} ${tar_target_dir})
+  tmpfile=$(mktemp)
+  printf "$cmd" > $tmpfile
+  lftp -f $tmpfile
+  rm $tmpfile  
+  rm ${ltmpfile}
+fi