split ucftar to flow field and scalar
This commit is contained in:
Michael Stumpf (ifhcluster)
parent
6b9922dccf
commit
50d68cd2dd
|
|
@ -81,7 +81,6 @@ fgrid="grid_${seqnum}.bin"
|
|||
fproc="proc_${seqnum}.bin"
|
||||
fpart="particles_${seqnum}.bin"
|
||||
fbuvwp="uvwp_"
|
||||
fbscal="scal_"
|
||||
|
||||
# Check if obligatory files are present
|
||||
if [ ! -s ${din}/${fparam} ]; then
|
||||
|
|
@ -117,19 +116,6 @@ for (( ii=0; ii<$nproc; ii++ )); do
|
|||
fuvwp+=($ftmp)
|
||||
done
|
||||
|
||||
# Check if all scalar fields exist (if any) and are not empty
|
||||
if grep -q "scalar" "${din}/${fparam}"; then
|
||||
fscal=()
|
||||
for (( ii=0; ii<$nproc; ii++ )); do
|
||||
ftmp=$(printf ${fbscal}${seqnum}.%05d $ii)
|
||||
if [ ! -s ${din}/${ftmp} ]; then
|
||||
(>&2 echo "[Error] File not found or empty: ${din}/${ftmp}")
|
||||
exit 1
|
||||
fi
|
||||
fscal+=($ftmp)
|
||||
done
|
||||
fi
|
||||
|
||||
# Verbose info
|
||||
if [ $verbose -eq 1 ]; then
|
||||
(>&2 echo "[x] parameters")
|
||||
|
|
@ -137,15 +123,10 @@ if [ $verbose -eq 1 ]; then
|
|||
(>&2 echo "[x] processor grid")
|
||||
(>&2 echo "[x] particles")
|
||||
(>&2 echo "[x] fluid fields")
|
||||
if [ ${#fscal[@]} -ne 0 ]; then
|
||||
(>&2 echo "[x] scalar fields")
|
||||
else
|
||||
(>&2 echo "[ ] scalar fields")
|
||||
fi
|
||||
fi
|
||||
|
||||
# Create a full file list for the archive
|
||||
flist=(${fparam} ${fgrid} ${fproc} ${fpart} ${fuvwp[@]} ${fscal[@]})
|
||||
flist=(${fparam} ${fgrid} ${fproc} ${fpart} ${fuvwp[@]})
|
||||
|
||||
# Now tar them and remove seqence number from file names while doing so
|
||||
flagtar=""
|
||||
|
|
|
|||
|
|
@ -0,0 +1,175 @@
|
|||
#!/bin/bash
|
||||
display_help() {
|
||||
(>&2 echo "Usage: $(basename "$0") [-chqv] [-o outfile] indir iseq")
|
||||
(>&2 echo "UCF tar scalar packer")
|
||||
(>&2 echo)
|
||||
(>&2 echo " indir path to input directory")
|
||||
(>&2 echo " iseq sequence number with leading zeros")
|
||||
(>&2 echo " -c, --checksum compares checksums after the archive is created")
|
||||
(>&2 echo " -h, --help display this help message")
|
||||
(>&2 echo " -o, --outfile output file (default: snapshot_XXXX.ucf.tar)")
|
||||
(>&2 echo " -q, --quicksum same as --checksum, but compares only first bytes")
|
||||
(>&2 echo " -v, --verbose verbose output")
|
||||
}
|
||||
exit_script() {
|
||||
#trap - SIGINT SIGTERM # clear the trap
|
||||
#kill -- -$$ # Sends SIGTERM to child/sub processes
|
||||
(>&2 echo "SIGINT/SIGTERM received: removing archive")
|
||||
rm $fout
|
||||
}
|
||||
# Parse command line arguments
|
||||
if [ $# -eq 0 ]; then
|
||||
display_help
|
||||
exit -1
|
||||
fi
|
||||
fout=""
|
||||
verbose=0
|
||||
checksum=0
|
||||
quicksum=0
|
||||
POSITIONAL=()
|
||||
while [[ $# -gt 0 ]]
|
||||
do
|
||||
key="$1"
|
||||
case $key in
|
||||
-h|--help)
|
||||
display_help
|
||||
exit 0
|
||||
shift # past argument
|
||||
;;
|
||||
-o|--outfile)
|
||||
fout="$2"
|
||||
shift # past argument
|
||||
shift # past value
|
||||
;;
|
||||
-v|--verbose)
|
||||
verbose=1
|
||||
shift # past argument
|
||||
;;
|
||||
-c|--checksum)
|
||||
checksum=1
|
||||
shift # past argument
|
||||
;;
|
||||
-q|--quicksum)
|
||||
quicksum=1
|
||||
shift # past argument
|
||||
;;
|
||||
*) # unknown option
|
||||
POSITIONAL+=("$1") # save it in an array for later
|
||||
shift # past argument
|
||||
;;
|
||||
esac
|
||||
done
|
||||
set -- "${POSITIONAL[@]}"
|
||||
|
||||
# Parse input filename
|
||||
din=$1
|
||||
seqnum=$2
|
||||
|
||||
# Construct output filename if not specified
|
||||
if [ -z "$fout" ]; then
|
||||
fout="snapshot_scal_${seqnum}.ucf.tar"
|
||||
fi
|
||||
|
||||
# Display verbose info
|
||||
if [ $verbose -eq 1 ]; then
|
||||
(>&2 echo "Creating archive: $fout")
|
||||
fi
|
||||
|
||||
# Check input files
|
||||
fparam="parameters_${seqnum}.asc"
|
||||
fgrid="grid_${seqnum}.bin"
|
||||
fproc="proc_${seqnum}.bin"
|
||||
fpart="particles_${seqnum}.bin"
|
||||
fbscal="scal_"
|
||||
|
||||
# Check if obligatory files are present
|
||||
if [ ! -s ${din}/${fparam} ]; then
|
||||
(>&2 echo "[Error] File not found or empty: ${din}/${fparam}")
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -s ${din}/${fgrid} ]; then
|
||||
(>&2 echo "[Error] File not found or empty: ${din}/${fgrid}")
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -s ${din}/${fproc} ]; then
|
||||
(>&2 echo "[Error] File not found or empty: ${din}/${fproc}")
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -s ${din}/${fpart} ]; then
|
||||
(>&2 echo "[Error] File not found or empty: ${din}/${fpart}")
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if all velocity fields exist and are not empty
|
||||
nproc=$(cat ${din}/${fparam} | grep "nprocs" | awk '{print $NF}')
|
||||
if [ $verbose -eq 1 ]; then
|
||||
(>&2 echo "Number of processors: $nproc")
|
||||
fi
|
||||
|
||||
# Check if all scalar fields exist (if any) and are not empty
|
||||
fscal=()
|
||||
for (( ii=0; ii<$nproc; ii++ )); do
|
||||
ftmp=$(printf ${fbscal}${seqnum}.%05d $ii)
|
||||
if [ ! -s ${din}/${ftmp} ]; then
|
||||
(>&2 echo "[Error] File not found or empty: ${din}/${ftmp}")
|
||||
exit 1
|
||||
fi
|
||||
fscal+=($ftmp)
|
||||
done
|
||||
|
||||
# Verbose info
|
||||
if [ $verbose -eq 1 ]; then
|
||||
(>&2 echo "[x] parameters")
|
||||
(>&2 echo "[x] grid")
|
||||
(>&2 echo "[x] processor grid")
|
||||
(>&2 echo "[x] particles")
|
||||
(>&2 echo "[x] scalar fields")
|
||||
fi
|
||||
|
||||
# Create a full file list for the archive
|
||||
flist=(${fparam} ${fgrid} ${fproc} ${fpart} ${fscal[@]})
|
||||
|
||||
# Now tar them and remove seqence number from file names while doing so
|
||||
flagtar=""
|
||||
if [ $verbose -eq 1 ]; then
|
||||
flagtar="$flagtar --verbose"
|
||||
fi
|
||||
trap exit_script SIGINT SIGTERM
|
||||
tar $flagtar --format ustar --transform="flags=r;s|_$seqnum||" --directory=${din} -cf ${fout} ${flist[@]}
|
||||
tarexit=$?
|
||||
# Set exit status accoring to tar
|
||||
if [ $tarexit -ne 0 ]; then
|
||||
(>&2 echo "tar failed with exit code $tarexit")
|
||||
exit 254
|
||||
fi
|
||||
|
||||
# Compare checksums (CNC32), if flag is set
|
||||
#din="./archive/" #for testing
|
||||
flistx=($(echo ${flist[@]} | sed s/"_$seqnum"/""/g))
|
||||
if [ $checksum -eq 1 ]; then
|
||||
for ii in "${!flistx[@]}"; do
|
||||
if [ $verbose -eq 1 ]; then
|
||||
(>&2 echo "Verifying checksum: ${flist[$ii]}")
|
||||
fi
|
||||
crcori=$(cksum ${din}/${flist[$ii]} | awk '{ print $1, $2 }')
|
||||
crctar=$(tar --to-command='cksum -' -xf ${fout} ${flistx[$ii]} | awk '{ print $1, $2 }')
|
||||
if [ "$crcori" != "$crctar" ]; then
|
||||
(>&2 echo "Verification failed: ${flist[$ii]} ${flistx[$ii]}")
|
||||
exit 5
|
||||
fi
|
||||
done
|
||||
elif [ $quicksum -eq 1 ]; then
|
||||
for ii in "${!flistx[@]}"; do
|
||||
if [ $verbose -eq 1 ]; then
|
||||
(>&2 echo "Verifying partial checksum: ${flist[$ii]}")
|
||||
fi
|
||||
crcori=$(head -c 1M ${din}/${flist[$ii]} | cksum -)
|
||||
crctar=$(tar --to-command='head -c 1M' -xf ${fout} ${flistx[$ii]} | cksum -)
|
||||
if [ "$crcori" != "$crctar" ]; then
|
||||
(>&2 echo "Verification failed: ${flist[$ii]} ${flistx[$ii]}")
|
||||
exit 5
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
exit 0
|
||||
Loading…
Reference in New Issue