fixes in the read_uvwp_complete_legacy

.gitignore

@@ -0,0 +1 @@
+*__pycache__

bash/ucftar_aux_pack

@@ -0,0 +1,152 @@
+#!/bin/bash
+display_help() {
+    (>&2 echo "Usage: $(basename "$0") [-chqv] [-o outfile] indir iseq")
+    (>&2 echo "UCF tar auxiliary packer")
+    (>&2 echo)
+    (>&2 echo "   indir                   path to input directory")
+    (>&2 echo "   iseq                    sequence number with leading zeros")
+    (>&2 echo "   -c, --checksum          compares checksums after the archive is created")
+    (>&2 echo "   -h, --help              display this help message")
+    (>&2 echo "   -o, --outfile           output file (default: snapshot_XXXX.ucf.tar)")
+    (>&2 echo "   -q, --quicksum          same as --checksum, but compares only first bytes")
+    (>&2 echo "   -v, --verbose           verbose output")
+}
+exit_script() {
+    #trap - SIGINT SIGTERM  # clear the trap
+    #kill -- -$$            # Sends SIGTERM to child/sub processes
+    (>&2 echo "SIGINT/SIGTERM received: removing archive")
+    rm $fout
+}
+# Parse command line arguments
+if [ $# -eq 0 ]; then
+  display_help
+  exit -1
+fi
+fout=""
+verbose=0
+checksum=0
+quicksum=0
+POSITIONAL=()
+while [[ $# -gt 0 ]]
+do
+key="$1"
+case $key in
+    -h|--help)
+    display_help
+    exit 0
+    shift # past argument
+    ;;
+    -o|--outfile)
+    fout="$2"
+    shift # past argument
+    shift # past value
+    ;;
+    -v|--verbose)
+    verbose=1
+    shift # past argument
+    ;;
+    -c|--checksum)
+    checksum=1
+    shift # past argument
+    ;;
+    -q|--quicksum)
+    quicksum=1
+    shift # past argument
+    ;;
+    *)    # unknown option
+    POSITIONAL+=("$1") # save it in an array for later
+    shift # past argument
+    ;;
+esac
+done
+set -- "${POSITIONAL[@]}"
+
+# Parse input filename
+din=$1
+seqnum=$2 
+
+# Construct output filename if not specified
+if [ -z "$fout" ]; then
+  fout="auxiliary_${seqnum}.ucf.tar"
+fi
+
+# Display verbose info
+if [ $verbose -eq 1 ]; then
+  (>&2 echo "Creating archive: $fout")
+fi
+
+# Get input files: 
+# following files must exist:
+#   parameters, grid, proc
+# glob auxiliary files:
+#   every file which matches "*_${seqnum}.*", except uvwp, scal, particles
+fparam="parameters_${seqnum}.asc"
+fgrid="grid_${seqnum}.bin"
+fproc="proc_${seqnum}.bin"
+
+#GLOBIGNORE="${din}/parameters_*:${din}/grid_*:${din}/proc_*:${din}/uvwp_*:${din}/scal_*:${din}/particles_*"
+#faux=( $(basename $din/*_${seqnum}.*) )
+#unset GLOBIGNORE
+faux=( $(find ${din} -name "*_${seqnum}.*" ! -name "parameters_*" ! -name "grid_*" ! -name "proc_*" ! -name "particles_*" ! -name "uvwp_*" ! -name "scal_*" -exec basename {} \;) )
+
+# Check if obligatory files are present
+if [ ! -s ${din}/${fparam} ]; then
+  (>&2 echo "[Error] File not found or empty: ${din}/${fparam}")
+  exit 1
+fi
+if [ ! -s ${din}/${fgrid} ]; then
+  (>&2 echo "[Error] File not found or empty: ${din}/${fgrid}")
+  exit 1
+fi
+if [ ! -s ${din}/${fproc} ]; then
+  (>&2 echo "[Error] File not found or empty: ${din}/${fproc}")
+  exit 1
+fi
+
+# Create a full file list for the archive
+flist=(${fparam} ${fgrid} ${fproc} ${faux[@]})
+
+# Now tar them and remove seqence number from file names while doing so
+flagtar=""
+if [ $verbose -eq 1 ]; then
+  flagtar="$flagtar --verbose"
+fi
+trap exit_script SIGINT SIGTERM
+tar $flagtar --format ustar --transform="flags=r;s|_$seqnum||" --directory=${din} -cf ${fout} ${flist[@]}
+tarexit=$?
+# Set exit status accoring to tar
+if [ $tarexit -ne 0 ]; then
+  (>&2 echo "tar failed with exit code $tarexit")
+  exit 254
+fi
+
+# Compare checksums (CNC32), if flag is set
+#din="./archive/" #for testing
+flistx=($(echo ${flist[@]} | sed s/"_$seqnum"/""/g))
+if [ $checksum -eq 1 ]; then
+  for ii in "${!flistx[@]}"; do 
+    if [ $verbose -eq 1 ]; then
+      (>&2 echo "Verifying checksum: ${flist[$ii]}")
+    fi
+    crcori=$(cksum ${din}/${flist[$ii]} | awk '{ print $1, $2 }')
+    crctar=$(tar --to-command='cksum -' -xf ${fout} ${flistx[$ii]} | awk '{ print $1, $2 }')
+    if [ "$crcori" != "$crctar" ]; then
+      (>&2 echo "Verification failed: ${flist[$ii]} ${flistx[$ii]}")
+      exit 5
+    fi
+  done
+elif [ $quicksum -eq 1 ]; then
+  for ii in "${!flistx[@]}"; do
+    if [ $verbose -eq 1 ]; then
+      (>&2 echo "Verifying partial checksum: ${flist[$ii]}")
+    fi
+    crcori=$(head -c 1M ${din}/${flist[$ii]} | cksum -)
+    crctar=$(tar --to-command='head -c 1M' -xf ${fout} ${flistx[$ii]} | cksum -)
+    if [ "$crcori" != "$crctar" ]; then
+      (>&2 echo "Verification failed: ${flist[$ii]} ${flistx[$ii]}")
+      exit 5
+    fi
+  done
+fi
+
+exit 0

bash/ucftar_pack

@@ -0,0 +1,245 @@
+#!/bin/bash
+display_help() {
+    (>&2 echo "Usage: $(basename "$0") [-chqv] [-s size] [-o outfile] indir iseq")
+    (>&2 echo "UCF tar packer")
+    (>&2 echo)
+    (>&2 echo "   indir                   path to input directory")
+    (>&2 echo "   iseq                    sequence number with leading zeros")
+    (>&2 echo "   -c, --checksum          compares checksums after the archive is created")
+    (>&2 echo "   -h, --help              display this help message")
+    (>&2 echo "   -o, --outfile           output file (default: snapshot_XXXX.ucf.tar)")
+    (>&2 echo "   -q, --quicksum          same as --checksum, but compares only first bytes")
+    (>&2 echo "   -s, --split             split archives so that they are smaller than given size (in GiB)")
+    (>&2 echo "   -v, --verbose           verbose output")
+}
+exit_script() {
+    #trap - SIGINT SIGTERM  # clear the trap
+    #kill -- -$$            # Sends SIGTERM to child/sub processes
+    (>&2 echo "SIGINT/SIGTERM received: removing archive")
+    rm $fout
+}
+# Parse command line arguments
+if [ $# -eq 0 ]; then
+  display_help
+  exit -1
+fi
+fout=""
+maxsize=""
+verbose=0
+checksum=0
+quicksum=0
+split=0
+POSITIONAL=()
+while [[ $# -gt 0 ]]
+do
+key="$1"
+case $key in
+    -h|--help)
+    display_help
+    exit 0
+    shift # past argument
+    ;;
+    -o|--outfile)
+    fout="$2"
+    shift # past argument
+    shift # past value
+    ;;
+    -v|--verbose)
+    verbose=1
+    shift # past argument
+    ;;
+    -c|--checksum)
+    checksum=1
+    shift # past argument
+    ;;
+    -q|--quicksum)
+    quicksum=1
+    shift # past argument
+    ;;
+    -s|--split)
+    split=1
+    maxsize=$(echo "$2*1024*1024*1024" | bc -l | awk -F  "." '{print $1}') # Convert GiB to bytes
+    shift # past argument
+    shift # past value
+    ;;
+    *)    # unknown option
+    POSITIONAL+=("$1") # save it in an array for later
+    shift # past argument
+    ;;
+esac
+done
+set -- "${POSITIONAL[@]}"
+
+# Parse input filename
+din=$1
+seqnum=$2
+
+# Construct output filename if not specified
+if [ -z "$fout" ]; then
+  fout="snapshot_${seqnum}.ucf.tar"
+fi
+
+# Display verbose info
+if [ $verbose -eq 1 ]; then
+  (>&2 echo "Creating archive: $fout")
+fi
+
+# Check input files
+fparam="parameters_${seqnum}.asc"
+fgrid="grid_${seqnum}.bin"
+fproc="proc_${seqnum}.bin"
+fpart="particles_${seqnum}.bin"
+fbuvwp="uvwp_"
+
+# Check if obligatory files are present
+if [ ! -s ${din}/${fparam} ]; then
+  (>&2 echo "[Error] File not found or empty: ${din}/${fparam}")
+  exit 1
+fi
+if [ ! -s ${din}/${fgrid} ]; then
+  (>&2 echo "[Error] File not found or empty: ${din}/${fgrid}")
+  exit 1
+fi
+if [ ! -s ${din}/${fproc} ]; then
+  (>&2 echo "[Error] File not found or empty: ${din}/${fproc}")
+  exit 1
+fi
+if [ ! -s ${din}/${fpart} ]; then
+  (>&2 echo "[Error] File not found or empty: ${din}/${fpart}")
+  # exit 1
+  fpart=
+fi
+
+# Check if all velocity fields exist and are not empty
+nproc=$(cat ${din}/${fparam} | grep "nprocs" | awk '{print $NF}')
+if [ $verbose -eq 1 ]; then
+  (>&2 echo "Number of processors: $nproc")
+fi
+
+fuvwp=()
+for (( ii=0; ii<$nproc; ii++ )); do
+  ftmp=$(printf ${fbuvwp}${seqnum}.%05d $ii)
+  if [ ! -s ${din}/${ftmp} ]; then
+    (>&2 echo "[Error] File not found or empty: ${din}/${ftmp}")
+    exit 1
+  fi
+  fuvwp+=($ftmp)
+done
+
+# Verbose info
+if [ $verbose -eq 1 ]; then
+  (>&2 echo "[x] parameters")
+  (>&2 echo "[x] grid")
+  (>&2 echo "[x] processor grid")
+  (>&2 echo "[x] particles")
+  (>&2 echo "[x] fluid fields")
+fi
+
+# Create a full file list for the archive
+flist=(${fparam} ${fgrid} ${fproc} ${fpart} ${fuvwp[@]})
+
+# Construct flags of tar command
+flagtar="--format=ustar --transform=flags=r;s|_$seqnum||"
+if [ $verbose -eq 1 ]; then
+  flagtar="$flagtar --verbose"
+fi
+if [ $split -eq 1 ]; then
+  flagtar="$flagtar --blocking-factor=1"
+fi
+
+# Initialize variables needed for splitting
+if [ $split -eq 1 ]; then
+  sarchivenum=0                # Current archive number (suffix to file name)
+  scounter=0                   # Counter of current file
+  snumfilecur=0                # Current number of files to be archived
+  snumfiletot=${#flist[@]}     # Total number of files to be archived
+  scursize=0                   # Current size of archive in bytes
+  sfilesize=0                  # Size of current file in archive
+  spredictsize=0               # Predicted size of archive if current file is added
+  fout_sbase=$fout             # Original outfile name (number is appended)
+  flist_sbase=("${flist[@]}")  # Original list of files (to be split)
+  flag_splitdone=0             # Flag to indicate if all files have been packed
+  unset flist
+fi
+
+# Start packing loop (only executed once, if archive is not to be split)
+packloop=1
+trap exit_script SIGINT SIGTERM
+while [ $packloop -eq 1 ]; do
+  # Check if archive is to be split. If so, determine files to be packed in this iteration.
+  if [ $split -eq 1 ]; then
+    # Construct outfile name
+    fout="${fout_sbase}.${sarchivenum}"
+    ((sarchivenum++))
+    # Cumulate size of archive and construct file list
+    snumfilecur=0
+    scursize=1024   # Initialize with size of trailing zero bloc
+    unset flist
+    while true; do
+      sfilesize=$(wc -c ${din}/${flist_sbase[${scounter}]} | awk '{print $1}') # raw size
+      sfilesize=$(((${sfilesize}+511)/512*512+512))                            # adjust to 512 byte-blocks and add header
+      spredictsize=$((${scursize}+${sfilesize}))
+      if [ ${spredictsize} -lt ${maxsize} ]; then
+        flist[${snumfilecur}]=${flist_sbase[${scounter}]}
+        ((snumfilecur++))
+        ((scounter++))
+        scursize=${spredictsize}
+      elif [ ${snumfilecur} -eq 0 ]; then
+        (>&2 echo "Error: file larger than maximum archive size: ${flist_sbase[${scounter}]}")
+        exit 101
+      else
+        if [ $verbose -eq 1 ]; then
+          echo "Splitter: ${fout} (${snumfilecur} files, ${scursize} bytes)"
+        fi
+        break
+      fi
+      if [ ${scounter} -ge ${snumfiletot} ]; then
+        if [ $verbose -eq 1 ]; then
+          echo "Splitter: ${fout} (${snumfilecur} files, ${scursize} bytes)"
+        fi
+        flag_splitdone=1
+        break
+      fi
+    done
+  fi
+  # Create tar archive
+  tar $flagtar --directory=${din} -cf ${fout} ${flist[@]}
+  tarexit=$?
+  if [ $tarexit -ne 0 ]; then
+    (>&2 echo "tar failed with exit code $tarexit")
+    exit 254
+  fi
+  # Compare checksums (CNC32), if flag is set
+  flistx=($(echo ${flist[@]} | sed s/"_$seqnum"/""/g))
+  if [ $checksum -eq 1 ]; then
+    for ii in "${!flistx[@]}"; do
+      if [ $verbose -eq 1 ]; then
+        (>&2 echo "Verifying checksum: ${flist[$ii]}")
+      fi
+      crcori=$(cksum ${din}/${flist[$ii]} | awk '{ print $1, $2 }')
+      crctar=$(tar --to-command='cksum -' -xf ${fout} ${flistx[$ii]} | awk '{ print $1, $2 }')
+      if [ "$crcori" != "$crctar" ]; then
+        (>&2 echo "Verification failed: ${flist[$ii]} ${flistx[$ii]}")
+        exit 5
+      fi
+    done
+  elif [ $quicksum -eq 1 ]; then
+    for ii in "${!flistx[@]}"; do
+      if [ $verbose -eq 1 ]; then
+        (>&2 echo "Verifying partial checksum: ${flist[$ii]}")
+      fi
+      crcori=$(head -c 1M ${din}/${flist[$ii]} | cksum -)
+      crctar=$(tar --to-command='head -c 1M' -xf ${fout} ${flistx[$ii]} | cksum -)
+      if [ "$crcori" != "$crctar" ]; then
+        (>&2 echo "Verification failed: ${flist[$ii]} ${flistx[$ii]}")
+        exit 5
+      fi
+    done
+  fi
+  # Continue looping?
+  if ([ $split -eq 1 ] && [ $flag_splitdone -eq 1 ]) || [ $split -eq 0 ]; then
+    packloop=0
+  fi
+done
+
+exit 0

bash/ucftar_scalar_pack

@@ -0,0 +1,245 @@
+#!/bin/bash
+display_help() {
+    (>&2 echo "Usage: $(basename "$0") [-chqv] [-s size] [-o outfile] indir iseq")
+    (>&2 echo "UCF tar scalar packer")
+    (>&2 echo)
+    (>&2 echo "   indir                   path to input directory")
+    (>&2 echo "   iseq                    sequence number with leading zeros")
+    (>&2 echo "   -c, --checksum          compares checksums after the archive is created")
+    (>&2 echo "   -h, --help              display this help message")
+    (>&2 echo "   -o, --outfile           output file (default: snapshot_XXXX.ucf.tar)")
+    (>&2 echo "   -q, --quicksum          same as --checksum, but compares only first bytes")
+    (>&2 echo "   -s, --split             split archives so that they are smaller than given size (in GiB)")
+    (>&2 echo "   -v, --verbose           verbose output")
+}
+exit_script() {
+    #trap - SIGINT SIGTERM  # clear the trap
+    #kill -- -$$            # Sends SIGTERM to child/sub processes
+    (>&2 echo "SIGINT/SIGTERM received: removing archive")
+    rm $fout
+}
+# Parse command line arguments
+if [ $# -eq 0 ]; then
+  display_help
+  exit -1
+fi
+fout=""
+maxsize=""
+verbose=0
+checksum=0
+quicksum=0
+split=0
+POSITIONAL=()
+while [[ $# -gt 0 ]]
+do
+key="$1"
+case $key in
+    -h|--help)
+    display_help
+    exit 0
+    shift # past argument
+    ;;
+    -o|--outfile)
+    fout="$2"
+    shift # past argument
+    shift # past value
+    ;;
+    -v|--verbose)
+    verbose=1
+    shift # past argument
+    ;;
+    -c|--checksum)
+    checksum=1
+    shift # past argument
+    ;;
+    -q|--quicksum)
+    quicksum=1
+    shift # past argument
+    ;;
+    -s|--split)
+    split=1
+    maxsize=$(echo "$2*1024*1024*1024" | bc -l | awk -F  "." '{print $1}') # Convert GiB to bytes
+    shift # past argument
+    shift # past value
+    ;;
+    *)    # unknown option
+    POSITIONAL+=("$1") # save it in an array for later
+    shift # past argument
+    ;;
+esac
+done
+set -- "${POSITIONAL[@]}"
+
+# Parse input filename
+din=$1
+seqnum=$2 
+
+# Construct output filename if not specified
+if [ -z "$fout" ]; then
+  fout="snapshot_scal_${seqnum}.ucf.tar"
+fi
+
+# Display verbose info
+if [ $verbose -eq 1 ]; then
+  (>&2 echo "Creating archive: $fout")
+fi
+
+# Check input files
+fparam="parameters_${seqnum}.asc"
+fgrid="grid_${seqnum}.bin"
+fproc="proc_${seqnum}.bin"
+fpart="particles_${seqnum}.bin"
+fbscal="scal_"
+
+# Check if obligatory files are present
+if [ ! -s ${din}/${fparam} ]; then
+  (>&2 echo "[Error] File not found or empty: ${din}/${fparam}")
+  exit 1
+fi
+if [ ! -s ${din}/${fgrid} ]; then
+  (>&2 echo "[Error] File not found or empty: ${din}/${fgrid}")
+  exit 1
+fi
+if [ ! -s ${din}/${fproc} ]; then
+  (>&2 echo "[Error] File not found or empty: ${din}/${fproc}")
+  exit 1
+fi
+if [ ! -s ${din}/${fpart} ]; then
+  (>&2 echo "[Error] File not found or empty: ${din}/${fpart}")
+  exit 1
+fi
+
+# Check if all velocity fields exist and are not empty
+nproc=$(cat ${din}/${fparam} | grep "nprocs" | awk '{print $NF}')
+if [ $verbose -eq 1 ]; then
+  (>&2 echo "Number of processors: $nproc")
+fi
+
+# Check if all scalar fields exist (if any) and are not empty
+fscal=()
+for (( ii=0; ii<$nproc; ii++ )); do
+  ftmp=$(printf ${fbscal}${seqnum}.%05d $ii)
+  if [ ! -s ${din}/${ftmp} ]; then
+    (>&2 echo "[Error] File not found or empty: ${din}/${ftmp}")
+    exit 1
+  fi
+  fscal+=($ftmp)
+done
+
+# Verbose info
+if [ $verbose -eq 1 ]; then
+  (>&2 echo "[x] parameters")
+  (>&2 echo "[x] grid")
+  (>&2 echo "[x] processor grid")
+  (>&2 echo "[x] particles")
+  (>&2 echo "[x] scalar fields")
+fi
+
+# Create a full file list for the archive
+flist=(${fparam} ${fgrid} ${fproc} ${fpart} ${fscal[@]})
+
+# Construct flags of tar command
+flagtar="--format=ustar --transform=flags=r;s|_$seqnum||"
+if [ $verbose -eq 1 ]; then
+  flagtar="$flagtar --verbose"
+fi
+if [ $split -eq 1 ]; then
+  flagtar="$flagtar --blocking-factor=1"
+fi
+
+# Initialize variables needed for splitting
+if [ $split -eq 1 ]; then
+  sarchivenum=0                # Current archive number (suffix to file name)
+  scounter=0                   # Counter of current file
+  snumfilecur=0                # Current number of files to be archived
+  snumfiletot=${#flist[@]}     # Total number of files to be archived
+  scursize=0                   # Current size of archive in bytes
+  sfilesize=0                  # Size of current file in archive
+  spredictsize=0               # Predicted size of archive if current file is added
+  fout_sbase=$fout             # Original outfile name (number is appended)
+  flist_sbase=("${flist[@]}")  # Original list of files (to be split)
+  flag_splitdone=0             # Flag to indicate if all files have been packed
+  unset flist
+fi
+
+# Start packing loop (only executed once, if archive is not to be split)
+packloop=1
+trap exit_script SIGINT SIGTERM
+while [ $packloop -eq 1 ]; do
+  # Check if archive is to be split. If so, determine files to be packed in this iteration.
+  if [ $split -eq 1 ]; then
+    # Construct outfile name
+    fout="${fout_sbase}.${sarchivenum}"
+    ((sarchivenum++))
+    # Cumulate size of archive and construct file list
+    snumfilecur=0
+    scursize=1024   # Initialize with size of trailing zero bloc
+    unset flist
+    while true; do 
+      sfilesize=$(wc -c ${din}/${flist_sbase[${scounter}]} | awk '{print $1}') # raw size
+      sfilesize=$(((${sfilesize}+511)/512*512+512))                            # adjust to 512 byte-blocks and add header
+      spredictsize=$((${scursize}+${sfilesize}))
+      if [ ${spredictsize} -lt ${maxsize} ]; then
+        flist[${snumfilecur}]=${flist_sbase[${scounter}]}
+        ((snumfilecur++))
+        ((scounter++))
+        scursize=${spredictsize}
+      elif [ ${snumfilecur} -eq 0 ]; then
+        (>&2 echo "Error: file larger than maximum archive size: ${flist_sbase[${scounter}]}")
+        exit 101
+      else
+        if [ $verbose -eq 1 ]; then
+          echo "Splitter: ${fout} (${snumfilecur} files, ${scursize} bytes)"
+        fi
+        break
+      fi
+      if [ ${scounter} -ge ${snumfiletot} ]; then
+        if [ $verbose -eq 1 ]; then
+          echo "Splitter: ${fout} (${snumfilecur} files, ${scursize} bytes)"
+        fi
+        flag_splitdone=1
+        break
+      fi 
+    done 
+  fi
+  # Create tar archive
+  tar $flagtar --directory=${din} -cf ${fout} ${flist[@]}
+  tarexit=$?
+  if [ $tarexit -ne 0 ]; then
+    (>&2 echo "tar failed with exit code $tarexit")
+    exit 254
+  fi
+  # Compare checksums (CNC32), if flag is set
+  flistx=($(echo ${flist[@]} | sed s/"_$seqnum"/""/g))
+  if [ $checksum -eq 1 ]; then
+    for ii in "${!flistx[@]}"; do 
+      if [ $verbose -eq 1 ]; then
+        (>&2 echo "Verifying checksum: ${flist[$ii]}")
+      fi
+      crcori=$(cksum ${din}/${flist[$ii]} | awk '{ print $1, $2 }')
+      crctar=$(tar --to-command='cksum -' -xf ${fout} ${flistx[$ii]} | awk '{ print $1, $2 }')
+      if [ "$crcori" != "$crctar" ]; then
+        (>&2 echo "Verification failed: ${flist[$ii]} ${flistx[$ii]}")
+        exit 5
+      fi
+    done
+  elif [ $quicksum -eq 1 ]; then
+    for ii in "${!flistx[@]}"; do
+      if [ $verbose -eq 1 ]; then
+        (>&2 echo "Verifying partial checksum: ${flist[$ii]}")
+      fi
+      crcori=$(head -c 1M ${din}/${flist[$ii]} | cksum -)
+      crctar=$(tar --to-command='head -c 1M' -xf ${fout} ${flistx[$ii]} | cksum -)
+      if [ "$crcori" != "$crctar" ]; then
+        (>&2 echo "Verification failed: ${flist[$ii]} ${flistx[$ii]}")
+        exit 5
+      fi
+    done
+  fi
+  # Continue looping?
+  if ([ $split -eq 1 ] && [ $flag_splitdone -eq 1 ]) || [ $split -eq 0 ]; then
+    packloop=0
+  fi
+done
+
+exit 0

matlab/+jsonlab/AUTHORS.txt

@@ -0,0 +1,63 @@
+The author of "jsonlab" toolbox is Qianqian Fang. Qianqian
+is currently an Assistant Professor in the Department of Bioengineering,
+Northeastern University.
+
+Address: Qianqian Fang
+         Department of Bioengineering
+         Northeastern University
+         212A Lake Hall
+         360 Huntington Ave, Boston, MA 02115, USA
+         Office:   503 Holmes Hall
+         Phone[O]: 617-373-3829
+URL: http://fanglab.org
+Email: <q.fang at neu.edu> and <fangqq at gmail.com>
+
+
+The script loadjson.m was built upon previous works by
+
+- Nedialko Krouchev: http://www.mathworks.com/matlabcentral/fileexchange/25713
+       date: 2009/11/02
+- François Glineur: http://www.mathworks.com/matlabcentral/fileexchange/23393
+       date: 2009/03/22
+- Joel Feenstra: http://www.mathworks.com/matlabcentral/fileexchange/20565
+       date: 2008/07/03
+
+The data compression/decompression utilities ({zlib,gzip,base64}{encode,decode}.m)
+were copied from 
+
+- "Byte encoding utilities" by Kota Yamaguchi
+       https://www.mathworks.com/matlabcentral/fileexchange/39526-byte-encoding-utilities
+       date: 2013/01/04
+
+
+This toolbox contains patches submitted by the following contributors:
+
+- Blake Johnson <bjohnso at bbn.com>
+  part of revision 341
+
+- Niclas Borlin <Niclas.Borlin at cs.umu.se>
+  various fixes in revision 394, including
+  - loadjson crashes for all-zero sparse matrix.
+  - loadjson crashes for empty sparse matrix.
+  - Non-zero size of 0-by-N and N-by-0 empty matrices is lost after savejson/loadjson.
+  - loadjson crashes for sparse real column vector.
+  - loadjson crashes for sparse complex column vector.
+  - Data is corrupted by savejson for sparse real row vector.
+  - savejson crashes for sparse complex row vector. 
+
+- Yul Kang <yul.kang.on at gmail.com>
+  patches for svn revision 415.
+  - savejson saves an empty cell array as [] instead of null
+  - loadjson differentiates an empty struct from an empty array
+
+- Mykhailo Bratukha <bratukha.m at gmail.com>
+  (Pull#14) Bug fix: File path is wrongly inerpreted as JSON string
+
+- Insik Kim <insik92 at gmail.com>
+  (Pull#12) Bug fix: Resolving bug that cell type is converted to json with transposed data
+
+- Sertan Senturk <contact at sertansenturk.com>
+  (Pull#10,#11) Feature: Added matlab object saving to savejson and saveubjson
+
+- Paul Koprowski <https://github.com/pjkoprowski>
+  (Issue#29) Feature: Added support to save MATLAB tables to json.

matlab/+jsonlab/ChangeLog.txt

@@ -0,0 +1,140 @@
+============================================================================
+
+   JSONlab - a toolbox to encode/decode JSON/UBJSON files in MATLAB/Octave
+
+----------------------------------------------------------------------------
+
+JSONlab ChangeLog (key features marked by *):
+
+== JSONlab 1.9 (codename: Magnus - alpha), FangQ <q.fang <at> neu.edu> ==
+
+ 2019-05-06 [25ad795] unescape string in loadjson.m
+ 2019-05-04 [2e317c9] explain extra compression fields
+ 2019-05-02 [1b1be65] avoid side effect of removing singletarray
+ 2019-05-02*[8360fd1] support zmat based base64 encoding and decoding
+ 2019-05-01*[c797bb2] integrating zmat, for zlib/gzip data compression
+ 2019-04-29 [70551fe] remove warnings from matlab
+ 2019-04-28 [0d61c4b] complete data compression support, close #52
+ 2019-04-27 [804115b] avoid typecast error
+ 2019-04-27 [c166aa7] change default compressarraysize to 100
+ 2019-04-27*[3322f6f] major new feature: support array compression and decompression
+ 2019-03-13*[9c01046] support saving function handles, close #51
+ 2019-03-13 [a8fde38] add option to parse string array or convert to char, close #50
+ 2019-03-12 [ed2645e] treat string array as cell array in newer matlab
+ 2018-11-18 [c3eb021] allow saving uint64 integers in saveubjson, fix #49
+
+== JSONlab 1.8 (codename: Nominus), FangQ <q.fang <at> neu.edu> ==
+
+ 2018-07-12 [03a6c25] update documentation, bump version to 1.8, tag Nominus
+ 2018-07-12*[1597106] add patch provided by pjkoprowski to support MATLAB table, add RowNames support, fix #29
+ 2018-07-12 [f16cc57] fix #31, throw an error when : array construct is used
+ 2018-07-12 [956e000] drop octave 3.x support, fix ubjson error in octave
+ 2018-07-12 [e090f0a] fix octave warning for saveubjson
+ 2018-07-12*[34284c7] fix issues #34 #39 #44 and #45, support double-quoted strings
+ 2017-09-06 [474d8c8] Merge pull request #41 from dasantonym/master
+ 2017-08-07*[38b24fb] added package.json to be able to intall via npm package manager, converted readme to utf-8, added basic .gitignore file
+ 2017-07-19 [ae7a5d9] Merge pull request #40 from astorfi/master
+ 2017-07-17 [154ef61] Rename README.txt to README.rst
+ 2017-03-27 [31b5bdc] simplify condition flow in matching_bracket
+ 2017-03-27 [86ef12a] avoid error in matlab 2017a, close #34
+ 2017-02-18 [4a09ac3] Merge pull request #32 from vrichter/master
+ 2017-02-14 [e67d3a3] respect integer types
+
+== JSONlab 1.5 (codename: Nominus - alpha), FangQ <q.fang <at> neu.edu> ==
+
+ 2017/01/02 *use Big-endian format to store floating points (d/D) in saveubjson (Issue #25)
+ 2017/01/02 *speedup parsing large unstructured data by 2x (Issue #9)
+ 2017/01/01  make parsing independent of white space (Issue #30)
+ 2016/08/27  allow to parse array of homogeneous elements (Issue 5)
+ 2016/08/22  permit [] inside file names in savejson
+ 2016/01/06  fix a bug that prevents saving to a file in savejson
+ 
+
+== JSONlab 1.2 (codename: Optimus - Update 2), FangQ <q.fang <at> neu.edu> ==
+
+ 2015/12/16 *replacing string concatenation by str cells to gain 2x speed in savejson (Issue#17)
+ 2015/12/11  fix FileName option case bug (SVN rev#495)
+ 2015/12/11  add SingletCell option, add SingletArray to replace NoRowBracket (Issue#15,#8)
+ 2015/11/10  fix bug for inerpreting file names as JSON string - by Mykhailo Bratukha (Pull#14)
+ 2015/10/16  fix bug for cell with transposed data - by Insik Kim (Pull#12)
+ 2015/09/25  support exporting matlab object to JSON - by Sertan Senturk (Pull#10, #11)
+
+== JSONlab 1.1 (codename: Optimus - Update 1), FangQ <q.fang <at> neu.edu> ==
+
+ 2015/05/05 *massively accelerating loadjson for parsing large collection of unstructured small objects
+ 2015/05/05  force array bracket in 1x1 struct to maintain depth (Issue#1)
+ 2015/05/05  parse logicals in loadjson
+ 2015/05/05  make options case insensitive
+ 2015/05/01  reading unicode encoded json files (thanks to Sertan Senturk,Issue#3)
+ 2015/04/30  allow \uXXXX to represent a unicode in a string (Issue#2)
+ 2015/03/30  save a 0x0 solid real empty array as null and handel empty struct array
+ 2015/03/30  properly handle escape characters in a string
+ 2015/01/24 *implement the UBJSON Draft12 new name format
+ 2015/01/13  correct cell array indentation inconsistency
+
+== JSONlab 1.0 (codename: Optimus - Final), FangQ <q.fang <at> neu.edu> ==
+
+ 2015/01/02  polish help info for all major functions, update examples, finalize 1.0
+ 2014/12/19  fix a bug to strictly respect NoRowBracket in savejson
+
+== JSONlab 1.0.0-RC2 (codename: Optimus - RC2), FangQ <q.fang <at> neu.edu> ==
+
+ 2014/11/22  show progress bar in loadjson ('ShowProgress') 
+ 2014/11/17 *add Compact option in savejson to output compact JSON format ('Compact')
+ 2014/11/17  add FastArrayParser in loadjson to specify fast parser applicable levels
+ 2014/09/18 *start official github mirror: https://github.com/fangq/jsonlab
+
+== JSONlab 1.0.0-RC1 (codename: Optimus - RC1), FangQ <q.fang <at> neu.edu> ==
+
+ 2014/09/17  fix several compatibility issues when running on octave versions 3.2-3.8
+ 2014/09/17 *support 2D cell and struct arrays in both savejson and saveubjson
+ 2014/08/04  escape special characters in a JSON string
+ 2014/02/16  fix a bug when saving ubjson files
+
+== JSONlab 0.9.9 (codename: Optimus - beta), FangQ <q.fang <at> neu.edu> ==
+
+ 2014/01/22  use binary read and write in saveubjson and loadubjson
+
+== JSONlab 0.9.8-1 (codename: Optimus - alpha update 1), FangQ <q.fang <at> neu.edu> ==
+
+ 2013/10/07  better round-trip conservation for empty arrays and structs (patch submitted by Yul Kang)
+
+== JSONlab 0.9.8 (codename: Optimus - alpha), FangQ <q.fang <at> neu.edu> ==
+ 2013/08/23 *universal Binary JSON (UBJSON) support, including both saveubjson and loadubjson
+
+== JSONlab 0.9.1 (codename: Rodimus, update 1), FangQ <q.fang <at> neu.edu> ==
+ 2012/12/18 *handling of various empty and sparse matrices (fixes submitted by Niclas Borlin)
+
+== JSONlab 0.9.0 (codename: Rodimus), FangQ <q.fang <at> neu.edu> ==
+
+ 2012/06/17 *new format for an invalid leading char, unpacking hex code in savejson
+ 2012/06/01  support JSONP in savejson
+ 2012/05/25  fix the empty cell bug (reported by Cyril Davin)
+ 2012/04/05  savejson can save to a file (suggested by Patrick Rapin)
+
+== JSONlab 0.8.1 (codename: Sentiel, Update 1), FangQ <q.fang <at> neu.edu> ==
+
+ 2012/02/28  loadjson quotation mark escape bug, see http://bit.ly/yyk1nS
+ 2012/01/25  patch to handle root-less objects, contributed by Blake Johnson
+
+== JSONlab 0.8.0 (codename: Sentiel), FangQ <q.fang <at> neu.edu> ==
+
+ 2012/01/13 *speed up loadjson by 20 fold when parsing large data arrays in matlab
+ 2012/01/11  remove row bracket if an array has 1 element, suggested by Mykel Kochenderfer
+ 2011/12/22 *accept sequence of 'param',value input in savejson and loadjson
+ 2011/11/18  fix struct array bug reported by Mykel Kochenderfer
+
+== JSONlab 0.5.1 (codename: Nexus Update 1), FangQ <q.fang <at> neu.edu> ==
+
+ 2011/10/21  fix a bug in loadjson, previous code does not use any of the acceleration
+ 2011/10/20  loadjson supports JSON collections - concatenated JSON objects
+
+== JSONlab 0.5.0 (codename: Nexus), FangQ <q.fang <at> neu.edu> ==
+
+ 2011/10/16  package and release jsonlab 0.5.0
+ 2011/10/15 *add json demo and regression test, support cpx numbers, fix double quote bug
+ 2011/10/11 *speed up readjson dramatically, interpret _Array* tags, show data in root level
+ 2011/10/10  create jsonlab project, start jsonlab website, add online documentation
+ 2011/10/07 *speed up savejson by 25x using sprintf instead of mat2str, add options support
+ 2011/10/06 *savejson works for structs, cells and arrays
+ 2011/09/09  derive loadjson from JSON parser from MATLAB Central, draft savejson.m

matlab/+jsonlab/LICENSE_BSD.txt

@@ -0,0 +1,55 @@
+Copyright 2011-2019 Qianqian Fang <q.fang <at> neu.edu>. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are
+permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice, this list of
+      conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice, this list
+      of conditions and the following disclaimer in the documentation and/or other materials
+      provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ''AS IS'' AND ANY EXPRESS OR IMPLIED
+WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS 
+OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The views and conclusions contained in the software and documentation are those of the
+authors and should not be interpreted as representing official policies, either expressed
+or implied, of the copyright holders.
+
+
+
+
+For the included compression/decompression utilities (base64encode.m, base64decode.m,
+gzipencode.m, gzipdecode.m, zlibencode.m, zlibdecode.m), the author Kota Yamaguchi
+requires the following copyright declaration:
+
+Copyright (c) 2012, Kota Yamaguchi
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

matlab/+jsonlab/README.rst

@@ -0,0 +1,579 @@
+##############################################################################                                                      
+      JSONLab: An open-source MATLAB/Octave JSON encoder and decoder             
+##############################################################################
+
+* Copyright (C) 2011-2019  Qianqian Fang <q.fang at neu.edu>
+* License: BSD, License_BSD.txt for details
+* Version: 1.9 (Magnus - alpha)
+
+
+#################
+Table of Contents
+#################
+.. contents::
+  :local:
+  :depth: 3
+
+============
+What's New
+============
+
+JSONLab v1.9 is the alpha release of the next milestone - code named "Magnus".
+Notable changes are summarized below, key features marked by *:
+
+- 2019-05-06 [25ad795] unescape string in loadjson.m
+- 2019-05-04 [2e317c9] explain extra compression fields
+- 2019-05-02 [1b1be65] avoid side effect of removing singletarray
+- 2019-05-02*[8360fd1] support zmat based base64 encoding and decoding
+- 2019-05-01*[c797bb2] integrating zmat, for zlib/gzip data compression
+- 2019-04-29 [70551fe] remove warnings from matlab
+- 2019-04-28 [0d61c4b] complete data compression support, close #52
+- 2019-04-27 [804115b] avoid typecast error
+- 2019-04-27 [c166aa7] change default compressarraysize to 100
+- 2019-04-27*[3322f6f] major new feature: support array compression and decompression
+- 2019-03-13*[9c01046] support saving function handles, close #51
+- 2019-03-13 [a8fde38] add option to parse string array or convert to char, close #50
+- 2019-03-12 [ed2645e] treat string array as cell array in newer matlab
+- 2018-11-18 [c3eb021] allow saving uint64 integers in saveubjson, fix #49
+
+The biggest change in this release, compared to v1.8 released in July 2018,
+is the support of data compression via the 'Compression' option for both
+savejson and saveubjson. Two compression methods are currently supported - 
+"zlib" and "gzip". The compression interfaces, zlibencode/zlibdecode/gzipencode/
+gzipdecode are modified from the "Byte Encoding Utilities" by Kota Yamaguchi [1],
+which has built-in support for java-based compression in MATLAB (when jvm is 
+enabled). To support Octave, as well as MATLAB in "nojvm" mode, a mex-based
+data compression/encoding toolbox, ZMat [2], written by Qianqian Fang, takes priority
+over the java-based utilities, if installed. For savejson, a 'base64' encoding is
+applied to convert the compressed binary stream into a string; 'base64' encoding
+is not used in saveubjson. The encoding and restoration of the binary matlab arrays
+are automatically handled in save*json/load*json round-trip conversions.
+
+To save matlab data with compression, one simply append 'Compression', 'method' pair
+in the savejson/saveubjson call. For example
+
+.. code:: matlab
+
+  jsonstr=savejson('',mydata,'compression','zlib');
+  data=loadjson(jsonstr);
+
+In addition, the below features are added to JSONLab
+
+  * save function handles
+  * support saving "string" class in MATLAB
+  * fix two bugs in saveubjson
+  * unescape strings in loadjson
+
+
+- [1] https://www.mathworks.com/matlabcentral/fileexchange/39526-byte-encoding-utilities
+- [2] http://github.com/fangq/zmat
+
+============
+Introduction
+============
+
+JSONLab is a free and open-source implementation of a JSON/UBJSON encoder 
+and a decoder in the native MATLAB language. It can be used to convert a MATLAB 
+data structure (array, struct, cell, struct array and cell array) into 
+JSON/UBJSON formatted strings, or to decode a JSON/UBJSON file into MATLAB 
+data structure. JSONLab supports both MATLAB and `GNU Octave <http://www.gnu.org/software/octave/>`_ (a free MATLAB clone).
+
+JSON (`JavaScript Object Notation <http://www.json.org/>`_) is a highly portable, 
+human-readable and " `fat-free <http://en.wikipedia.org/wiki/JSON>`_" text format 
+to represent complex and hierarchical data. It is as powerful as `XML <http://en.wikipedia.org/wiki/XML>`_, 
+but less verbose. JSON format is widely used for data-exchange in applications.
+
+UBJSON (`Universal Binary JSON <http://ubjson.org/>`_) is a binary JSON format, specifically 
+optimized for compact file size and better performance while keeping
+the semantics as simple as the text-based JSON format. Using the UBJSON
+format allows to wrap complex binary data in a flexible and extensible
+structure, making it possible to process complex and large dataset 
+without accuracy loss due to text conversions.
+
+We envision that both JSON and its binary version will play important roles
+as mainstream data-exchange formats for scientific research.
+It has both the flexibility and generality as offered by other popular 
+general-purpose file specifications, such as `HDF5 <http://www.hdfgroup.org/HDF5/whatishdf5.html>`_, 
+but with significantly reduced complexity and excellent readability.
+
+================
+Installation
+================
+
+The installation of JSONLab is no different than any other simple
+MATLAB toolbox. You only need to download/unzip the JSONLab package
+to a folder, and add the folder's path to MATLAB/Octave's path list
+by using the following command:
+
+.. code:: shell
+
+    addpath('/path/to/jsonlab');
+
+If you want to add this path permanently, you need to type "pathtool", 
+browse to the zmat root folder and add to the list, then click "Save".
+Then, run "rehash" in MATLAB, and type "which savejson", if you see an 
+output, that means JSONLab is installed for MATLAB/Octave.
+
+If you use MATLAB in a shared environment such as a Linux server, the
+best way to add path is to type 
+
+.. code:: shell
+
+   mkdir ~/matlab/
+   nano ~/matlab/startup.m
+
+and type addpath('/path/to/jsonlab') in this file, save and quit the editor.
+MATLAB will execute this file every time it starts. For Octave, the file
+you need to edit is ~/.octaverc , where "~" is your home directory.
+
+
+================
+Using JSONLab
+================
+
+JSONLab provides two functions, loadjson.m -- a MATLAB->JSON decoder, 
+and savejson.m -- a MATLAB->JSON encoder, for the text-based JSON, and 
+two equivallent functions -- loadubjson and saveubjson for the binary 
+JSON. The detailed help info for the four functions can be found below:
+
+----------
+loadjson.m
+----------
+
+.. code-block:: matlab
+
+  data=loadjson(fname,opt)
+     or
+  data=loadjson(fname,'param1',value1,'param2',value2,...)
+ 
+  parse a JSON (JavaScript Object Notation) file or string
+ 
+  authors:Qianqian Fang (q.fang <at> neu.edu)
+  created on 2011/09/09, including previous works from 
+ 
+          Nedialko Krouchev: http://www.mathworks.com/matlabcentral/fileexchange/25713
+             created on 2009/11/02
+          François Glineur: http://www.mathworks.com/matlabcentral/fileexchange/23393
+             created on  2009/03/22
+          Joel Feenstra:
+          http://www.mathworks.com/matlabcentral/fileexchange/20565
+             created on 2008/07/03
+ 
+  $Id$
+ 
+  input:
+       fname: input file name, if fname contains "{}" or "[]", fname
+              will be interpreted as a JSON string
+       opt: a struct to store parsing options, opt can be replaced by 
+            a list of ('param',value) pairs - the param string is equivallent
+            to a field in opt. opt can have the following 
+            fields (first in [.|.] is the default)
+ 
+            opt.SimplifyCell [0|1]: if set to 1, loadjson will call cell2mat
+                          for each element of the JSON data, and group 
+                          arrays based on the cell2mat rules.
+            opt.FastArrayParser [1|0 or integer]: if set to 1, use a
+                          speed-optimized array parser when loading an 
+                          array object. The fast array parser may 
+                          collapse block arrays into a single large
+                          array similar to rules defined in cell2mat; 0 to 
+                          use a legacy parser; if set to a larger-than-1
+                          value, this option will specify the minimum
+                          dimension to enable the fast array parser. For
+                          example, if the input is a 3D array, setting
+                          FastArrayParser to 1 will return a 3D array;
+                          setting to 2 will return a cell array of 2D
+                          arrays; setting to 3 will return to a 2D cell
+                          array of 1D vectors; setting to 4 will return a
+                          3D cell array.
+            opt.ShowProgress [0|1]: if set to 1, loadjson displays a progress bar.
+            opt.ParseStringArray [0|1]: if set to 1, loadjson displays a progress bar.
+ 
+  output:
+       dat: a cell array, where {...} blocks are converted into cell arrays,
+            and [...] are converted to arrays
+ 
+  examples:
+       dat=loadjson('{"obj":{"string":"value","array":[1,2,3]}}')
+       dat=loadjson(['examples' filesep 'example1.json'])
+       dat=loadjson(['examples' filesep 'example1.json'],'SimplifyCell',1)
+ 
+  license:
+      BSD, see LICENSE_BSD.txt file for details 
+
+----------
+savejson.m
+----------
+
+.. code-block:: matlab
+
+  json=savejson(rootname,obj,filename)
+     or
+  json=savejson(rootname,obj,opt)
+  json=savejson(rootname,obj,'param1',value1,'param2',value2,...)
+ 
+  convert a MATLAB object (cell, struct or array) into a JSON (JavaScript
+  Object Notation) string
+ 
+  author: Qianqian Fang (q.fang <at> neu.edu)
+  created on 2011/09/09
+ 
+  $Id$
+ 
+  input:
+       rootname: the name of the root-object, when set to '', the root name
+         is ignored, however, when opt.ForceRootName is set to 1 (see below),
+         the MATLAB variable name will be used as the root name.
+       obj: a MATLAB object (array, cell, cell array, struct, struct array,
+       class instance).
+       filename: a string for the file name to save the output JSON data.
+       opt: a struct for additional options, ignore to use default values.
+         opt can have the following fields (first in [.|.] is the default)
+ 
+         opt.FileName [''|string]: a file name to save the output JSON data
+         opt.FloatFormat ['%.10g'|string]: format to show each numeric element
+                          of a 1D/2D array;
+         opt.ArrayIndent [1|0]: if 1, output explicit data array with
+                          precedent indentation; if 0, no indentation
+         opt.ArrayToStruct[0|1]: when set to 0, savejson outputs 1D/2D
+                          array in JSON array format; if sets to 1, an
+                          array will be shown as a struct with fields
+                          "_ArrayType_", "_ArraySize_" and "_ArrayData_"; for
+                          sparse arrays, the non-zero elements will be
+                          saved to _ArrayData_ field in triplet-format i.e.
+                          (ix,iy,val) and "_ArrayIsSparse_" will be added
+                          with a value of 1; for a complex array, the 
+                          _ArrayData_ array will include two columns 
+                          (4 for sparse) to record the real and imaginary 
+                          parts, and also "_ArrayIsComplex_":1 is added. 
+         opt.ParseLogical [0|1]: if this is set to 1, logical array elem
+                          will use true/false rather than 1/0.
+         opt.SingletArray [0|1]: if this is set to 1, arrays with a single
+                          numerical element will be shown without a square
+                          bracket, unless it is the root object; if 0, square
+                          brackets are forced for any numerical arrays.
+         opt.SingletCell  [1|0]: if 1, always enclose a cell with "[]" 
+                          even it has only one element; if 0, brackets
+                          are ignored when a cell has only 1 element.
+         opt.ForceRootName [0|1]: when set to 1 and rootname is empty, savejson
+                          will use the name of the passed obj variable as the 
+                          root object name; if obj is an expression and 
+                          does not have a name, 'root' will be used; if this 
+                          is set to 0 and rootname is empty, the root level 
+                          will be merged down to the lower level.
+         opt.Inf ['"$1_Inf_"'|string]: a customized regular expression pattern
+                          to represent +/-Inf. The matched pattern is '([-+]*)Inf'
+                          and $1 represents the sign. For those who want to use
+                          1e999 to represent Inf, they can set opt.Inf to '$11e999'
+         opt.NaN ['"_NaN_"'|string]: a customized regular expression pattern
+                          to represent NaN
+         opt.JSONP [''|string]: to generate a JSONP output (JSON with padding),
+                          for example, if opt.JSONP='foo', the JSON data is
+                          wrapped inside a function call as 'foo(...);'
+         opt.UnpackHex [1|0]: conver the 0x[hex code] output by loadjson 
+                          back to the string form
+         opt.SaveBinary [0|1]: 1 - save the JSON file in binary mode; 0 - text mode.
+         opt.Compact [0|1]: 1- out compact JSON format (remove all newlines and tabs)
+         opt.Compression  'zlib' or 'gzip': specify array compression
+                          method; currently only supports 'gzip' or 'zlib'. The
+                          data compression only applicable to numerical arrays 
+                          in 3D or higher dimensions, or when ArrayToStruct
+                          is 1 for 1D or 2D arrays. If one wants to
+                          compress a long string, one must convert
+                          it to uint8 or int8 array first. The compressed
+                          array uses three extra fields
+                          "_ArrayCompressionMethod_": the opt.Compression value. 
+                          "_ArrayCompressionSize_": a 1D interger array to
+                             store the pre-compressed (but post-processed)
+                             array dimensions, and 
+                          "_ArrayCompressedData_": the "base64" encoded
+                              compressed binary array data. 
+         opt.CompressArraySize [100|int]: only to compress an array if the total 
+                          element count is larger than this number.
+         opt can be replaced by a list of ('param',value) pairs. The param 
+         string is equivallent to a field in opt and is case sensitive.
+  output:
+       json: a string in the JSON format (see http://json.org)
+ 
+  examples:
+       jsonmesh=struct('MeshNode',[0 0 0;1 0 0;0 1 0;1 1 0;0 0 1;1 0 1;0 1 1;1 1 1],... 
+                'MeshTetra',[1 2 4 8;1 3 4 8;1 2 6 8;1 5 6 8;1 5 7 8;1 3 7 8],...
+                'MeshTri',[1 2 4;1 2 6;1 3 4;1 3 7;1 5 6;1 5 7;...
+                           2 8 4;2 8 6;3 8 4;3 8 7;5 8 6;5 8 7],...
+                'MeshCreator','FangQ','MeshTitle','T6 Cube',...
+                'SpecialData',[nan, inf, -inf]);
+       savejson('jmesh',jsonmesh)
+       savejson('',jsonmesh,'ArrayIndent',0,'FloatFormat','\t%.5g')
+ 
+  license:
+      BSD, see LICENSE_BSD.txt file for details
+
+-------------
+loadubjson.m
+-------------
+
+.. code-block:: matlab
+
+  data=loadubjson(fname,opt)
+     or
+  data=loadubjson(fname,'param1',value1,'param2',value2,...)
+ 
+  parse a JSON (JavaScript Object Notation) file or string
+ 
+  authors:Qianqian Fang (q.fang <at> neu.edu)
+  created on 2013/08/01
+ 
+  $Id$
+ 
+  input:
+       fname: input file name, if fname contains "{}" or "[]", fname
+              will be interpreted as a UBJSON string
+       opt: a struct to store parsing options, opt can be replaced by 
+            a list of ('param',value) pairs - the param string is equivallent
+            to a field in opt. opt can have the following 
+            fields (first in [.|.] is the default)
+ 
+            opt.SimplifyCell [0|1]: if set to 1, loadubjson will call cell2mat
+                          for each element of the JSON data, and group 
+                          arrays based on the cell2mat rules.
+            opt.IntEndian [B|L]: specify the endianness of the integer fields
+                          in the UBJSON input data. B - Big-Endian format for 
+                          integers (as required in the UBJSON specification); 
+                          L - input integer fields are in Little-Endian order.
+            opt.NameIsString [0|1]: for UBJSON Specification Draft 8 or 
+                          earlier versions (JSONLab 1.0 final or earlier), 
+                          the "name" tag is treated as a string. To load 
+                          these UBJSON data, you need to manually set this 
+                          flag to 1.
+ 
+  output:
+       dat: a cell array, where {...} blocks are converted into cell arrays,
+            and [...] are converted to arrays
+ 
+  examples:
+       obj=struct('string','value','array',[1 2 3]);
+       ubjdata=saveubjson('obj',obj);
+       dat=loadubjson(ubjdata)
+       dat=loadubjson(['examples' filesep 'example1.ubj'])
+       dat=loadubjson(['examples' filesep 'example1.ubj'],'SimplifyCell',1)
+ 
+  license:
+      BSD, see LICENSE_BSD.txt file for details 
+
+-------------
+saveubjson.m
+-------------
+
+
+.. code-block:: matlab
+
+  json=saveubjson(rootname,obj,filename)
+     or
+  json=saveubjson(rootname,obj,opt)
+  json=saveubjson(rootname,obj,'param1',value1,'param2',value2,...)
+ 
+  convert a MATLAB object (cell, struct or array) into a Universal 
+  Binary JSON (UBJSON) binary string
+ 
+  author: Qianqian Fang (q.fang <at> neu.edu)
+  created on 2013/08/17
+ 
+  $Id$
+ 
+  input:
+       rootname: the name of the root-object, when set to '', the root name
+         is ignored, however, when opt.ForceRootName is set to 1 (see below),
+         the MATLAB variable name will be used as the root name.
+       obj: a MATLAB object (array, cell, cell array, struct, struct array,
+       class instance)
+       filename: a string for the file name to save the output UBJSON data
+       opt: a struct for additional options, ignore to use default values.
+         opt can have the following fields (first in [.|.] is the default)
+ 
+         opt.FileName [''|string]: a file name to save the output JSON data
+         opt.ArrayToStruct[0|1]: when set to 0, saveubjson outputs 1D/2D
+                          array in JSON array format; if sets to 1, an
+                          array will be shown as a struct with fields
+                          "_ArrayType_", "_ArraySize_" and "_ArrayData_"; for
+                          sparse arrays, the non-zero elements will be
+                          saved to _ArrayData_ field in triplet-format i.e.
+                          (ix,iy,val) and "_ArrayIsSparse_" will be added
+                          with a value of 1; for a complex array, the 
+                          _ArrayData_ array will include two columns 
+                          (4 for sparse) to record the real and imaginary 
+                          parts, and also "_ArrayIsComplex_":1 is added. 
+         opt.ParseLogical [1|0]: if this is set to 1, logical array elem
+                          will use true/false rather than 1/0.
+         opt.SingletArray [0|1]: if this is set to 1, arrays with a single
+                          numerical element will be shown without a square
+                          bracket, unless it is the root object; if 0, square
+                          brackets are forced for any numerical arrays.
+         opt.SingletCell  [1|0]: if 1, always enclose a cell with "[]" 
+                          even it has only one element; if 0, brackets
+                          are ignored when a cell has only 1 element.
+         opt.ForceRootName [0|1]: when set to 1 and rootname is empty, saveubjson
+                          will use the name of the passed obj variable as the 
+                          root object name; if obj is an expression and 
+                          does not have a name, 'root' will be used; if this 
+                          is set to 0 and rootname is empty, the root level 
+                          will be merged down to the lower level.
+         opt.JSONP [''|string]: to generate a JSONP output (JSON with padding),
+                          for example, if opt.JSON='foo', the JSON data is
+                          wrapped inside a function call as 'foo(...);'
+         opt.UnpackHex [1|0]: conver the 0x[hex code] output by loadjson 
+                          back to the string form
+         opt.Compression  'zlib' or 'gzip': specify array compression
+                          method; currently only supports 'gzip' or 'zlib'. The
+                          data compression only applicable to numerical arrays 
+                          in 3D or higher dimensions, or when ArrayToStruct
+                          is 1 for 1D or 2D arrays. If one wants to
+                          compress a long string, one must convert
+                          it to uint8 or int8 array first. The compressed
+                          array uses three extra fields
+                          "_ArrayCompressionMethod_": the opt.Compression value. 
+                          "_ArrayCompressionSize_": a 1D interger array to
+                             store the pre-compressed (but post-processed)
+                             array dimensions, and 
+                          "_ArrayCompressedData_": the binary stream of
+                             the compressed binary array data WITHOUT
+                             'base64' encoding
+         opt.CompressArraySize [100|int]: only to compress an array if the total 
+                          element count is larger than this number.
+ 
+         opt can be replaced by a list of ('param',value) pairs. The param 
+         string is equivallent to a field in opt and is case sensitive.
+  output:
+       json: a binary string in the UBJSON format (see http://ubjson.org)
+ 
+  examples:
+       jsonmesh=struct('MeshNode',[0 0 0;1 0 0;0 1 0;1 1 0;0 0 1;1 0 1;0 1 1;1 1 1],... 
+                'MeshTetra',[1 2 4 8;1 3 4 8;1 2 6 8;1 5 6 8;1 5 7 8;1 3 7 8],...
+                'MeshTri',[1 2 4;1 2 6;1 3 4;1 3 7;1 5 6;1 5 7;...
+                           2 8 4;2 8 6;3 8 4;3 8 7;5 8 6;5 8 7],...
+                'MeshCreator','FangQ','MeshTitle','T6 Cube',...
+                'SpecialData',[nan, inf, -inf]);
+       saveubjson('jsonmesh',jsonmesh)
+       saveubjson('jsonmesh',jsonmesh,'meshdata.ubj')
+ 
+  license:
+      BSD, see LICENSE_BSD.txt file for details
+
+---------
+examples
+---------
+
+Under the ``"examples"`` folder, you can find several scripts to demonstrate the
+basic utilities of JSONLab. Running the ``"demo_jsonlab_basic.m"`` script, you 
+will see the conversions from MATLAB data structure to JSON text and backward.
+In ``"jsonlab_selftest.m"``, we load complex JSON files downloaded from the Internet
+and validate the ``loadjson/savejson`` functions for regression testing purposes.
+Similarly, a ``"demo_ubjson_basic.m"`` script is provided to test the saveubjson
+and loadubjson functions for various matlab data structures.
+
+Please run these examples and understand how JSONLab works before you use
+it to process your data.
+
+=======================
+Known Issues and TODOs
+=======================
+
+JSONLab has several known limitations. We are striving to make it more general
+and robust. Hopefully in a few future releases, the limitations become less.
+
+Here are the known issues:
+
+  * 3D or higher dimensional cell/struct-arrays will be converted to 2D arrays
+  
+  * When processing names containing multi-byte characters, Octave and MATLAB can give different field-names; you can use feature('DefaultCharacterSet','latin1') in MATLAB to get consistant results
+  
+  * savejson can not handle class and dataset.
+  
+  * saveubjson converts a logical array into a uint8 ([U]) array
+  
+  * an unofficial N-D array count syntax is implemented in saveubjson. We are actively communicating with the UBJSON spec maintainer to investigate the possibility of making it upstream 
+  
+  * loadubjson can not parse all UBJSON Specification (Draft 9) compliant files, however, it can parse all UBJSON files produced by saveubjson.
+
+==========================
+Contribution and feedback
+==========================
+
+JSONLab is an open-source project. This means you can not only use it and modify
+it as you wish, but also you can contribute your changes back to JSONLab so
+that everyone else can enjoy the improvement. For anyone who want to contribute,
+please download JSONLab source code from its source code repositories by using the
+following command:
+
+
+.. code:: shell
+
+      git clone https://github.com/fangq/jsonlab.git jsonlab
+
+or browsing the github site at
+
+      https://github.com/fangq/jsonlab
+
+Please report any bugs or issues to the below URL:
+
+      https://github.com/fangq/jsonlab/issues
+
+Sometimes, you may find it is necessary to modify JSONLab to achieve your 
+goals, or attempt to modify JSONLab functions to fix a bug that you have 
+encountered. If you are happy with your changes and willing to share those
+changes to the upstream author, you are recommended to create a pull-request
+on github. 
+
+To create a pull-request, you first need to "fork" jsonlab on Github by 
+clicking on the "fork" button on top-right of jsonlab's github page. Once you forked
+jsonlab to your own directory, you should then implement the changes in your
+own fork. After thoroughly testing it and you are confident the modification 
+is complete and effective, you can then click on the "New pull request" 
+button, and on the left, select fangq/jsonlab as the "base". Then type
+in the description of the changes. You are responsible to format the code
+updates using the same convention (tab-width: 8, indentation: 4 spaces) as
+the upstream code.
+
+We appreciate any suggestions and feedbacks from you. Please use the following
+mailing list to report any questions you may have regarding JSONLab:
+
+      https://github.com/fangq/jsonlab/issues
+
+(Subscription to the mailing list is needed in order to post messages).
+
+
+==========================
+Acknowledgement
+==========================
+
+---------
+zlibdecode.m, zlibencode.m, gzipencode.m, gzipdecode.m, base64encode.m, base64decode.m
+---------
+
+* Author: Kota Yamaguchi
+* URL: https://www.mathworks.com/matlabcentral/fileexchange/39526-byte-encoding-utilities
+* License: BSD License, see below
+
+```
+Copyright (c) 2012, Kota Yamaguchi
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+```

matlab/+jsonlab/README.txt

@@ -0,0 +1,565 @@
+===============================================================================
+=                                 JSONLab                                     =
+=           An open-source MATLAB/Octave JSON encoder and decoder             =
+===============================================================================
+
+*Copyright (C) 2011-2019  Qianqian Fang <q.fang at neu.edu>
+*License: BSD, License_BSD.txt for details
+*Version: 1.9 (Magnus - alpha)
+
+-------------------------------------------------------------------------------
+
+Table of Content:
+
+0.  What's New
+I.  Introduction
+II. Installation
+III.Using JSONLab
+IV. Known Issues and TODOs
+V.  Contribution and feedback
+V.  Acknowledgement
+
+-------------------------------------------------------------------------------
+
+0. What's New
+
+JSONLab v1.9 is the alpha release of the next milestone - code named "Magnus".
+Notable changes are summarized below, key features marked by *:
+
+ 2019-05-06 [25ad795] unescape string in loadjson.m
+ 2019-05-04 [2e317c9] explain extra compression fields
+ 2019-05-02 [1b1be65] avoid side effect of removing singletarray
+ 2019-05-02*[8360fd1] support zmat based base64 encoding and decoding
+ 2019-05-01*[c797bb2] integrating zmat, for zlib/gzip data compression
+ 2019-04-29 [70551fe] remove warnings from matlab
+ 2019-04-28 [0d61c4b] complete data compression support, close #52
+ 2019-04-27 [804115b] avoid typecast error
+ 2019-04-27 [c166aa7] change default compressarraysize to 100
+ 2019-04-27*[3322f6f] major new feature: support array compression and decompression
+ 2019-03-13*[9c01046] support saving function handles, close #51
+ 2019-03-13 [a8fde38] add option to parse string array or convert to char, close #50
+ 2019-03-12 [ed2645e] treat string array as cell array in newer matlab
+ 2018-11-18 [c3eb021] allow saving uint64 integers in saveubjson, fix #49
+
+The biggest change in this release, compared to v1.8 released in July 2018,
+is the support of data compression via the 'Compression' option for both
+savejson and saveubjson. Two compression methods are currently supported - 
+"zlib" and "gzip". The compression interfaces, zlibencode/zlibdecode/gzipencode/
+gzipdecode are modified from the "Byte Encoding Utilities" by Kota Yamaguchi [1],
+which has built-in support for java-based compression in MATLAB (when jvm is 
+enabled). To support Octave, as well as MATLAB in "nojvm" mode, a mex-based
+data compression/encoding toolbox, ZMat [2], written by Qianqian Fang, takes priority
+over the java-based utilities, if installed. For savejson, a 'base64' encoding is
+applied to convert the compressed binary stream into a string; 'base64' encoding
+is not used in saveubjson. The encoding and restoration of the binary matlab arrays
+are automatically handled in save*json/load*json round-trip conversions.
+
+To save matlab data with compression, one simply append 'Compression', 'method' pair
+in the savejson/saveubjson call. For example
+
+  jsonstr=savejson('',mydata,'compression','zlib');
+  data=loadjson(jsonstr);
+
+In addition, the below features are added to JSONLab
+
+* save function handles
+* support saving "string" class in MATLAB
+* fix two bugs in saveubjson
+* unescape strings in loadjson
+
+
+* [1] https://www.mathworks.com/matlabcentral/fileexchange/39526-byte-encoding-utilities
+* [2] http://github.com/fangq/zmat
+
+-------------------------------------------------------------------------------
+
+I.  Introduction
+
+JSONLab is a free and open-source implementation of a JSON/UBJSON encoder 
+and a decoder in the native MATLAB language. It can be used to convert a MATLAB 
+data structure (array, struct, cell, struct array and cell array) into 
+JSON/UBJSON formatted strings, or to decode a JSON/UBJSON file into MATLAB 
+data structure. JSONLab supports both MATLAB and  
+[http://www.gnu.org/software/octave/ GNU Octave] (a free MATLAB clone).
+
+JSON ([http://www.json.org/ JavaScript Object Notation]) is a highly portable, 
+human-readable and "[http://en.wikipedia.org/wiki/JSON fat-free]" text format 
+to represent complex and hierarchical data. It is as powerful as 
+[http://en.wikipedia.org/wiki/XML XML], but less verbose. JSON format is widely 
+used for data-exchange in applications, and is essential for the wild success 
+of [http://en.wikipedia.org/wiki/Ajax_(programming) Ajax] and 
+[http://en.wikipedia.org/wiki/Web_2.0 Web2.0]. 
+
+UBJSON ([<http://ubjson.org/ Universal Binary JSON]) is a binary JSON format, specifically 
+optimized for compact file size and better performance while keeping
+the semantics as simple as the text-based JSON format. Using the UBJSON
+format allows to wrap complex binary data in a flexible and extensible
+structure, making it possible to process complex and large dataset 
+without accuracy loss due to text conversions.
+
+We envision that both JSON and its binary version will serve as part of 
+the mainstream data-exchange formats for scientific research in the future. 
+It will provide the flexibility and generality achieved by other popular 
+general-purpose file specifications, such as
+[http://www.hdfgroup.org/HDF5/whatishdf5.html HDF5], with significantly 
+reduced complexity and enhanced performance.
+
+-------------------------------------------------------------------------------
+
+II. Installation
+
+The installation of JSONLab is no different than any other simple
+MATLAB toolbox. You only need to download/unzip the JSONLab package
+to a folder, and add the folder's path to MATLAB/Octave's path list
+by using the following command:
+
+    addpath('/path/to/jsonlab');
+
+If you want to add this path permanently, you need to type "pathtool", 
+browse to the zmat root folder and add to the list, then click "Save".
+Then, run "rehash" in MATLAB, and type "which savejson", if you see an 
+output, that means JSONLab is installed for MATLAB/Octave.
+
+If you use MATLAB in a shared environment such as a Linux server, the
+best way to add path is to type 
+
+   mkdir ~/matlab/
+   nano ~/matlab/startup.m
+
+and type addpath('/path/to/jsonlab') in this file, save and quit the editor.
+MATLAB will execute this file every time it starts. For Octave, the file
+you need to edit is ~/.octaverc , where "~" is your home directory.
+
+-------------------------------------------------------------------------------
+
+III.Using JSONLab
+
+JSONLab provides two functions, loadjson.m -- a MATLAB->JSON decoder, 
+and savejson.m -- a MATLAB->JSON encoder, for the text-based JSON, and 
+two equivallent functions -- loadubjson and saveubjson for the binary 
+JSON. The detailed help info for the four functions can be found below:
+
+=== loadjson.m ===
+<pre>
+  data=loadjson(fname,opt)
+     or
+  data=loadjson(fname,'param1',value1,'param2',value2,...)
+ 
+  parse a JSON (JavaScript Object Notation) file or string
+ 
+  authors:Qianqian Fang (q.fang <at> neu.edu)
+  created on 2011/09/09, including previous works from 
+ 
+          Nedialko Krouchev: http://www.mathworks.com/matlabcentral/fileexchange/25713
+             created on 2009/11/02
+          François Glineur: http://www.mathworks.com/matlabcentral/fileexchange/23393
+             created on  2009/03/22
+          Joel Feenstra:
+          http://www.mathworks.com/matlabcentral/fileexchange/20565
+             created on 2008/07/03
+ 
+  $Id$
+ 
+  input:
+       fname: input file name, if fname contains "{}" or "[]", fname
+              will be interpreted as a JSON string
+       opt: a struct to store parsing options, opt can be replaced by 
+            a list of ('param',value) pairs - the param string is equivallent
+            to a field in opt. opt can have the following 
+            fields (first in [.|.] is the default)
+ 
+            opt.SimplifyCell [0|1]: if set to 1, loadjson will call cell2mat
+                          for each element of the JSON data, and group 
+                          arrays based on the cell2mat rules.
+            opt.FastArrayParser [1|0 or integer]: if set to 1, use a
+                          speed-optimized array parser when loading an 
+                          array object. The fast array parser may 
+                          collapse block arrays into a single large
+                          array similar to rules defined in cell2mat; 0 to 
+                          use a legacy parser; if set to a larger-than-1
+                          value, this option will specify the minimum
+                          dimension to enable the fast array parser. For
+                          example, if the input is a 3D array, setting
+                          FastArrayParser to 1 will return a 3D array;
+                          setting to 2 will return a cell array of 2D
+                          arrays; setting to 3 will return to a 2D cell
+                          array of 1D vectors; setting to 4 will return a
+                          3D cell array.
+            opt.ShowProgress [0|1]: if set to 1, loadjson displays a progress bar.
+            opt.ParseStringArray [0|1]: if set to 1, loadjson displays a progress bar.
+ 
+  output:
+       dat: a cell array, where {...} blocks are converted into cell arrays,
+            and [...] are converted to arrays
+ 
+  examples:
+       dat=loadjson('{"obj":{"string":"value","array":[1,2,3]}}')
+       dat=loadjson(['examples' filesep 'example1.json'])
+       dat=loadjson(['examples' filesep 'example1.json'],'SimplifyCell',1)
+ 
+  license:
+      BSD, see LICENSE_BSD.txt file for details 
+  </pre>
+
+=== savejson.m ===
+
+<pre>
+  json=savejson(rootname,obj,filename)
+     or
+  json=savejson(rootname,obj,opt)
+  json=savejson(rootname,obj,'param1',value1,'param2',value2,...)
+ 
+  convert a MATLAB object (cell, struct or array) into a JSON (JavaScript
+  Object Notation) string
+ 
+  author: Qianqian Fang (q.fang <at> neu.edu)
+  created on 2011/09/09
+ 
+  $Id$
+ 
+  input:
+       rootname: the name of the root-object, when set to '', the root name
+         is ignored, however, when opt.ForceRootName is set to 1 (see below),
+         the MATLAB variable name will be used as the root name.
+       obj: a MATLAB object (array, cell, cell array, struct, struct array,
+       class instance).
+       filename: a string for the file name to save the output JSON data.
+       opt: a struct for additional options, ignore to use default values.
+         opt can have the following fields (first in [.|.] is the default)
+ 
+         opt.FileName [''|string]: a file name to save the output JSON data
+         opt.FloatFormat ['%.10g'|string]: format to show each numeric element
+                          of a 1D/2D array;
+         opt.ArrayIndent [1|0]: if 1, output explicit data array with
+                          precedent indentation; if 0, no indentation
+         opt.ArrayToStruct[0|1]: when set to 0, savejson outputs 1D/2D
+                          array in JSON array format; if sets to 1, an
+                          array will be shown as a struct with fields
+                          "_ArrayType_", "_ArraySize_" and "_ArrayData_"; for
+                          sparse arrays, the non-zero elements will be
+                          saved to _ArrayData_ field in triplet-format i.e.
+                          (ix,iy,val) and "_ArrayIsSparse_" will be added
+                          with a value of 1; for a complex array, the 
+                          _ArrayData_ array will include two columns 
+                          (4 for sparse) to record the real and imaginary 
+                          parts, and also "_ArrayIsComplex_":1 is added. 
+         opt.ParseLogical [0|1]: if this is set to 1, logical array elem
+                          will use true/false rather than 1/0.
+         opt.SingletArray [0|1]: if this is set to 1, arrays with a single
+                          numerical element will be shown without a square
+                          bracket, unless it is the root object; if 0, square
+                          brackets are forced for any numerical arrays.
+         opt.SingletCell  [1|0]: if 1, always enclose a cell with "[]" 
+                          even it has only one element; if 0, brackets
+                          are ignored when a cell has only 1 element.
+         opt.ForceRootName [0|1]: when set to 1 and rootname is empty, savejson
+                          will use the name of the passed obj variable as the 
+                          root object name; if obj is an expression and 
+                          does not have a name, 'root' will be used; if this 
+                          is set to 0 and rootname is empty, the root level 
+                          will be merged down to the lower level.
+         opt.Inf ['"$1_Inf_"'|string]: a customized regular expression pattern
+                          to represent +/-Inf. The matched pattern is '([-+]*)Inf'
+                          and $1 represents the sign. For those who want to use
+                          1e999 to represent Inf, they can set opt.Inf to '$11e999'
+         opt.NaN ['"_NaN_"'|string]: a customized regular expression pattern
+                          to represent NaN
+         opt.JSONP [''|string]: to generate a JSONP output (JSON with padding),
+                          for example, if opt.JSONP='foo', the JSON data is
+                          wrapped inside a function call as 'foo(...);'
+         opt.UnpackHex [1|0]: conver the 0x[hex code] output by loadjson 
+                          back to the string form
+         opt.SaveBinary [0|1]: 1 - save the JSON file in binary mode; 0 - text mode.
+         opt.Compact [0|1]: 1- out compact JSON format (remove all newlines and tabs)
+         opt.Compression  'zlib' or 'gzip': specify array compression
+                          method; currently only supports 'gzip' or 'zlib'. The
+                          data compression only applicable to numerical arrays 
+                          in 3D or higher dimensions, or when ArrayToStruct
+                          is 1 for 1D or 2D arrays. If one wants to
+                          compress a long string, one must convert
+                          it to uint8 or int8 array first. The compressed
+                          array uses three extra fields
+                          "_ArrayCompressionMethod_": the opt.Compression value. 
+                          "_ArrayCompressionSize_": a 1D interger array to
+                             store the pre-compressed (but post-processed)
+                             array dimensions, and 
+                          "_ArrayCompressedData_": the "base64" encoded
+                              compressed binary array data. 
+         opt.CompressArraySize [100|int]: only to compress an array if the total 
+                          element count is larger than this number.
+         opt can be replaced by a list of ('param',value) pairs. The param 
+         string is equivallent to a field in opt and is case sensitive.
+  output:
+       json: a string in the JSON format (see http://json.org)
+ 
+  examples:
+       jsonmesh=struct('MeshNode',[0 0 0;1 0 0;0 1 0;1 1 0;0 0 1;1 0 1;0 1 1;1 1 1],... 
+                'MeshTetra',[1 2 4 8;1 3 4 8;1 2 6 8;1 5 6 8;1 5 7 8;1 3 7 8],...
+                'MeshTri',[1 2 4;1 2 6;1 3 4;1 3 7;1 5 6;1 5 7;...
+                           2 8 4;2 8 6;3 8 4;3 8 7;5 8 6;5 8 7],...
+                'MeshCreator','FangQ','MeshTitle','T6 Cube',...
+                'SpecialData',[nan, inf, -inf]);
+       savejson('jmesh',jsonmesh)
+       savejson('',jsonmesh,'ArrayIndent',0,'FloatFormat','\t%.5g')
+ 
+  license:
+      BSD, see LICENSE_BSD.txt file for details
+ </pre>
+
+=== loadubjson.m ===
+
+<pre>
+  data=loadubjson(fname,opt)
+     or
+  data=loadubjson(fname,'param1',value1,'param2',value2,...)
+ 
+  parse a JSON (JavaScript Object Notation) file or string
+ 
+  authors:Qianqian Fang (q.fang <at> neu.edu)
+  created on 2013/08/01
+ 
+  $Id$
+ 
+  input:
+       fname: input file name, if fname contains "{}" or "[]", fname
+              will be interpreted as a UBJSON string
+       opt: a struct to store parsing options, opt can be replaced by 
+            a list of ('param',value) pairs - the param string is equivallent
+            to a field in opt. opt can have the following 
+            fields (first in [.|.] is the default)
+ 
+            opt.SimplifyCell [0|1]: if set to 1, loadubjson will call cell2mat
+                          for each element of the JSON data, and group 
+                          arrays based on the cell2mat rules.
+            opt.IntEndian [B|L]: specify the endianness of the integer fields
+                          in the UBJSON input data. B - Big-Endian format for 
+                          integers (as required in the UBJSON specification); 
+                          L - input integer fields are in Little-Endian order.
+            opt.NameIsString [0|1]: for UBJSON Specification Draft 8 or 
+                          earlier versions (JSONLab 1.0 final or earlier), 
+                          the "name" tag is treated as a string. To load 
+                          these UBJSON data, you need to manually set this 
+                          flag to 1.
+ 
+  output:
+       dat: a cell array, where {...} blocks are converted into cell arrays,
+            and [...] are converted to arrays
+ 
+  examples:
+       obj=struct('string','value','array',[1 2 3]);
+       ubjdata=saveubjson('obj',obj);
+       dat=loadubjson(ubjdata)
+       dat=loadubjson(['examples' filesep 'example1.ubj'])
+       dat=loadubjson(['examples' filesep 'example1.ubj'],'SimplifyCell',1)
+ 
+  license:
+      BSD, see LICENSE_BSD.txt file for details 
+</pre>
+
+=== saveubjson.m ===
+
+<pre>
+  json=saveubjson(rootname,obj,filename)
+     or
+  json=saveubjson(rootname,obj,opt)
+  json=saveubjson(rootname,obj,'param1',value1,'param2',value2,...)
+ 
+  convert a MATLAB object (cell, struct or array) into a Universal 
+  Binary JSON (UBJSON) binary string
+ 
+  author: Qianqian Fang (q.fang <at> neu.edu)
+  created on 2013/08/17
+ 
+  $Id$
+ 
+  input:
+       rootname: the name of the root-object, when set to '', the root name
+         is ignored, however, when opt.ForceRootName is set to 1 (see below),
+         the MATLAB variable name will be used as the root name.
+       obj: a MATLAB object (array, cell, cell array, struct, struct array,
+       class instance)
+       filename: a string for the file name to save the output UBJSON data
+       opt: a struct for additional options, ignore to use default values.
+         opt can have the following fields (first in [.|.] is the default)
+ 
+         opt.FileName [''|string]: a file name to save the output JSON data
+         opt.ArrayToStruct[0|1]: when set to 0, saveubjson outputs 1D/2D
+                          array in JSON array format; if sets to 1, an
+                          array will be shown as a struct with fields
+                          "_ArrayType_", "_ArraySize_" and "_ArrayData_"; for
+                          sparse arrays, the non-zero elements will be
+                          saved to _ArrayData_ field in triplet-format i.e.
+                          (ix,iy,val) and "_ArrayIsSparse_" will be added
+                          with a value of 1; for a complex array, the 
+                          _ArrayData_ array will include two columns 
+                          (4 for sparse) to record the real and imaginary 
+                          parts, and also "_ArrayIsComplex_":1 is added. 
+         opt.ParseLogical [1|0]: if this is set to 1, logical array elem
+                          will use true/false rather than 1/0.
+         opt.SingletArray [0|1]: if this is set to 1, arrays with a single
+                          numerical element will be shown without a square
+                          bracket, unless it is the root object; if 0, square
+                          brackets are forced for any numerical arrays.
+         opt.SingletCell  [1|0]: if 1, always enclose a cell with "[]" 
+                          even it has only one element; if 0, brackets
+                          are ignored when a cell has only 1 element.
+         opt.ForceRootName [0|1]: when set to 1 and rootname is empty, saveubjson
+                          will use the name of the passed obj variable as the 
+                          root object name; if obj is an expression and 
+                          does not have a name, 'root' will be used; if this 
+                          is set to 0 and rootname is empty, the root level 
+                          will be merged down to the lower level.
+         opt.JSONP [''|string]: to generate a JSONP output (JSON with padding),
+                          for example, if opt.JSON='foo', the JSON data is
+                          wrapped inside a function call as 'foo(...);'
+         opt.UnpackHex [1|0]: conver the 0x[hex code] output by loadjson 
+                          back to the string form
+         opt.Compression  'zlib' or 'gzip': specify array compression
+                          method; currently only supports 'gzip' or 'zlib'. The
+                          data compression only applicable to numerical arrays 
+                          in 3D or higher dimensions, or when ArrayToStruct
+                          is 1 for 1D or 2D arrays. If one wants to
+                          compress a long string, one must convert
+                          it to uint8 or int8 array first. The compressed
+                          array uses three extra fields
+                          "_ArrayCompressionMethod_": the opt.Compression value. 
+                          "_ArrayCompressionSize_": a 1D interger array to
+                             store the pre-compressed (but post-processed)
+                             array dimensions, and 
+                          "_ArrayCompressedData_": the binary stream of
+                             the compressed binary array data WITHOUT
+                             'base64' encoding
+         opt.CompressArraySize [100|int]: only to compress an array if the total 
+                          element count is larger than this number.
+ 
+         opt can be replaced by a list of ('param',value) pairs. The param 
+         string is equivallent to a field in opt and is case sensitive.
+  output:
+       json: a binary string in the UBJSON format (see http://ubjson.org)
+ 
+  examples:
+       jsonmesh=struct('MeshNode',[0 0 0;1 0 0;0 1 0;1 1 0;0 0 1;1 0 1;0 1 1;1 1 1],... 
+                'MeshTetra',[1 2 4 8;1 3 4 8;1 2 6 8;1 5 6 8;1 5 7 8;1 3 7 8],...
+                'MeshTri',[1 2 4;1 2 6;1 3 4;1 3 7;1 5 6;1 5 7;...
+                           2 8 4;2 8 6;3 8 4;3 8 7;5 8 6;5 8 7],...
+                'MeshCreator','FangQ','MeshTitle','T6 Cube',...
+                'SpecialData',[nan, inf, -inf]);
+       saveubjson('jsonmesh',jsonmesh)
+       saveubjson('jsonmesh',jsonmesh,'meshdata.ubj')
+ 
+  license:
+      BSD, see LICENSE_BSD.txt file for details
+</pre>
+
+
+=== examples ===
+
+Under the "examples" folder, you can find several scripts to demonstrate the
+basic utilities of JSONLab. Running the "demo_jsonlab_basic.m" script, you 
+will see the conversions from MATLAB data structure to JSON text and backward.
+In "jsonlab_selftest.m", we load complex JSON files downloaded from the Internet
+and validate the loadjson/savejson functions for regression testing purposes.
+Similarly, a "demo_ubjson_basic.m" script is provided to test the saveubjson
+and loadubjson functions for various matlab data structures.
+
+Please run these examples and understand how JSONLab works before you use
+it to process your data.
+
+-------------------------------------------------------------------------------
+
+IV. Known Issues and TODOs
+
+JSONLab has several known limitations. We are striving to make it more general
+and robust. Hopefully in a few future releases, the limitations become less.
+
+Here are the known issues:
+
+# 3D or higher dimensional cell/struct-arrays will be converted to 2D arrays;
+# When processing names containing multi-byte characters, Octave and MATLAB \
+can give different field-names; you can use feature('DefaultCharacterSet','latin1') \
+in MATLAB to get consistant results
+# savejson can not handle class and dataset.
+# saveubjson converts a logical array into a uint8 ([U]) array
+# an unofficial N-D array count syntax is implemented in saveubjson. We are \
+actively communicating with the UBJSON spec maintainer to investigate the \
+possibility of making it upstream
+# loadubjson can not parse all UBJSON Specification (Draft 9) compliant \
+files, however, it can parse all UBJSON files produced by saveubjson.
+
+-------------------------------------------------------------------------------
+
+V. Contribution and feedback
+
+JSONLab is an open-source project. This means you can not only use it and modify
+it as you wish, but also you can contribute your changes back to JSONLab so
+that everyone else can enjoy the improvement. For anyone who want to contribute,
+please download JSONLab source code from its source code repositories by using the
+following command:
+
+      git clone https://github.com/fangq/jsonlab.git jsonlab
+
+or browsing the github site at
+
+      https://github.com/fangq/jsonlab
+
+Please report any bugs or issues to the below URL:
+
+      https://github.com/fangq/jsonlab/issues
+
+Sometimes, you may find it is necessary to modify JSONLab to achieve your 
+goals, or attempt to modify JSONLab functions to fix a bug that you have 
+encountered. If you are happy with your changes and willing to share those
+changes to the upstream author, you are recommended to create a pull-request
+on github. 
+
+To create a pull-request, you first need to "fork" jsonlab on Github by 
+clicking on the "fork" button on top-right of jsonlab's github page. Once you forked
+jsonlab to your own directory, you should then implement the changes in your
+own fork. After thoroughly testing it and you are confident the modification 
+is complete and effective, you can then click on the "New pull request" 
+button, and on the left, select fangq/jsonlab as the "base". Then type
+in the description of the changes. You are responsible to format the code
+updates using the same convention (tab-width: 8, indentation: 4 spaces) as
+the upstream code.
+
+We appreciate any suggestions and feedbacks from you. Please use the following
+mailing list to report any questions you may have regarding JSONLab:
+
+      https://github.com/fangq/jsonlab/issues
+
+(Subscription to the mailing list is needed in order to post messages).
+
+-------------------------------------------------------------------------------
+
+V.  Acknowledgement
+
+This toolbox contains modified functions from the below toolboxes:
+
+== zlibdecode.m, zlibencode.m, gzipencode.m, gzipdecode.m, base64encode.m, base64decode.m ==
+
+* Author: Kota Yamaguchi
+* URL:https://www.mathworks.com/matlabcentral/fileexchange/39526-byte-encoding-utilities
+* License: BSD License, see below
+
+Copyright (c) 2012, Kota Yamaguchi
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

matlab/+jsonlab/base64decode.m

@@ -0,0 +1,39 @@
+function output = base64decode(input)
+import jsonlab.*
+%BASE64DECODE Decode Base64 string to a byte array.
+%
+%    output = base64decode(input)
+%
+% The function takes a Base64 string INPUT and returns a uint8 array
+% OUTPUT. JAVA must be running to use this function. The result is always
+% given as a 1-by-N array, and doesn't retrieve the original dimensions.
+%
+% See also base64encode
+%
+% Copyright (c) 2012, Kota Yamaguchi
+% URL: https://www.mathworks.com/matlabcentral/fileexchange/39526-byte-encoding-utilities
+% License : BSD, see LICENSE_*.txt
+%
+
+if(nargin==0)
+    error('you must provide at least 1 input');
+end
+if(exist('zmat')==3)
+    output=zmat(uint8(input),0,'base64');
+    return;
+end
+if(exist('OCTAVE_VERSION','builtin'))
+    len=rem(numel(input),8)
+    if(len)
+       input=[input(:)', repmat(sprintf('\0'),1,(8-len))];
+    end
+    output = base64_decode(input);
+    return;
+end
+error(javachk('jvm'));
+if ischar(input), input = uint8(input); end
+
+output = typecast(org.apache.commons.codec.binary.Base64.decodeBase64(input), 'uint8')';
+
+end
+

matlab/+jsonlab/base64encode.m

@@ -0,0 +1,34 @@
+function output = base64encode(input)
+import jsonlab.*
+%BASE64ENCODE Encode a byte array using Base64 codec.
+%
+%    output = base64encode(input)
+%
+% The function takes a char, int8, or uint8 array INPUT and returns Base64
+% encoded string OUTPUT. JAVA must be running to use this function. Note
+% that encoding doesn't preserve input dimensions.
+%
+% See also base64decode
+%
+% Copyright (c) 2012, Kota Yamaguchi
+% URL: https://www.mathworks.com/matlabcentral/fileexchange/39526-byte-encoding-utilities
+% License : BSD, see LICENSE_*.txt
+%
+
+if(nargin==0)
+    error('you must provide at least 1 input');
+end
+if(exist('zmat')==3)
+    output=zmat(uint8(input),1,'base64');
+    return;
+end
+if(exist('OCTAVE_VERSION','builtin'))
+    output = base64_encode(uint8(input));
+    return;
+end
+error(javachk('jvm'));
+if ischar(input), input = uint8(input); end
+
+output = char(org.apache.commons.codec.binary.Base64.encodeBase64Chunked(input))';
+output = regexprep(output,'\r','');
+end

matlab/+jsonlab/examples/demo_jsonlab_basic.m

@@ -0,0 +1,181 @@
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%         Demonstration of Basic Utilities of JSONlab
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+rngstate = rand ('state');
+randseed=hex2dec('623F9A9E');
+clear data2json json2data
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  a simple scalar value \n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=pi
+savejson('',data2json)
+json2data=loadjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  a complex number\n')
+fprintf(1,'%%=================================================\n\n')
+
+clear i;
+data2json=1+2*i
+savejson('',data2json)
+json2data=loadjson(ans) 
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  a complex matrix\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=magic(6);
+data2json=data2json(:,1:3)+data2json(:,4:6)*i
+savejson('',data2json)
+json2data=loadjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  MATLAB special constants\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=[NaN Inf -Inf]
+savejson('specials',data2json)
+json2data=loadjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  a real sparse matrix\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=sprand(10,10,0.1)
+savejson('sparse',data2json)
+json2data=loadjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  a complex sparse matrix\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=data2json-data2json*i
+savejson('complex_sparse',data2json)
+json2data=loadjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  an all-zero sparse matrix\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=sparse(2,3);
+savejson('all_zero_sparse',data2json)
+json2data=loadjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  an empty sparse matrix\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=sparse([]);
+savejson('empty_sparse',data2json)
+json2data=loadjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  an empty 0-by-0 real matrix\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=[];
+savejson('empty_0by0_real',data2json)
+json2data=loadjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  an empty 0-by-3 real matrix\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=zeros(0,3);
+savejson('empty_0by3_real',data2json)
+json2data=loadjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  a sparse real column vector\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=sparse([0,3,0,1,4]');
+savejson('sparse_column_vector',data2json)
+json2data=loadjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  a sparse complex column vector\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=data2json-1i*data2json;
+savejson('complex_sparse_column_vector',data2json)
+json2data=loadjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  a sparse real row vector\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=sparse([0,3,0,1,4]);
+savejson('sparse_row_vector',data2json)
+json2data=loadjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  a sparse complex row vector\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=data2json-1i*data2json;
+savejson('complex_sparse_row_vector',data2json)
+json2data=loadjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  a structure\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=struct('name','Think Different','year',1997,'magic',magic(3),...
+                 'misfits',[Inf,NaN],'embedded',struct('left',true,'right',false))
+savejson('astruct',data2json,struct('ParseLogical',1))
+json2data=loadjson(ans)
+class(json2data.astruct.embedded.left)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  a structure array\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=struct('name','Nexus Prime','rank',9);
+data2json(2)=struct('name','Sentinel Prime','rank',9);
+data2json(3)=struct('name','Optimus Prime','rank',9);
+savejson('Supreme Commander',data2json)
+json2data=loadjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  a cell array\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=cell(3,1);
+data2json{1}=struct('buzz',1.1,'rex',1.2,'bo',1.3,'hamm',2.0,'slink',2.1,'potato',2.2,...
+              'woody',3.0,'sarge',3.1,'etch',4.0,'lenny',5.0,'squeeze',6.0,'wheezy',7.0);
+data2json{2}=struct('Ubuntu',['Kubuntu';'Xubuntu';'Lubuntu']);
+data2json{3}=[10.04,10.10,11.04,11.10]
+savejson('debian',data2json,struct('FloatFormat','%.2f'))
+json2data=loadjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  invalid field-name handling\n')
+fprintf(1,'%%=================================================\n\n')
+
+json2data=loadjson('{"ValidName":1, "_InvalidName":2, ":Field:":3, "项目":"绝密"}')
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  a 2D cell array\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json={{1,{2,3}},{4,5},{6};{7},{8,9},{10}};
+savejson('data2json',data2json)
+json2data=loadjson(ans)  % only savejson works for cell arrays, loadjson has issues
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  a 2D struct array\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=repmat(struct('idx',0,'data','structs'),[2,3])
+for i=1:6
+    data2json(i).idx=i;
+end
+savejson('data2json',data2json)
+json2data=loadjson(ans)
+
+rand ('state',rngstate);
+

matlab/+jsonlab/examples/demo_ubjson_basic.m

@@ -0,0 +1,180 @@
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%         Demonstration of Basic Utilities of JSONlab
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+rngstate = rand ('state');
+randseed=hex2dec('623F9A9E');
+clear data2json json2data
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  a simple scalar value \n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=pi
+saveubjson('',data2json)
+json2data=loadubjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  a complex number\n')
+fprintf(1,'%%=================================================\n\n')
+
+clear i;
+data2json=1+2*i
+saveubjson('',data2json)
+json2data=loadubjson(ans) 
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  a complex matrix\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=magic(6);
+data2json=data2json(:,1:3)+data2json(:,4:6)*i
+saveubjson('',data2json)
+json2data=loadubjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  MATLAB special constants\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=[NaN Inf -Inf]
+saveubjson('specials',data2json)
+json2data=loadubjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  a real sparse matrix\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=sprand(10,10,0.1)
+saveubjson('sparse',data2json)
+json2data=loadubjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  a complex sparse matrix\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=data2json-data2json*i
+saveubjson('complex_sparse',data2json)
+json2data=loadubjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  an all-zero sparse matrix\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=sparse(2,3);
+saveubjson('all_zero_sparse',data2json)
+json2data=loadubjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  an empty sparse matrix\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=sparse([]);
+saveubjson('empty_sparse',data2json)
+json2data=loadubjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  an empty 0-by-0 real matrix\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=[];
+saveubjson('empty_0by0_real',data2json)
+json2data=loadubjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  an empty 0-by-3 real matrix\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=zeros(0,3);
+saveubjson('empty_0by3_real',data2json)
+json2data=loadubjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  a sparse real column vector\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=sparse([0,3,0,1,4]');
+saveubjson('sparse_column_vector',data2json)
+json2data=loadubjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  a sparse complex column vector\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=data2json-1i*data2json;
+saveubjson('complex_sparse_column_vector',data2json)
+json2data=loadubjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  a sparse real row vector\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=sparse([0,3,0,1,4]);
+saveubjson('sparse_row_vector',data2json)
+json2data=loadubjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  a sparse complex row vector\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=data2json-1i*data2json;
+saveubjson('complex_sparse_row_vector',data2json)
+json2data=loadubjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  a structure\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=struct('name','Think Different','year',1997,'magic',magic(3),...
+                 'misfits',[Inf,NaN],'embedded',struct('left',true,'right',false))
+saveubjson('astruct',data2json,struct('ParseLogical',1))
+json2data=loadubjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  a structure array\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=struct('name','Nexus Prime','rank',9);
+data2json(2)=struct('name','Sentinel Prime','rank',9);
+data2json(3)=struct('name','Optimus Prime','rank',9);
+saveubjson('Supreme Commander',data2json)
+json2data=loadubjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  a cell array\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=cell(3,1);
+data2json{1}=struct('buzz',1.1,'rex',1.2,'bo',1.3,'hamm',2.0,'slink',2.1,'potato',2.2,...
+              'woody',3.0,'sarge',3.1,'etch',4.0,'lenny',5.0,'squeeze',6.0,'wheezy',7.0);
+data2json{2}=struct('Ubuntu',['Kubuntu';'Xubuntu';'Lubuntu']);
+data2json{3}=[10.04,10.10,11.04,11.10]
+saveubjson('debian',data2json,struct('FloatFormat','%.2f'))
+json2data=loadubjson(ans)
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  invalid field-name handling\n')
+fprintf(1,'%%=================================================\n\n')
+
+json2data=loadubjson(saveubjson('',loadjson('{"ValidName":1, "_InvalidName":2, ":Field:":3, "项目":"绝密"}')))
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  a 2D cell array\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json={{1,{2,3}},{4,5},{6};{7},{8,9},{10}};
+saveubjson('data2json',data2json)
+json2data=loadubjson(ans)  % only savejson works for cell arrays, loadjson has issues
+
+fprintf(1,'\n%%=================================================\n')
+fprintf(1,'%%  a 2D struct array\n')
+fprintf(1,'%%=================================================\n\n')
+
+data2json=repmat(struct('idx',0,'data','structs'),[2,3])
+for i=1:6
+    data2json(i).idx=i;
+end
+saveubjson('data2json',data2json)
+json2data=loadubjson(ans)
+
+rand ('state',rngstate);
+

matlab/+jsonlab/examples/example1.json

@@ -0,0 +1,23 @@
+ {
+     "firstName": "John",
+     "lastName": "Smith",
+     "age": 25,
+     "address":
+     {
+         "streetAddress": "21 2nd Street",
+         "city": "New York",
+         "state": "NY",
+         "postalCode": "10021"
+     },
+     "phoneNumber":
+     [
+         {
+           "type": "home",
+           "number": "212 555-1234"
+         },
+         {
+           "type": "fax",
+           "number": "646 555-4567"
+         }
+     ]
+ }

matlab/+jsonlab/examples/example2.json

@@ -0,0 +1,22 @@
+{
+    "glossary": {
+        "title": "example glossary",
+		"GlossDiv": {
+            "title": "S",
+			"GlossList": {
+                "GlossEntry": {
+                    "ID": "SGML",
+					"SortAs": "SGML",
+					"GlossTerm": "Standard Generalized Markup Language",
+					"Acronym": "SGML",
+					"Abbrev": "ISO 8879:1986",
+					"GlossDef": {
+                        "para": "A meta-markup language, used to create markup languages such as DocBook.",
+						"GlossSeeAlso": ["GML", "XML"]
+                    },
+					"GlossSee": "markup"
+                }
+            }
+        }
+    }
+}

matlab/+jsonlab/examples/example3.json

@@ -0,0 +1,11 @@
+{"menu": {
+  "id": "file",
+  "value": "_&File",
+  "popup": {
+    "menuitem": [
+      {"value": "_&New", "onclick": "CreateNewDoc(\"'\\\"Untitled\\\"'\")"},
+      {"value": "_&Open", "onclick": "OpenDoc()"},
+      {"value": "_&Close", "onclick": "CloseDoc()"}
+    ]
+  }
+}}

matlab/+jsonlab/examples/example4.json

@@ -0,0 +1,35 @@
+[
+  {
+      "sample" : {
+          "rho" : 1
+      }
+  },
+  {
+      "sample" : {
+          "rho" : 2
+      }
+  },
+  [
+    {
+        "_ArrayType_" : "double",
+        "_ArraySize_" : [1,2],
+        "_ArrayData_" : [1,0]
+    },
+    {
+        "_ArrayType_" : "double",
+        "_ArraySize_" : [1,2],
+        "_ArrayData_" : [1,1]
+    },
+    {
+        "_ArrayType_" : "double",
+        "_ArraySize_" : [1,2],
+        "_ArrayData_" : [1,2]
+    }
+  ],
+  [
+     "Paper",
+     "Scissors",
+     "Stone"
+  ],
+  ["a", "b\\", "c\"","d\\\"","e\"[","f\\\"[","g[\\","h[\\\""]
+]

matlab/+jsonlab/examples/jsonlab_basictest.matlab

@@ -0,0 +1,671 @@
+
+                            < M A T L A B (R) >
+                  Copyright 1984-2010 The MathWorks, Inc.
+                Version 7.11.0.584 (R2010b) 64-bit (glnxa64)
+                              August 16, 2010
+
+ 
+  To get started, type one of these: helpwin, helpdesk, or demo.
+  For product information, visit www.mathworks.com.
+ 
+>> >> >> >> >> >> >> >> >> 
+%=================================================
+>> %  a simple scalar value 
+>> %=================================================
+
+>> >> 
+data2json =
+
+    3.1416
+
+>> 
+ans =
+
+[3.141592654]
+
+
+>> 
+json2data =
+
+    3.1416
+
+>> >> 
+%=================================================
+>> %  a complex number
+>> %=================================================
+
+>> >> >> 
+data2json =
+
+   1.0000 + 2.0000i
+
+>> 
+ans =
+
+{
+	"_ArrayType_": "double",
+	"_ArraySize_": [1,1],
+	"_ArrayIsComplex_": 1,
+	"_ArrayData_": [1,2]
+}
+
+
+>> 
+json2data =
+
+   1.0000 + 2.0000i
+
+>> >> 
+%=================================================
+>> %  a complex matrix
+>> %=================================================
+
+>> >> >> 
+data2json =
+
+  35.0000 +26.0000i   1.0000 +19.0000i   6.0000 +24.0000i
+   3.0000 +21.0000i  32.0000 +23.0000i   7.0000 +25.0000i
+  31.0000 +22.0000i   9.0000 +27.0000i   2.0000 +20.0000i
+   8.0000 +17.0000i  28.0000 +10.0000i  33.0000 +15.0000i
+  30.0000 +12.0000i   5.0000 +14.0000i  34.0000 +16.0000i
+   4.0000 +13.0000i  36.0000 +18.0000i  29.0000 +11.0000i
+
+>> 
+ans =
+
+{
+	"_ArrayType_": "double",
+	"_ArraySize_": [6,3],
+	"_ArrayIsComplex_": 1,
+	"_ArrayData_": [
+		[35,26],
+		[3,21],
+		[31,22],
+		[8,17],
+		[30,12],
+		[4,13],
+		[1,19],
+		[32,23],
+		[9,27],
+		[28,10],
+		[5,14],
+		[36,18],
+		[6,24],
+		[7,25],
+		[2,20],
+		[33,15],
+		[34,16],
+		[29,11]
+	]
+}
+
+
+>> 
+json2data =
+
+  35.0000 +26.0000i   1.0000 +19.0000i   6.0000 +24.0000i
+   3.0000 +21.0000i  32.0000 +23.0000i   7.0000 +25.0000i
+  31.0000 +22.0000i   9.0000 +27.0000i   2.0000 +20.0000i
+   8.0000 +17.0000i  28.0000 +10.0000i  33.0000 +15.0000i
+  30.0000 +12.0000i   5.0000 +14.0000i  34.0000 +16.0000i
+   4.0000 +13.0000i  36.0000 +18.0000i  29.0000 +11.0000i
+
+>> >> 
+%=================================================
+>> %  MATLAB special constants
+>> %=================================================
+
+>> >> 
+data2json =
+
+   NaN   Inf  -Inf
+
+>> 
+ans =
+
+{
+	"specials": ["_NaN_","_Inf_","-_Inf_"]
+}
+
+
+>> 
+json2data = 
+
+    specials: [NaN Inf -Inf]
+
+>> >> 
+%=================================================
+>> %  a real sparse matrix
+>> %=================================================
+
+>> >> 
+data2json =
+
+   (1,2)       0.6557
+   (9,2)       0.7577
+   (3,5)       0.8491
+  (10,5)       0.7431
+  (10,8)       0.3922
+   (7,9)       0.6787
+   (2,10)      0.0357
+   (6,10)      0.9340
+  (10,10)      0.6555
+
+>> 
+ans =
+
+{
+	"sparse": {
+		"_ArrayType_": "double",
+		"_ArraySize_": [10,10],
+		"_ArrayIsSparse_": 1,
+		"_ArrayData_": [
+			[1,2,0.6557406992],
+			[9,2,0.7577401306],
+			[3,5,0.8491293059],
+			[10,5,0.7431324681],
+			[10,8,0.3922270195],
+			[7,9,0.6787351549],
+			[2,10,0.03571167857],
+			[6,10,0.9339932478],
+			[10,10,0.6554778902]
+		]
+	}
+}
+
+
+>> 
+json2data = 
+
+    sparse: [10x10 double]
+
+>> >> 
+%=================================================
+>> %  a complex sparse matrix
+>> %=================================================
+
+>> >> 
+data2json =
+
+   (1,2)      0.6557 - 0.6557i
+   (9,2)      0.7577 - 0.7577i
+   (3,5)      0.8491 - 0.8491i
+  (10,5)      0.7431 - 0.7431i
+  (10,8)      0.3922 - 0.3922i
+   (7,9)      0.6787 - 0.6787i
+   (2,10)     0.0357 - 0.0357i
+   (6,10)     0.9340 - 0.9340i
+  (10,10)     0.6555 - 0.6555i
+
+>> 
+ans =
+
+{
+	"complex_sparse": {
+		"_ArrayType_": "double",
+		"_ArraySize_": [10,10],
+		"_ArrayIsComplex_": 1,
+		"_ArrayIsSparse_": 1,
+		"_ArrayData_": [
+			[1,2,0.6557406992,-0.6557406992],
+			[9,2,0.7577401306,-0.7577401306],
+			[3,5,0.8491293059,-0.8491293059],
+			[10,5,0.7431324681,-0.7431324681],
+			[10,8,0.3922270195,-0.3922270195],
+			[7,9,0.6787351549,-0.6787351549],
+			[2,10,0.03571167857,-0.03571167857],
+			[6,10,0.9339932478,-0.9339932478],
+			[10,10,0.6554778902,-0.6554778902]
+		]
+	}
+}
+
+
+>> 
+json2data = 
+
+    complex_sparse: [10x10 double]
+
+>> >> 
+%=================================================
+>> %  an all-zero sparse matrix
+>> %=================================================
+
+>> >> >> 
+ans =
+
+{
+	"all_zero_sparse": {
+		"_ArrayType_": "double",
+		"_ArraySize_": [2,3],
+		"_ArrayIsSparse_": 1,
+		"_ArrayData_": null
+	}
+}
+
+
+>> 
+json2data = 
+
+    all_zero_sparse: [2x3 double]
+
+>> >> 
+%=================================================
+>> %  an empty sparse matrix
+>> %=================================================
+
+>> >> >> 
+ans =
+
+{
+	"empty_sparse": {
+		"_ArrayType_": "double",
+		"_ArraySize_": [0,0],
+		"_ArrayIsSparse_": 1,
+		"_ArrayData_": null
+	}
+}
+
+
+>> 
+json2data = 
+
+    empty_sparse: []
+
+>> >> 
+%=================================================
+>> %  an empty 0-by-0 real matrix
+>> %=================================================
+
+>> >> >> 
+ans =
+
+{
+	"empty_0by0_real": null
+}
+
+
+>> 
+json2data = 
+
+    empty_0by0_real: []
+
+>> >> 
+%=================================================
+>> %  an empty 0-by-3 real matrix
+>> %=================================================
+
+>> >> >> 
+ans =
+
+{
+	"empty_0by3_real": {
+		"_ArrayType_": "double",
+		"_ArraySize_": [0,3],
+		"_ArrayData_": null
+	}
+}
+
+
+>> 
+json2data = 
+
+    empty_0by3_real: [0x3 double]
+
+>> >> 
+%=================================================
+>> %  a sparse real column vector
+>> %=================================================
+
+>> >> >> 
+ans =
+
+{
+	"sparse_column_vector": {
+		"_ArrayType_": "double",
+		"_ArraySize_": [5,1],
+		"_ArrayIsSparse_": 1,
+		"_ArrayData_": [
+			[2,3],
+			[4,1],
+			[5,4]
+		]
+	}
+}
+
+
+>> 
+json2data = 
+
+    sparse_column_vector: [5x1 double]
+
+>> >> 
+%=================================================
+>> %  a sparse complex column vector
+>> %=================================================
+
+>> >> >> 
+ans =
+
+{
+	"complex_sparse_column_vector": {
+		"_ArrayType_": "double",
+		"_ArraySize_": [5,1],
+		"_ArrayIsComplex_": 1,
+		"_ArrayIsSparse_": 1,
+		"_ArrayData_": [
+			[2,3,-3],
+			[4,1,-1],
+			[5,4,-4]
+		]
+	}
+}
+
+
+>> 
+json2data = 
+
+    complex_sparse_column_vector: [5x1 double]
+
+>> >> 
+%=================================================
+>> %  a sparse real row vector
+>> %=================================================
+
+>> >> >> 
+ans =
+
+{
+	"sparse_row_vector": {
+		"_ArrayType_": "double",
+		"_ArraySize_": [1,5],
+		"_ArrayIsSparse_": 1,
+		"_ArrayData_": [
+			[2,3],
+			[4,1],
+			[5,4]
+		]
+	}
+}
+
+
+>> 
+json2data = 
+
+    sparse_row_vector: [0 3 0 1 4]
+
+>> >> 
+%=================================================
+>> %  a sparse complex row vector
+>> %=================================================
+
+>> >> >> 
+ans =
+
+{
+	"complex_sparse_row_vector": {
+		"_ArrayType_": "double",
+		"_ArraySize_": [1,5],
+		"_ArrayIsComplex_": 1,
+		"_ArrayIsSparse_": 1,
+		"_ArrayData_": [
+			[2,3,-3],
+			[4,1,-1],
+			[5,4,-4]
+		]
+	}
+}
+
+
+>> 
+json2data = 
+
+    complex_sparse_row_vector: [1x5 double]
+
+>> >> 
+%=================================================
+>> %  a structure
+>> %=================================================
+
+>> >> 
+data2json = 
+
+        name: 'Think Different'
+        year: 1997
+       magic: [3x3 double]
+     misfits: [Inf NaN]
+    embedded: [1x1 struct]
+
+>> 
+ans =
+
+{
+	"astruct": {
+		"name": "Think Different",
+		"year": 1997,
+		"magic": [
+			[8,1,6],
+			[3,5,7],
+			[4,9,2]
+		],
+		"misfits": ["_Inf_","_NaN_"],
+		"embedded": {
+			"left": true,
+			"right": false
+		}
+	}
+}
+
+
+>> 
+json2data = 
+
+    astruct: [1x1 struct]
+
+>> 
+ans =
+
+logical
+
+>> >> 
+%=================================================
+>> %  a structure array
+>> %=================================================
+
+>> >> >> >> >> 
+ans =
+
+{
+	"Supreme Commander": [
+		{
+			"name": "Nexus Prime",
+			"rank": 9
+		},
+		{
+			"name": "Sentinel Prime",
+			"rank": 9
+		},
+		{
+			"name": "Optimus Prime",
+			"rank": 9
+		}
+	]
+}
+
+
+>> 
+json2data = 
+
+    Supreme_0x20_Commander: {[1x1 struct]  [1x1 struct]  [1x1 struct]}
+
+>> >> 
+%=================================================
+>> %  a cell array
+>> %=================================================
+
+>> >> >> >> >> 
+data2json = 
+
+    [1x1 struct]
+    [1x1 struct]
+    [1x4 double]
+
+>> 
+ans =
+
+{
+	"debian": [
+		[
+			{
+				"buzz": 1.10,
+				"rex": 1.20,
+				"bo": 1.30,
+				"hamm": 2.00,
+				"slink": 2.10,
+				"potato": 2.20,
+				"woody": 3.00,
+				"sarge": 3.10,
+				"etch": 4.00,
+				"lenny": 5.00,
+				"squeeze": 6.00,
+				"wheezy": 7.00
+			}
+		],
+		[
+			{
+				"Ubuntu": [
+					"Kubuntu",
+					"Xubuntu",
+					"Lubuntu"
+				]
+			}
+		],
+		[
+			[10.04,10.10,11.04,11.10]
+		]
+	]
+}
+
+
+>> 
+json2data = 
+
+    debian: {{1x1 cell}  {1x1 cell}  [10.0400 10.1000 11.0400 11.1000]}
+
+>> >> 
+%=================================================
+>> %  invalid field-name handling
+>> %=================================================
+
+>> >> 
+json2data = 
+
+               ValidName: 1
+       x0x5F_InvalidName: 2
+       x0x3A_Field_0x3A_: 3
+    x0xE9A1B9__0xE79BAE_: '绝密'
+
+>> >> 
+%=================================================
+>> %  a 2D cell array
+>> %=================================================
+
+>> >> >> 
+ans =
+
+{
+	"data2json": [
+		[
+			[
+				1,
+				[
+					2,
+					3
+				]
+			],
+			[
+				4,
+				5
+			],
+			[
+				6
+			]
+		],
+		[
+			[
+				7
+			],
+			[
+				8,
+				9
+			],
+			[
+				10
+			]
+		]
+	]
+}
+
+
+>> 
+json2data = 
+
+    data2json: {{3x1 cell}  {3x1 cell}}
+
+>> >> 
+%=================================================
+>> %  a 2D struct array
+>> %=================================================
+
+>> >> 
+data2json = 
+
+2x3 struct array with fields:
+    idx
+    data
+
+>> >> 
+ans =
+
+{
+	"data2json": [
+		[
+			{
+				"idx": 1,
+				"data": "structs"
+			},
+			{
+				"idx": 2,
+				"data": "structs"
+			}
+		],
+		[
+			{
+				"idx": 3,
+				"data": "structs"
+			},
+			{
+				"idx": 4,
+				"data": "structs"
+			}
+		],
+		[
+			{
+				"idx": 5,
+				"data": "structs"
+			},
+			{
+				"idx": 6,
+				"data": "structs"
+			}
+		]
+	]
+}
+
+
+>> 
+json2data = 
+
+    data2json: {{1x2 cell}  {1x2 cell}  {1x2 cell}}
+
+>> >> >> >> 

matlab/+jsonlab/examples/jsonlab_selftest.m

@@ -0,0 +1,27 @@
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%         Regression Test Unit of loadjson and savejson
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+for i=1:4
+    fname=sprintf('example%d.json',i);
+    if(exist(fname,'file')==0) break; end
+    fprintf(1,'===============================================\n>> %s\n',fname);
+    json=savejson('data',loadjson(fname));
+    fprintf(1,'%s\n',json);
+    fprintf(1,'%s\n',savejson('data',loadjson(fname),'Compact',1));
+    data=loadjson(json);
+    savejson('data',data,'selftest.json');
+    data=loadjson('selftest.json');
+end
+
+for i=1:4
+    fname=sprintf('example%d.json',i);
+    if(exist(fname,'file')==0) break; end
+    fprintf(1,'===============================================\n>> %s\n',fname);
+    json=saveubjson('data',loadjson(fname));
+    fprintf(1,'%s\n',json);
+    data=loadubjson(json);
+    savejson('',data);
+    saveubjson('data',data,'selftest.ubj');
+    data=loadubjson('selftest.ubj');
+end

matlab/+jsonlab/examples/jsonlab_selftest.matlab

@@ -0,0 +1,154 @@
+
+                            < M A T L A B (R) >
+                  Copyright 1984-2010 The MathWorks, Inc.
+                Version 7.11.0.584 (R2010b) 64-bit (glnxa64)
+                              August 16, 2010
+
+ 
+  To get started, type one of these: helpwin, helpdesk, or demo.
+  For product information, visit www.mathworks.com.
+ 
+>> >> >> >> >> ===============================================
+>> example1.json
+{
+	"data": {
+		"firstName": "John",
+		"lastName": "Smith",
+		"age": 25,
+		"address": {
+			"streetAddress": "21 2nd Street",
+			"city": "New York",
+			"state": "NY",
+			"postalCode": "10021"
+		},
+		"phoneNumber": [
+			{
+				"type": "home",
+				"number": "212 555-1234"
+			},
+			{
+				"type": "fax",
+				"number": "646 555-4567"
+			}
+		]
+	}
+}
+
+{"data": {"firstName": "John","lastName": "Smith","age": 25,"address": {"streetAddress": "21 2nd Street","city": "New York","state": "NY","postalCode": "10021"},"phoneNumber": [{"type": "home","number": "212 555-1234"},{"type": "fax","number": "646 555-4567"}]}}
+
+===============================================
+>> example2.json
+{
+	"data": {
+		"glossary": {
+			"title": "example glossary",
+			"GlossDiv": {
+				"title": "S",
+				"GlossList": {
+					"GlossEntry": {
+						"ID": "SGML",
+						"SortAs": "SGML",
+						"GlossTerm": "Standard Generalized Markup Language",
+						"Acronym": "SGML",
+						"Abbrev": "ISO 8879:1986",
+						"GlossDef": {
+							"para": "A meta-markup language, used to create markup languages such as DocBook.",
+							"GlossSeeAlso": [
+								"GML",
+								"XML"
+							]
+						},
+						"GlossSee": "markup"
+					}
+				}
+			}
+		}
+	}
+}
+
+{"data": {"glossary": {"title": "example glossary","GlossDiv": {"title": "S","GlossList": {"GlossEntry": {"ID": "SGML","SortAs": "SGML","GlossTerm": "Standard Generalized Markup Language","Acronym": "SGML","Abbrev": "ISO 8879:1986","GlossDef": {"para": "A meta-markup language, used to create markup languages such as DocBook.","GlossSeeAlso": ["GML","XML"]},"GlossSee": "markup"}}}}}}
+
+===============================================
+>> example3.json
+{
+	"data": {
+		"menu": {
+			"id": "file",
+			"value": "_&File",
+			"popup": {
+				"menuitem": [
+					{
+						"value": "_&New",
+						"onclick": "CreateNewDoc(\"'\\\"Untitled\\\"'\")"
+					},
+					{
+						"value": "_&Open",
+						"onclick": "OpenDoc()"
+					},
+					{
+						"value": "_&Close",
+						"onclick": "CloseDoc()"
+					}
+				]
+			}
+		}
+	}
+}
+
+{"data": {"menu": {"id": "file","value": "_&File","popup": {"menuitem": [{"value": "_&New","onclick": "CreateNewDoc(\"'\\\"Untitled\\\"'\")"},{"value": "_&Open","onclick": "OpenDoc()"},{"value": "_&Close","onclick": "CloseDoc()"}]}}}}
+
+===============================================
+>> example4.json
+{
+	"data": [
+		{
+			"sample": {
+				"rho": 1
+			}
+		},
+		{
+			"sample": {
+				"rho": 2
+			}
+		},
+		[
+			[1,0],
+			[1,1],
+			[1,2]
+		],
+		[
+			"Paper",
+			"Scissors",
+			"Stone"
+		],
+		[
+			"a",
+			"b\\",
+			"c\"",
+			"d\\\"",
+			"e\"[",
+			"f\\\"[",
+			"g[\\",
+			"h[\\\""
+		]
+	]
+}
+
+{"data": [{"sample": {"rho": 1}},{"sample": {"rho": 2}},[[1,0],[1,1],[1,2]],["Paper","Scissors","Stone"],["a","b\\","c\"","d\\\"","e\"[","f\\\"[","g[\\","h[\\\""]]}
+
+>> >> ===============================================
+>> example1.json
+{Udata{U	firstNameSUJohnUlastNameSUSmithUageiUaddress{U
streetAddressSU
21 2nd StreetUcitySUNew YorkUstateSUNYU
+postalCodeSU10021}UphoneNumber[{UtypeSUhomeUnumberSU212 555-1234}{UtypeSUfaxUnumberSU646 555-4567}]}}
+===============================================
+>> example2.json
+{Udata{Uglossary{UtitleSUexample glossaryUGlossDiv{UtitleCSU	GlossList{U
+GlossEntry{UIDSUSGMLUSortAsSUSGMLU	GlossTermSU$Standard Generalized Markup LanguageUAcronymSUSGMLUAbbrevSU
ISO 8879:1986UGlossDef{UparaSUHA meta-markup language, used to create markup languages such as DocBook.UGlossSeeAlso[SUGMLSUXML]}UGlossSeeSUmarkup}}}}}}
+===============================================
+>> example3.json
+{Udata{Umenu{UidSUfileUvalueSU_&FileUpopup{Umenuitem[{UvalueSU_&NewUonclickSUCreateNewDoc("'\"Untitled\"'")}{UvalueSU_&OpenUonclickSU	OpenDoc()}{UvalueSU_&CloseUonclickSU
+CloseDoc()}]}}}}
+===============================================
+>> example4.json
+{Udata[{Usample{Urhoi
}}{Usample{Urhoi}}[[$i#U
[$i#U

[$i#U
][SUPaperSUScissorsSUStone][CaSUb\SUc"SUd\"SUe"[SUf\"[SUg[\SUh[\"]]}
+>> 

matlab/+jsonlab/examples/jsonlab_speedtest.m

@@ -0,0 +1,21 @@
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%         Benchmarking processing speed of savejson and loadjson
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+datalen=[1e3 1e4 1e5 1e6];
+len=length(datalen);
+tsave=zeros(len,1);
+tload=zeros(len,1);
+for i=1:len
+    tic;
+    json=savejson('data',struct('d1',rand(datalen(i),3),'d2',rand(datalen(i),3)>0.5));
+    tsave(i)=toc;
+    data=loadjson(json);
+    tload(i)=toc-tsave(i);
+    fprintf(1,'matrix size: %d\n',datalen(i));
+end
+
+loglog(datalen,tsave,'o-',datalen,tload,'r*-');
+legend('savejson runtime (s)','loadjson runtime (s)');
+xlabel('array size');
+ylabel('running time (s)');

matlab/+jsonlab/examples/jsonlab_ubjson_basictest.matlab

@@ -0,0 +1,394 @@
+
+                            < M A T L A B (R) >
+                  Copyright 1984-2010 The MathWorks, Inc.
+                Version 7.11.0.584 (R2010b) 64-bit (glnxa64)
+                              August 16, 2010
+
+ 
+  To get started, type one of these: helpwin, helpdesk, or demo.
+  For product information, visit www.mathworks.com.
+ 
+>> >> >> >> >> >> >> >> >> 
+%=================================================
+>> %  a simple scalar value 
+>> %=================================================
+
+>> >> 
+data2json =
+
+    3.1416
+
+>> 
+ans =
+
+[D@	!ûTD-]
+
+>> 
+json2data = 
+
+    [3.1416]
+
+>> >> 
+%=================================================
+>> %  a complex number
+>> %=================================================
+
+>> >> >> 
+data2json =
+
+   1.0000 + 2.0000i
+
+>> 
+ans =
+
+{U_ArrayType_SUdoubleU_ArraySize_[$U#U

U_ArrayIsComplex_TU_ArrayData_[$i#U
}
+
+>> 
+json2data =
+
+   1.0000 + 2.0000i
+
+>> >> 
+%=================================================
+>> %  a complex matrix
+>> %=================================================
+
+>> >> >> 
+data2json =
+
+  35.0000 +26.0000i   1.0000 +19.0000i   6.0000 +24.0000i
+   3.0000 +21.0000i  32.0000 +23.0000i   7.0000 +25.0000i
+  31.0000 +22.0000i   9.0000 +27.0000i   2.0000 +20.0000i
+   8.0000 +17.0000i  28.0000 +10.0000i  33.0000 +15.0000i
+  30.0000 +12.0000i   5.0000 +14.0000i  34.0000 +16.0000i
+   4.0000 +13.0000i  36.0000 +18.0000i  29.0000 +11.0000i
+
+>> 
+ans =
+
+{U_ArrayType_SUdoubleU_ArraySize_[$U#UU_ArrayIsComplex_TU_ArrayData_[$i#[$U#U#
 	$!"

+}
+
+>> 
+json2data =
+
+  35.0000 +26.0000i   1.0000 +19.0000i   6.0000 +24.0000i
+   3.0000 +21.0000i  32.0000 +23.0000i   7.0000 +25.0000i
+  31.0000 +22.0000i   9.0000 +27.0000i   2.0000 +20.0000i
+   8.0000 +17.0000i  28.0000 +10.0000i  33.0000 +15.0000i
+  30.0000 +12.0000i   5.0000 +14.0000i  34.0000 +16.0000i
+   4.0000 +13.0000i  36.0000 +18.0000i  29.0000 +11.0000i
+
+>> >> 
+%=================================================
+>> %  MATLAB special constants
+>> %=================================================
+
+>> >> 
+data2json =
+
+   NaN   Inf  -Inf
+
+>> 
+ans =
+
+{Uspecials[$D#Uÿø      ð      ÿð      }
+
+>> 
+json2data = 
+
+    specials: [NaN Inf -Inf]
+
+>> >> 
+%=================================================
+>> %  a real sparse matrix
+>> %=================================================
+
+>> >> 
+data2json =
+
+   (1,2)       0.6557
+   (9,2)       0.7577
+   (3,5)       0.8491
+  (10,5)       0.7431
+  (10,8)       0.3922
+   (7,9)       0.6787
+   (2,10)      0.0357
+   (6,10)      0.9340
+  (10,10)      0.6555
+
+>> 
+ans =
+
+{Usparse{U_ArrayType_SUdoubleU_ArraySize_[$U#U
+
+U_ArrayIsSparse_TU_ArrayData_[$D#[$U#U	?ð      @"      @      @$      @$      @      @       @      @$      @       @       @      @      @       @"      @$      @$      @$      ?äûÓë12?è?h:öl;?ë,8Ù±?çǽ½æ'#?Ù?[`o€?å¸2ɘNé?¢HÍpà?íãEι¶P?äù¬Ä²	¶}}
+
+>> 
+json2data = 
+
+    sparse: [10x10 double]
+
+>> >> 
+%=================================================
+>> %  a complex sparse matrix
+>> %=================================================
+
+>> >> 
+data2json =
+
+   (1,2)      0.6557 - 0.6557i
+   (9,2)      0.7577 - 0.7577i
+   (3,5)      0.8491 - 0.8491i
+  (10,5)      0.7431 - 0.7431i
+  (10,8)      0.3922 - 0.3922i
+   (7,9)      0.6787 - 0.6787i
+   (2,10)     0.0357 - 0.0357i
+   (6,10)     0.9340 - 0.9340i
+  (10,10)     0.6555 - 0.6555i
+
+>> 
+ans =
+
+{Ucomplex_sparse{U_ArrayType_SUdoubleU_ArraySize_[$U#U
+
+U_ArrayIsComplex_TU_ArrayIsSparse_TU_ArrayData_[$D#[$U#U	?ð      @"      @      @$      @$      @      @       @      @$      @       @       @      @      @       @"      @$      @$      @$      ?äûÓë12?è?h:öl;?ë,8Ù±?çǽ½æ'#?Ù?[`o€?å¸2ɘNé?¢HÍpà?íãEι¶P?äù¬Ä²	¶¿äûÓë12¿è?h:öl;¿ë,8Ù±¿çǽ½æ'#¿Ù?[`o€¿å¸2ɘNé¿¢HÍpà¿íãEι¶P¿äù¬Ä²	¶}}
+
+>> 
+json2data = 
+
+    complex_sparse: [10x10 double]
+
+>> >> 
+%=================================================
+>> %  an all-zero sparse matrix
+>> %=================================================
+
+>> >> >> 
+ans =
+
+{Uall_zero_sparse{U_ArrayType_SUdoubleU_ArraySize_[$U#UU_ArrayIsSparse_TU_ArrayData_Z}}
+
+>> 
+json2data = 
+
+    all_zero_sparse: [2x3 double]
+
+>> >> 
+%=================================================
+>> %  an empty sparse matrix
+>> %=================================================
+
+>> >> >> 
+ans =
+
+{Uempty_sparseZ}
+
+>> 
+json2data = 
+
+    empty_sparse: []
+
+>> >> 
+%=================================================
+>> %  an empty 0-by-0 real matrix
+>> %=================================================
+
+>> >> >> 
+ans =
+
+{Uempty_0by0_realZ}
+
+>> 
+json2data = 
+
+    empty_0by0_real: []
+
+>> >> 
+%=================================================
+>> %  an empty 0-by-3 real matrix
+>> %=================================================
+
+>> >> >> 
+ans =
+
+{Uempty_0by3_realZ}
+
+>> 
+json2data = 
+
+    empty_0by3_real: []
+
+>> >> 
+%=================================================
+>> %  a sparse real column vector
+>> %=================================================
+
+>> >> >> 
+ans =
+
+{Usparse_column_vector{U_ArrayType_SUdoubleU_ArraySize_[$U#U
U_ArrayIsSparse_TU_ArrayData_[$i#[$U#U
}}
+
+>> 
+json2data = 
+
+    sparse_column_vector: [5x1 double]
+
+>> >> 
+%=================================================
+>> %  a sparse complex column vector
+>> %=================================================
+
+>> >> >> 
+ans =
+
+{Ucomplex_sparse_column_vector{U_ArrayType_SUdoubleU_ArraySize_[$U#U
U_ArrayIsComplex_TU_ArrayIsSparse_TU_ArrayData_[$i#[$U#U
ýÿü}}
+
+>> 
+json2data = 
+
+    complex_sparse_column_vector: [5x1 double]
+
+>> >> 
+%=================================================
+>> %  a sparse real row vector
+>> %=================================================
+
+>> >> >> 
+ans =
+
+{Usparse_row_vector{U_ArrayType_SUdoubleU_ArraySize_[$U#U
U_ArrayIsSparse_TU_ArrayData_[$i#[$U#U
}}
+
+>> 
+json2data = 
+
+    sparse_row_vector: [0 3 0 1 4]
+
+>> >> 
+%=================================================
+>> %  a sparse complex row vector
+>> %=================================================
+
+>> >> >> 
+ans =
+
+{Ucomplex_sparse_row_vector{U_ArrayType_SUdoubleU_ArraySize_[$U#U
U_ArrayIsComplex_TU_ArrayIsSparse_TU_ArrayData_[$i#[$U#U
ýÿü}}
+
+>> 
+json2data = 
+
+    complex_sparse_row_vector: [1x5 double]
+
+>> >> 
+%=================================================
+>> %  a structure
+>> %=================================================
+
+>> >> 
+data2json = 
+
+        name: 'Think Different'
+        year: 1997
+       magic: [3x3 double]
+     misfits: [Inf NaN]
+    embedded: [1x1 struct]
+
+>> 
+ans =
+
+{Uastruct{UnameSUThink DifferentUyearIÍUmagic[$i#[$U#U
	Umisfits[$D#Uð      ÿø      Uembedded{UleftTUrightF}}}
+
+>> 
+json2data = 
+
+    astruct: [1x1 struct]
+
+>> >> 
+%=================================================
+>> %  a structure array
+>> %=================================================
+
+>> >> >> >> >> 
+ans =
+
+{USupreme Commander[{UnameSUNexus PrimeUranki	}{UnameSUSentinel PrimeUranki	}{UnameSU
Optimus PrimeUranki	}]}
+
+>> 
+json2data = 
+
+    Supreme_0x20_Commander: {[1x1 struct]  [1x1 struct]  [1x1 struct]}
+
+>> >> 
+%=================================================
+>> %  a cell array
+>> %=================================================
+
+>> >> >> >> >> 
+data2json = 
+
+    [1x1 struct]
+    [1x1 struct]
+    [1x4 double]
+
+>> 
+ans =
+
+{Udebian[[{UbuzzD?ñ™™™™™šUrexD?ó333333UboD?ôÌÌÌÌÌÍUhammiUslinkD@ ÌÌÌÌÌÍUpotatoD@
™™™™™šUwoodyiUsargeD@ÌÌÌÌÌÍUetchiUlennyiUsqueezeiUwheezyi}{UUbuntu[SUKubuntuSUXubuntuSULubuntu]}[$D#U@$záG®@$333333@&záG®@&333333]]}
+
+>> 
+json2data = 
+
+    debian: {{1x3 cell}}
+
+>> >> 
+%=================================================
+>> %  invalid field-name handling
+>> %=================================================
+
+>> >> 
+json2data = 
+
+               ValidName: 1
+       x0x5F_InvalidName: 2
+       x0x3A_Field_0x3A_: 3
+    x0xEFBFBD__0xEFBFBD_: '绝密'
+
+>> >> 
+%=================================================
+>> %  a 2D cell array
+>> %=================================================
+
+>> >> >> 
+ans =
+
+{U	data2json[[[[i
][[i][i]]][[i]]][[[i][i]][[i][i	]]][[[i]][[i
+]]]]}
+
+>> 
+json2data = 
+
+    data2json: {{1x2 cell}  {1x2 cell}  {1x2 cell}}
+
+>> >> 
+%=================================================
+>> %  a 2D struct array
+>> %=================================================
+
+>> >> 
+data2json = 
+
+2x3 struct array with fields:
+    idx
+    data
+
+>> >> 
+ans =
+
+{U	data2json[[{Uidxi
UdataSUstructs}{UidxiUdataSUstructs}][{UidxiUdataSUstructs}{UidxiUdataSUstructs}][{UidxiUdataSUstructs}{UidxiUdataSUstructs}]]}
+
+>> 
+json2data = 
+
+    data2json: {{1x2 cell}  {1x2 cell}  {1x2 cell}}
+
+>> >> >> >> 

matlab/+jsonlab/gzipdecode.m

@@ -0,0 +1,41 @@
+function output = gzipdecode(input)
+import jsonlab.*
+%GZIPDECODE Decompress input bytes using GZIP.
+%
+%    output = gzipdecode(input)
+%
+% The function takes a compressed byte array INPUT and returns inflated
+% bytes OUTPUT. The INPUT is a result of GZIPENCODE function. The OUTPUT
+% is always an 1-by-N uint8 array. JAVA must be enabled to use the function.
+%
+% See also gzipencode typecast
+%
+% Copyright (c) 2012, Kota Yamaguchi
+% URL: https://www.mathworks.com/matlabcentral/fileexchange/39526-byte-encoding-utilities
+% License : BSD, see LICENSE_*.txt
+%
+
+if(nargin==0)
+    error('you must provide at least 1 input');
+end
+if(exist('zmat')==3)
+    output=zmat(uint8(input),0,'gzip');
+    return;
+end
+error(javachk('jvm'));
+if ischar(input)
+  warning('gzipdecode:inputTypeMismatch', ...
+          'Input is char, but treated as uint8.');
+  input = uint8(input);
+end
+if ~isa(input, 'int8') && ~isa(input, 'uint8')
+    error('Input must be either int8 or uint8.');
+end
+
+gzip = java.util.zip.GZIPInputStream(java.io.ByteArrayInputStream(input));
+buffer = java.io.ByteArrayOutputStream();
+org.apache.commons.io.IOUtils.copy(gzip, buffer);
+gzip.close();
+output = typecast(buffer.toByteArray(), 'uint8')';
+
+end

matlab/+jsonlab/gzipencode.m

@@ -0,0 +1,39 @@
+function output = gzipencode(input)
+import jsonlab.*
+%GZIPENCODE Compress input bytes with GZIP.
+%
+%    output = gzipencode(input)
+%
+% The function takes a char, int8, or uint8 array INPUT and returns
+% compressed bytes OUTPUT as a uint8 array. Note that the compression
+% doesn't preserve input dimensions. JAVA must be enabled to use the
+% function.
+%
+% See also gzipdecode typecast
+%
+% Copyright (c) 2012, Kota Yamaguchi
+% URL: https://www.mathworks.com/matlabcentral/fileexchange/39526-byte-encoding-utilities
+% License : BSD, see LICENSE_*.txt
+%
+
+if(nargin==0)
+    error('you must provide at least 1 input');
+end
+if(exist('zmat')==3)
+    output=zmat(uint8(input),1,'gzip');
+    return;
+end
+error(javachk('jvm'));
+if ischar(input), input = uint8(input); end
+if ~isa(input, 'int8') && ~isa(input, 'uint8')
+    error('Input must be either char, int8 or uint8.');
+end
+
+buffer = java.io.ByteArrayOutputStream();
+gzip = java.util.zip.GZIPOutputStream(buffer);
+gzip.write(input, 0, numel(input));
+gzip.close();
+output = typecast(buffer.toByteArray(), 'uint8')';
+
+end
+

matlab/+jsonlab/jsonopt.m

@@ -0,0 +1,36 @@
+function val=jsonopt(key,default,varargin)
+import jsonlab.*
+%
+% val=jsonopt(key,default,optstruct)
+%
+% setting options based on a struct. The struct can be produced
+% by varargin2struct from a list of 'param','value' pairs
+%
+% authors:Qianqian Fang (q.fang <at> neu.edu)
+%
+% $Id: loadjson.m 371 2012-06-20 12:43:06Z fangq $
+%
+% input:
+%      key: a string with which one look up a value from a struct
+%      default: if the key does not exist, return default
+%      optstruct: a struct where each sub-field is a key 
+%
+% output:
+%      val: if key exists, val=optstruct.key; otherwise val=default
+%
+% license:
+%     BSD, see LICENSE_BSD.txt file for details
+%
+% -- this function is part of jsonlab toolbox (http://iso2mesh.sf.net/cgi-bin/index.cgi?jsonlab)
+% 
+
+val=default;
+if(nargin<=2) return; end
+opt=varargin{1};
+if(isstruct(opt))
+    if(isfield(opt,key))
+       val=getfield(opt,key);
+    elseif(isfield(opt,lower(key)))
+       val=getfield(opt,lower(key));
+    end
+end

matlab/+jsonlab/license.txt

@@ -0,0 +1,35 @@
+Copyright (c) 2019, Qianqian Fang
+Copyright (c) 2009, Nedialko
+Copyright (c) 2016, The MathWorks, Inc.
+Copyright (c) 2011, François Glineur
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+* Neither the name of the  nor the names of its
+  contributors may be used to endorse or promote products derived from this
+  software without specific prior written permission.
+* Neither the name of the The MathWorks, Inc. nor the names of its
+  contributors may be used to endorse or promote products derived from this
+  software without specific prior written permission.
+* In all cases, the software is, and all modifications and derivatives of the
+  software shall be, licensed to you solely for use in conjunction with
+  MathWorks products and service offerings.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

matlab/+jsonlab/loadjson.m

@@ -0,0 +1,537 @@
+function data = loadjson(fname,varargin)
+%
+% data=loadjson(fname,opt)
+%    or
+% data=loadjson(fname,'param1',value1,'param2',value2,...)
+%
+% parse a JSON (JavaScript Object Notation) file or string
+%
+% authors:Qianqian Fang (q.fang <at> neu.edu)
+% created on 2011/09/09, including previous works from 
+%
+%         Nedialko Krouchev: http://www.mathworks.com/matlabcentral/fileexchange/25713
+%            created on 2009/11/02
+%         François Glineur: http://www.mathworks.com/matlabcentral/fileexchange/23393
+%            created on  2009/03/22
+%         Joel Feenstra:
+%         http://www.mathworks.com/matlabcentral/fileexchange/20565
+%            created on 2008/07/03
+%
+% $Id$
+%
+% input:
+%      fname: input file name, if fname contains "{}" or "[]", fname
+%             will be interpreted as a JSON string
+%      opt: a struct to store parsing options, opt can be replaced by 
+%           a list of ('param',value) pairs - the param string is equivallent
+%           to a field in opt. opt can have the following 
+%           fields (first in [.|.] is the default)
+%
+%           opt.SimplifyCell [0|1]: if set to 1, loadjson will call cell2mat
+%                         for each element of the JSON data, and group 
+%                         arrays based on the cell2mat rules.
+%           opt.FastArrayParser [1|0 or integer]: if set to 1, use a
+%                         speed-optimized array parser when loading an 
+%                         array object. The fast array parser may 
+%                         collapse block arrays into a single large
+%                         array similar to rules defined in cell2mat; 0 to 
+%                         use a legacy parser; if set to a larger-than-1
+%                         value, this option will specify the minimum
+%                         dimension to enable the fast array parser. For
+%                         example, if the input is a 3D array, setting
+%                         FastArrayParser to 1 will return a 3D array;
+%                         setting to 2 will return a cell array of 2D
+%                         arrays; setting to 3 will return to a 2D cell
+%                         array of 1D vectors; setting to 4 will return a
+%                         3D cell array.
+%           opt.ShowProgress [0|1]: if set to 1, loadjson displays a progress bar.
+%           opt.ParseStringArray [0|1]: if set to 1, loadjson displays a progress bar.
+%
+% output:
+%      dat: a cell array, where {...} blocks are converted into cell arrays,
+%           and [...] are converted to arrays
+%
+% examples:
+%      dat=loadjson('{"obj":{"string":"value","array":[1,2,3]}}')
+%      dat=loadjson(['examples' filesep 'example1.json'])
+%      dat=loadjson(['examples' filesep 'example1.json'],'SimplifyCell',1)
+%
+% license:
+%     BSD, see LICENSE_BSD.txt file for details 
+%
+% -- this function is part of JSONLab toolbox (http://iso2mesh.sf.net/cgi-bin/index.cgi?jsonlab)
+%
+import jsonlab.*
+
+global pos index_esc isoct arraytoken
+
+if(regexp(fname,'^\s*(?:\[.*\])|(?:\{.*\})\s*$','once'))
+   string=fname;
+elseif(exist(fname,'file'))
+   try
+       string = fileread(fname);
+   catch
+       try
+           string = urlread(['file://',fname]);
+       catch
+           string = urlread(['file://',fullfile(pwd,fname)]);
+       end
+   end
+else
+   error('input file does not exist');
+end
+
+pos = 1; len = length(string); inStr = string;
+isoct=exist('OCTAVE_VERSION','builtin');
+arraytoken=find(inStr=='[' | inStr==']' | inStr=='"');
+jstr=regexprep(inStr,'\\\\','  ');
+escquote=regexp(jstr,'\\"');
+arraytoken=sort([arraytoken escquote]);
+
+% String delimiters and escape chars identified to improve speed:
+esc = find(inStr=='"' | inStr=='\' ); % comparable to: regexp(inStr, '["\\]');
+index_esc = 1;
+
+opt=varargin2struct(varargin{:});
+
+if(jsonopt('ShowProgress',0,opt)==1)
+    opt.progressbar_=waitbar(0,'loading ...');
+end
+jsoncount=1;
+while pos <= len
+    switch(next_char(inStr))
+        case '{'
+            data{jsoncount} = parse_object(inStr, esc, opt);
+        case '['
+            data{jsoncount} = parse_array(inStr, esc, opt);
+        otherwise
+            error_pos('Outer level structure must be an object or an array',inStr);
+    end
+    jsoncount=jsoncount+1;
+end % while
+
+jsoncount=length(data);
+if(jsoncount==1 && iscell(data))
+    data=data{1};
+end
+
+if(isfield(opt,'progressbar_'))
+    close(opt.progressbar_);
+end
+
+%%-------------------------------------------------------------------------
+function object = parse_object(inStr, esc, varargin)
+import jsonlab.*
+    parse_char(inStr, '{');
+    object = [];
+    if next_char(inStr) ~= '}'
+        while 1
+            str = parseStr(inStr, esc, varargin{:});
+            if isempty(str)
+                error_pos('Name of value at position %d cannot be empty',inStr);
+            end
+            parse_char(inStr, ':');
+            val = parse_value(inStr, esc, varargin{:});
+            object.(valid_field(str))=val;
+            if next_char(inStr) == '}'
+                break;
+            end
+            parse_char(inStr, ',');
+        end
+    end
+    parse_char(inStr, '}');
+    if(isstruct(object))
+        object=struct2jdata(object);
+    end
+
+%%-------------------------------------------------------------------------
+
+function object = parse_array(inStr, esc, varargin) % JSON array is written in row-major order
+import jsonlab.*
+    global pos isoct
+    parse_char(inStr, '[');
+    object = cell(0, 1);
+    dim2=[];
+    arraydepth=jsonopt('JSONLAB_ArrayDepth_',1,varargin{:});
+    pbar=-1;
+    if(isfield(varargin{1},'progressbar_'))
+        pbar=varargin{1}.progressbar_;
+    end
+
+    if next_char(inStr) ~= ']'
+	if(jsonopt('FastArrayParser',1,varargin{:})>=1 && arraydepth>=jsonopt('FastArrayParser',1,varargin{:}))
+            [endpos, e1l, e1r]=matching_bracket(inStr,pos);
+            arraystr=['[' inStr(pos:endpos)];
+            arraystr=regexprep(arraystr,'"_NaN_"','NaN');
+            arraystr=regexprep(arraystr,'"([-+]*)_Inf_"','$1Inf');
+            arraystr(arraystr==sprintf('\n'))=[];
+            arraystr(arraystr==sprintf('\r'))=[];
+            %arraystr=regexprep(arraystr,'\s*,',','); % this is slow,sometimes needed
+            if(~isempty(e1l) && ~isempty(e1r)) % the array is in 2D or higher D
+        	astr=inStr((e1l+1):(e1r-1));
+        	astr=regexprep(astr,'"_NaN_"','NaN');
+        	astr=regexprep(astr,'"([-+]*)_Inf_"','$1Inf');
+        	astr(astr==sprintf('\n'))=[];
+        	astr(astr==sprintf('\r'))=[];
+        	astr(astr==' ')='';
+        	if(isempty(find(astr=='[', 1))) % array is 2D
+                    dim2=length(sscanf(astr,'%f,',[1 inf]));
+        	end
+            else % array is 1D
+        	astr=arraystr(2:end-1);
+        	astr(astr==' ')='';
+        	[obj, count, errmsg, nextidx]=sscanf(astr,'%f,',[1,inf]);
+        	if(nextidx>=length(astr)-1)
+                    object=obj;
+                    pos=endpos;
+                    parse_char(inStr, ']');
+                    return;
+        	end
+            end
+
+            try
+              if(~isempty(dim2))
+        	astr=arraystr;
+        	astr(astr=='[')='';
+        	astr(astr==']')='';
+                astr=regexprep(astr,'\s*$','');
+        	astr(astr==' ')='';
+        	[obj, count, errmsg, nextidx]=sscanf(astr,'%f,',inf);
+        	if(nextidx>=length(astr)-1)
+                    object=reshape(obj,dim2,numel(obj)/dim2)';
+                    pos=endpos;
+                    parse_char(inStr, ']');
+                    if(pbar>0)
+                        waitbar(pos/length(inStr),pbar,'loading ...');
+                    end
+                    return;
+        	end
+              end
+              arraystr=regexprep(arraystr,'\]\s*,','];');
+            catch
+            end
+	else
+            arraystr='[';
+	end
+        try
+           arraystr=regexprep(arraystr,'^\s*\[','{','once');
+           arraystr=regexprep(arraystr,'\]\s*$','}','once');
+           if(isoct && regexp(arraystr,'"','once'))
+                error('Octave eval can produce empty cells for JSON-like input');
+           end
+           if(regexp(arraystr,':','once'))
+                error('One can not use MATLAB-like ":" construct inside a JSON array');
+           end
+           if(jsonopt('ParseStringArray',0,varargin{:})==0)
+               arraystr=regexprep(arraystr,'\"','''');
+           end
+           object=eval(arraystr);
+           if(iscell(object))
+               object=cellfun(@unescapejsonstring,object,'UniformOutput',false);
+           end
+           pos=endpos;
+        catch
+         while 1
+            newopt=varargin2struct(varargin{:},'JSONLAB_ArrayDepth_',arraydepth+1);
+            val = parse_value(inStr, esc, newopt);
+            object{end+1} = val;
+            if next_char(inStr) == ']'
+                break;
+            end
+            parse_char(inStr, ',');
+         end
+        end
+    end
+    if(jsonopt('SimplifyCell',0,varargin{:})==1)
+      try
+        oldobj=object;
+        object=cell2mat(object')';
+        if(iscell(oldobj) && isstruct(object) && numel(object)>1 && jsonopt('SimplifyCellArray',1,varargin{:})==0)
+            object=oldobj;
+        elseif(size(object,1)>1 && ismatrix(object))
+            object=object';
+        end
+      catch
+      end
+    end
+    parse_char(inStr, ']');
+    
+    if(pbar>0)
+        waitbar(pos/length(inStr),pbar,'loading ...');
+    end
+%%-------------------------------------------------------------------------
+
+function parse_char(inStr, c)
+import jsonlab.*
+    global pos
+    pos=skip_whitespace(pos, inStr);
+    if pos > length(inStr) || inStr(pos) ~= c
+        error_pos(sprintf('Expected %c at position %%d', c),inStr);
+    else
+        pos = pos + 1;
+        pos=skip_whitespace(pos, inStr);
+    end
+
+%%-------------------------------------------------------------------------
+
+function c = next_char(inStr)
+import jsonlab.*
+    global pos
+    pos=skip_whitespace(pos, inStr);
+    if pos > length(inStr)
+        c = [];
+    else
+        c = inStr(pos);
+    end
+
+%%-------------------------------------------------------------------------
+
+function newpos=skip_whitespace(pos, inStr)
+import jsonlab.*
+    newpos=pos;
+    while newpos <= length(inStr) && isspace(inStr(newpos))
+        newpos = newpos + 1;
+    end
+
+%%-------------------------------------------------------------------------
+function str = parseStr(inStr, esc, varargin)
+import jsonlab.*
+    global pos index_esc
+ % len, ns = length(inStr), keyboard
+    if inStr(pos) ~= '"'
+        error_pos('String starting with " expected at position %d',inStr);
+    else
+        pos = pos + 1;
+    end
+    str = '';
+    while pos <= length(inStr)
+        while index_esc <= length(esc) && esc(index_esc) < pos
+            index_esc = index_esc + 1;
+        end
+        if index_esc > length(esc)
+            str = [str inStr(pos:end)];
+            pos = length(inStr) + 1;
+            break;
+        else
+            str = [str inStr(pos:esc(index_esc)-1)];
+            pos = esc(index_esc);
+        end
+        nstr = length(str);
+        switch inStr(pos)
+            case '"'
+                pos = pos + 1;
+                if(~isempty(str))
+                    if(strcmp(str,'_Inf_'))
+                        str=Inf;
+                    elseif(strcmp(str,'-_Inf_'))
+                        str=-Inf;
+                    elseif(strcmp(str,'_NaN_'))
+                        str=NaN;
+                    end
+                end
+                return;
+            case '\'
+                if pos+1 > length(inStr)
+                    error_pos('End of file reached right after escape character',inStr);
+                end
+                pos = pos + 1;
+                switch inStr(pos)
+                    case {'"' '\' '/'}
+                        str(nstr+1) = inStr(pos);
+                        pos = pos + 1;
+                    case {'b' 'f' 'n' 'r' 't'}
+                        str(nstr+1) = sprintf(['\' inStr(pos)]);
+                        pos = pos + 1;
+                    case 'u'
+                        if pos+4 > length(inStr)
+                            error_pos('End of file reached in escaped unicode character',inStr);
+                        end
+                        str(nstr+(1:6)) = inStr(pos-1:pos+4);
+                        pos = pos + 5;
+                end
+            otherwise % should never happen
+                str(nstr+1) = inStr(pos);
+                keyboard;
+                pos = pos + 1;
+        end
+    end
+    str=unescapejsonstring(str);
+    error_pos('End of file while expecting end of inStr',inStr);
+
+%%-------------------------------------------------------------------------
+
+function num = parse_number(inStr, varargin)
+import jsonlab.*
+    global pos isoct
+    currstr=inStr(pos:min(pos+30,end));
+    if(isoct~=0)
+        numstr=regexp(currstr,'^\s*-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+\-]?\d+)?','end');
+        [num] = sscanf(currstr, '%f', 1);
+        delta=numstr+1;
+    else
+        [num, one, err, delta] = sscanf(currstr, '%f', 1);
+        if ~isempty(err)
+            error_pos('Error reading number at position %d',inStr);
+        end
+    end
+    pos = pos + delta-1;
+
+%%-------------------------------------------------------------------------
+
+function val = parse_value(inStr, esc, varargin)
+import jsonlab.*
+    global pos
+    len=length(inStr);
+    if(isfield(varargin{1},'progressbar_'))
+        waitbar(pos/len,varargin{1}.progressbar_,'loading ...');
+    end
+    
+    switch(inStr(pos))
+        case '"'
+            val = parseStr(inStr, esc, varargin{:});
+            return;
+        case '['
+            val = parse_array(inStr, esc, varargin{:});
+            return;
+        case '{'
+            val = parse_object(inStr, esc, varargin{:});
+            return;
+        case {'-','0','1','2','3','4','5','6','7','8','9'}
+            val = parse_number(inStr, varargin{:});
+            return;
+        case 't'
+            if pos+3 <= len && strcmpi(inStr(pos:pos+3), 'true')
+                val = true;
+                pos = pos + 4;
+                return;
+            end
+        case 'f'
+            if pos+4 <= len && strcmpi(inStr(pos:pos+4), 'false')
+                val = false;
+                pos = pos + 5;
+                return;
+            end
+        case 'n'
+            if pos+3 <= len && strcmpi(inStr(pos:pos+3), 'null')
+                val = [];
+                pos = pos + 4;
+                return;
+            end
+    end
+    error_pos('Value expected at position %d',inStr);
+%%-------------------------------------------------------------------------
+
+function error_pos(msg, inStr)
+import jsonlab.*
+    global pos len
+    poShow = max(min([pos-15 pos-1 pos pos+20],len),1);
+    if poShow(3) == poShow(2)
+        poShow(3:4) = poShow(2)+[0 -1];  % display nothing after
+    end
+    msg = [sprintf(msg, pos) ': ' ...
+    inStr(poShow(1):poShow(2)) '<error>' inStr(poShow(3):poShow(4)) ];
+    error( ['JSONparser:invalidFormat: ' msg] );
+
+%%-------------------------------------------------------------------------
+
+function str = valid_field(str)
+import jsonlab.*
+global isoct
+% From MATLAB doc: field names must begin with a letter, which may be
+% followed by any combination of letters, digits, and underscores.
+% Invalid characters will be converted to underscores, and the prefix
+% "x0x[Hex code]_" will be added if the first character is not a letter.
+    pos=regexp(str,'^[^A-Za-z]','once');
+    if(~isempty(pos))
+        if(~isoct && str(1)+0 > 255)
+            str=regexprep(str,'^([^A-Za-z])','x0x${sprintf(''%X'',unicode2native($1))}_','once');
+        else
+            str=sprintf('x0x%X_%s',char(str(1)),str(2:end));
+        end
+    end
+    if(isempty(regexp(str,'[^0-9A-Za-z_]', 'once' )))
+        return;
+    end
+    if(~isoct)
+        str=regexprep(str,'([^0-9A-Za-z_])','_0x${sprintf(''%X'',unicode2native($1))}_');
+    else
+        pos=regexp(str,'[^0-9A-Za-z_]');
+        if(isempty(pos))
+            return;
+        end
+        str0=str;
+        pos0=[0 pos(:)' length(str)];
+        str='';
+        for i=1:length(pos)
+            str=[str str0(pos0(i)+1:pos(i)-1) sprintf('_0x%X_',str0(pos(i)))];
+        end
+        if(pos(end)~=length(str))
+            str=[str str0(pos0(end-1)+1:pos0(end))];
+        end
+    end
+    %str(~isletter(str) & ~('0' <= str & str <= '9')) = '_';
+
+%%-------------------------------------------------------------------------
+function endpos = matching_quote(str,pos)
+import jsonlab.*
+len=length(str);
+while(pos<len)
+    if(str(pos)=='"')
+        if(~(pos>1 && str(pos-1)=='\'))
+            endpos=pos;
+            return;
+        end        
+    end
+    pos=pos+1;
+end
+error('unmatched quotation mark');
+%%-------------------------------------------------------------------------
+function [endpos, e1l, e1r, maxlevel] = matching_bracket(str,pos)
+import jsonlab.*
+global arraytoken
+level=1;
+maxlevel=level;
+endpos=0;
+bpos=arraytoken(arraytoken>=pos);
+tokens=str(bpos);
+len=length(tokens);
+pos=1;
+e1l=[];
+e1r=[];
+while(pos<=len)
+    c=tokens(pos);
+    if(c==']')
+        level=level-1;
+        if(isempty(e1r))
+            e1r=bpos(pos);
+        end
+        if(level==0)
+            endpos=bpos(pos);
+            return
+        end
+    elseif(c=='[')
+        if(isempty(e1l))
+            e1l=bpos(pos);
+        end
+        level=level+1;
+        maxlevel=max(maxlevel,level);
+    elseif(c=='"')
+        pos=matching_quote(tokens,pos+1);
+    end
+    pos=pos+1;
+end
+if(endpos==0) 
+    error('unmatched "]"');
+end
+
+function newstr=unescapejsonstring(str)
+import jsonlab.*
+newstr=str;
+if(~ischar(str))
+    return;
+end
+escapechars={'\\','\"','\/','\a','\b','\f','\n','\r','\t','\v'};
+for i=1:length(escapechars);
+    newstr=regexprep(newstr,regexprep(escapechars{i},'\\','\\\\'), escapechars{i});
+end
+newstr=regexprep(newstr,'\\u([0-9A-Fa-f]{4})', '${char(base2dec($1,16))}');

matlab/+jsonlab/loadubjson.m

@@ -0,0 +1,465 @@
+function data = loadubjson(fname,varargin)
+import jsonlab.*
+%
+% data=loadubjson(fname,opt)
+%    or
+% data=loadubjson(fname,'param1',value1,'param2',value2,...)
+%
+% parse a JSON (JavaScript Object Notation) file or string
+%
+% authors:Qianqian Fang (q.fang <at> neu.edu)
+% created on 2013/08/01
+%
+% $Id$
+%
+% input:
+%      fname: input file name, if fname contains "{}" or "[]", fname
+%             will be interpreted as a UBJSON string
+%      opt: a struct to store parsing options, opt can be replaced by 
+%           a list of ('param',value) pairs - the param string is equivallent
+%           to a field in opt. opt can have the following 
+%           fields (first in [.|.] is the default)
+%
+%           opt.SimplifyCell [0|1]: if set to 1, loadubjson will call cell2mat
+%                         for each element of the JSON data, and group 
+%                         arrays based on the cell2mat rules.
+%           opt.IntEndian [B|L]: specify the endianness of the integer fields
+%                         in the UBJSON input data. B - Big-Endian format for 
+%                         integers (as required in the UBJSON specification); 
+%                         L - input integer fields are in Little-Endian order.
+%           opt.NameIsString [0|1]: for UBJSON Specification Draft 8 or 
+%                         earlier versions (JSONLab 1.0 final or earlier), 
+%                         the "name" tag is treated as a string. To load 
+%                         these UBJSON data, you need to manually set this 
+%                         flag to 1.
+%
+% output:
+%      dat: a cell array, where {...} blocks are converted into cell arrays,
+%           and [...] are converted to arrays
+%
+% examples:
+%      obj=struct('string','value','array',[1 2 3]);
+%      ubjdata=saveubjson('obj',obj);
+%      dat=loadubjson(ubjdata)
+%      dat=loadubjson(['examples' filesep 'example1.ubj'])
+%      dat=loadubjson(['examples' filesep 'example1.ubj'],'SimplifyCell',1)
+%
+% license:
+%     BSD, see LICENSE_BSD.txt file for details 
+%
+% -- this function is part of JSONLab toolbox (http://iso2mesh.sf.net/cgi-bin/index.cgi?jsonlab)
+%
+
+global pos inStr len  esc index_esc len_esc isoct arraytoken fileendian systemendian
+
+if(regexp(fname,'[\{\}\]\[]','once'))
+   string=fname;
+elseif(exist(fname,'file'))
+   fid = fopen(fname,'rb');
+   string = fread(fid,inf,'uint8=>char')';
+   fclose(fid);
+else
+   error('input file does not exist');
+end
+
+pos = 1; len = length(string); inStr = string;
+isoct=exist('OCTAVE_VERSION','builtin');
+arraytoken=find(inStr=='[' | inStr==']' | inStr=='"');
+jstr=regexprep(inStr,'\\\\','  ');
+escquote=regexp(jstr,'\\"');
+arraytoken=sort([arraytoken escquote]);
+
+% String delimiters and escape chars identified to improve speed:
+esc = find(inStr=='"' | inStr=='\' ); % comparable to: regexp(inStr, '["\\]');
+index_esc = 1; len_esc = length(esc);
+
+opt=varargin2struct(varargin{:});
+fileendian=upper(jsonopt('IntEndian','B',opt));
+[os,maxelem,systemendian]=computer;
+
+jsoncount=1;
+while pos <= len
+    switch(next_char)
+        case '{'
+            data{jsoncount} = parse_object(opt);
+        case '['
+            data{jsoncount} = parse_array(opt);
+        otherwise
+            error_pos('Outer level structure must be an object or an array');
+    end
+    jsoncount=jsoncount+1;
+end % while
+
+jsoncount=length(data);
+if(jsoncount==1 && iscell(data))
+    data=data{1};
+end
+
+%%-------------------------------------------------------------------------
+function object = parse_object(varargin)
+import jsonlab.*
+    parse_char('{');
+    object = [];
+    type='';
+    count=-1;
+    if(next_char == '$')
+        type=inStr(pos+1); % TODO
+        pos=pos+2;
+    end
+    if(next_char == '#')
+        pos=pos+1;
+        count=double(parse_number());
+    end
+    if next_char ~= '}'
+        num=0;
+        while 1
+            if(jsonopt('NameIsString',0,varargin{:}))
+                str = parseStr(varargin{:});
+            else
+                str = parse_name(varargin{:});
+            end
+            if isempty(str)
+                error_pos('Name of value at position %d cannot be empty');
+            end
+            %parse_char(':');
+            val = parse_value(varargin{:});
+            num=num+1;
+            object.(valid_field(str))=val;
+            if next_char == '}' || (count>=0 && num>=count)
+                break;
+            end
+            %parse_char(',');
+        end
+    end
+    if(count==-1)
+        parse_char('}');
+    end
+    if(isstruct(object))
+        object=struct2jdata(object,struct('Base64',0));
+    end
+
+%%-------------------------------------------------------------------------
+function [cid,len]=elem_info(type)
+import jsonlab.*
+id=strfind('iUIlLdD',type);
+dataclass={'int8','uint8','int16','int32','int64','single','double'};
+bytelen=[1,1,2,4,8,4,8];
+if(id>0)
+    cid=dataclass{id};
+    len=bytelen(id);
+else
+    error_pos('unsupported type at position %d');
+end
+%%-------------------------------------------------------------------------
+
+
+function [data, adv]=parse_block(type,count,varargin)
+import jsonlab.*
+global pos inStr isoct fileendian systemendian
+[cid,len]=elem_info(type);
+datastr=inStr(pos:pos+len*count-1);
+newdata=uint8(datastr);
+id=strfind('iUIlLdD',type);
+if(fileendian~=systemendian)
+    newdata=swapbytes(typecast(newdata,cid));
+end
+data=typecast(newdata,cid);
+adv=double(len*count);
+
+%%-------------------------------------------------------------------------
+
+
+function object = parse_array(varargin) % JSON array is written in row-major order
+import jsonlab.*
+global pos inStr
+    parse_char('[');
+    object = cell(0, 1);
+    dim=[];
+    type='';
+    count=-1;
+    if(next_char == '$')
+        type=inStr(pos+1);
+        pos=pos+2;
+    end
+    if(next_char == '#')
+        pos=pos+1;
+        if(next_char=='[')
+            dim=parse_array(varargin{:});
+            count=prod(double(dim));
+        else
+            count=double(parse_number());
+        end
+    end
+    if(~isempty(type))
+        if(count>=0)
+            [object, adv]=parse_block(type,count,varargin{:});
+            if(~isempty(dim))
+                object=reshape(object,dim);
+            end
+            pos=pos+adv;
+            return;
+        else
+            endpos=matching_bracket(inStr,pos);
+            [cid,len]=elem_info(type);
+            count=(endpos-pos)/len;
+            [object, adv]=parse_block(type,count,varargin{:});
+            pos=pos+adv;
+            parse_char(']');
+            return;
+        end
+    end
+    if next_char ~= ']'
+         while 1
+            val = parse_value(varargin{:});
+            object{end+1} = val;
+            if next_char == ']'
+                break;
+            end
+            %parse_char(',');
+         end
+    end
+    if(jsonopt('SimplifyCell',0,varargin{:})==1)
+      try
+        oldobj=object;
+        object=cell2mat(object')';
+        if(iscell(oldobj) && isstruct(object) && numel(object)>1 && jsonopt('SimplifyCellArray',1,varargin{:})==0)
+            object=oldobj;
+        elseif(size(object,1)>1 && ismatrix(object))
+            object=object';
+        end
+      catch
+      end
+    end
+    if(count==-1)
+        parse_char(']');
+    end
+
+%%-------------------------------------------------------------------------
+
+function parse_char(c)
+import jsonlab.*
+    global pos inStr len
+    skip_whitespace;
+    if pos > len || inStr(pos) ~= c
+        error_pos(sprintf('Expected %c at position %%d', c));
+    else
+        pos = pos + 1;
+        skip_whitespace;
+    end
+
+%%-------------------------------------------------------------------------
+
+function c = next_char
+import jsonlab.*
+    global pos inStr len
+    skip_whitespace;
+    if pos > len
+        c = [];
+    else
+        c = inStr(pos);
+    end
+
+%%-------------------------------------------------------------------------
+
+function skip_whitespace
+import jsonlab.*
+    global pos inStr len
+    while pos <= len && isspace(inStr(pos))
+        pos = pos + 1;
+    end
+
+%%-------------------------------------------------------------------------
+function str = parse_name(varargin)
+import jsonlab.*
+    global pos inStr
+    bytelen=double(parse_number());
+    if(length(inStr)>=pos+bytelen-1)
+        str=inStr(pos:pos+bytelen-1);
+        pos=pos+bytelen;
+    else
+        error_pos('End of file while expecting end of name');
+    end
+%%-------------------------------------------------------------------------
+
+function str = parseStr(varargin)
+import jsonlab.*
+    global pos inStr
+ % len, ns = length(inStr), keyboard
+    type=inStr(pos);
+    if type ~= 'S' && type ~= 'C' && type ~= 'H'
+        error_pos('String starting with S expected at position %d');
+    else
+        pos = pos + 1;
+    end
+    if(type == 'C')
+        str=inStr(pos);
+        pos=pos+1;
+        return;
+    end
+    bytelen=double(parse_number());
+    if(length(inStr)>=pos+bytelen-1)
+        str=inStr(pos:pos+bytelen-1);
+        pos=pos+bytelen;
+    else
+        error_pos('End of file while expecting end of inStr');
+    end
+
+%%-------------------------------------------------------------------------
+
+function num = parse_number(varargin)
+import jsonlab.*
+    global pos inStr isoct fileendian systemendian
+    id=strfind('iUIlLdD',inStr(pos));
+    if(isempty(id))
+        error_pos('expecting a number at position %d');
+    end
+    type={'int8','uint8','int16','int32','int64','single','double'};
+    bytelen=[1,1,2,4,8,4,8];
+    datastr=inStr(pos+1:pos+bytelen(id));
+    newdata=uint8(datastr);
+    if(fileendian~=systemendian)
+        newdata=swapbytes(typecast(newdata,type{id}));
+    end
+    num=typecast(newdata,type{id});
+    pos = pos + bytelen(id)+1;
+
+%%-------------------------------------------------------------------------
+
+function val = parse_value(varargin)
+import jsonlab.*
+    global pos inStr
+
+    switch(inStr(pos))
+        case {'S','C','H'}
+            val = parseStr(varargin{:});
+            return;
+        case '['
+            val = parse_array(varargin{:});
+            return;
+        case '{'
+            val = parse_object(varargin{:});
+            return;
+        case {'i','U','I','l','L','d','D'}
+            val = parse_number(varargin{:});
+            return;
+        case 'T'
+            val = true;
+            pos = pos + 1;
+            return;
+        case 'F'
+            val = false;
+            pos = pos + 1;
+            return;
+        case {'Z','N'}
+            val = [];
+            pos = pos + 1;
+            return;
+    end
+    error_pos('Value expected at position %d');
+%%-------------------------------------------------------------------------
+
+function error_pos(msg)
+import jsonlab.*
+    global pos inStr len
+    poShow = max(min([pos-15 pos-1 pos pos+20],len),1);
+    if poShow(3) == poShow(2)
+        poShow(3:4) = poShow(2)+[0 -1];  % display nothing after
+    end
+    msg = [sprintf(msg, pos) ': ' ...
+    inStr(poShow(1):poShow(2)) '<error>' inStr(poShow(3):poShow(4)) ];
+    error( ['JSONparser:invalidFormat: ' msg] );
+
+%%-------------------------------------------------------------------------
+
+function str = valid_field(str)
+import jsonlab.*
+global isoct
+% From MATLAB doc: field names must begin with a letter, which may be
+% followed by any combination of letters, digits, and underscores.
+% Invalid characters will be converted to underscores, and the prefix
+% "x0x[Hex code]_" will be added if the first character is not a letter.
+    pos=regexp(str,'^[^A-Za-z]','once');
+    if(~isempty(pos))
+        if(~isoct)
+            str=regexprep(str,'^([^A-Za-z])','x0x${sprintf(''%X'',unicode2native($1))}_','once');
+        else
+            str=sprintf('x0x%X_%s',char(str(1)),str(2:end));
+        end
+    end
+    if(isempty(regexp(str,'[^0-9A-Za-z_]', 'once' )))
+        return;
+    end
+    if(~isoct)
+        str=regexprep(str,'([^0-9A-Za-z_])','_0x${sprintf(''%X'',unicode2native($1))}_');
+    else
+        pos=regexp(str,'[^0-9A-Za-z_]');
+        if(isempty(pos))
+            return;
+        end
+        str0=str;
+        pos0=[0 pos(:)' length(str)];
+        str='';
+        for i=1:length(pos)
+            str=[str str0(pos0(i)+1:pos(i)-1) sprintf('_0x%X_',str0(pos(i)))];
+        end
+        if(pos(end)~=length(str))
+            str=[str str0(pos0(end-1)+1:pos0(end))];
+        end
+    end
+    %str(~isletter(str) & ~('0' <= str & str <= '9')) = '_';
+
+%%-------------------------------------------------------------------------
+function endpos = matching_quote(str,pos)
+import jsonlab.*
+len=length(str);
+while(pos<len)
+    if(str(pos)=='"')
+        if(~(pos>1 && str(pos-1)=='\'))
+            endpos=pos;
+            return;
+        end        
+    end
+    pos=pos+1;
+end
+error('unmatched quotation mark');
+%%-------------------------------------------------------------------------
+function [endpos, e1l, e1r, maxlevel] = matching_bracket(str,pos)
+import jsonlab.*
+global arraytoken
+level=1;
+maxlevel=level;
+endpos=0;
+bpos=arraytoken(arraytoken>=pos);
+tokens=str(bpos);
+len=length(tokens);
+pos=1;
+e1l=[];
+e1r=[];
+while(pos<=len)
+    c=tokens(pos);
+    if(c==']')
+        level=level-1;
+        if(isempty(e1r))
+            e1r=bpos(pos);
+        end
+        if(level==0)
+            endpos=bpos(pos);
+            return
+        end
+    end
+    if(c=='[')
+        if(isempty(e1l))
+            e1l=bpos(pos);
+        end
+        level=level+1;
+        maxlevel=max(maxlevel,level);
+    end
+    if(c=='"')
+        pos=matching_quote(tokens,pos+1);
+    end
+    pos=pos+1;
+end
+if(endpos==0) 
+    error('unmatched "]"');
+end
+

matlab/+jsonlab/mergestruct.m

@@ -0,0 +1,34 @@
+function s=mergestruct(s1,s2)
+import jsonlab.*
+%
+% s=mergestruct(s1,s2)
+%
+% merge two struct objects into one
+%
+% authors:Qianqian Fang (q.fang <at> neu.edu)
+% date: 2012/12/22
+%
+% input:
+%      s1,s2: a struct object, s1 and s2 can not be arrays
+%
+% output:
+%      s: the merged struct object. fields in s1 and s2 will be combined in s.
+%
+% license:
+%     BSD, see LICENSE_BSD.txt file for details 
+%
+% -- this function is part of jsonlab toolbox (http://iso2mesh.sf.net/cgi-bin/index.cgi?jsonlab)
+%
+
+if(~isstruct(s1) || ~isstruct(s2))
+    error('input parameters contain non-struct');
+end
+if(length(s1)>1 || length(s2)>1)
+    error('can not merge struct arrays');
+end
+fn=fieldnames(s2);
+s=s1;
+for i=1:length(fn)              
+    s=setfield(s,fn{i},getfield(s2,fn{i}));
+end
+

matlab/+jsonlab/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "jsonlab",
+  "version": "1.9",
+  "description": "An open-source MATLAB/Octave JSON encoder and decoder",
+  "directories": {
+    "example": "examples"
+  },
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/fangq/jsonlab.git"
+  },
+  "author": "Qianqian Fang <q.fang at neu.edu>",
+  "license": "BSD",
+  "bugs": {
+    "url": "https://github.com/fangq/jsonlab/issues"
+  },
+  "homepage": "https://iso2mesh.sf.net/jsonlab/"
+}

matlab/+jsonlab/savejson.m

@@ -0,0 +1,686 @@
+function json=savejson(rootname,obj,varargin)
+import jsonlab.*
+%
+% json=savejson(rootname,obj,filename)
+%    or
+% json=savejson(rootname,obj,opt)
+% json=savejson(rootname,obj,'param1',value1,'param2',value2,...)
+%
+% convert a MATLAB object (cell, struct or array) into a JSON (JavaScript
+% Object Notation) string
+%
+% author: Qianqian Fang (q.fang <at> neu.edu)
+% created on 2011/09/09
+%
+% $Id$
+%
+% input:
+%      rootname: the name of the root-object, when set to '', the root name
+%        is ignored, however, when opt.ForceRootName is set to 1 (see below),
+%        the MATLAB variable name will be used as the root name.
+%      obj: a MATLAB object (array, cell, cell array, struct, struct array,
+%      class instance).
+%      filename: a string for the file name to save the output JSON data.
+%      opt: a struct for additional options, ignore to use default values.
+%        opt can have the following fields (first in [.|.] is the default)
+%
+%        opt.FileName [''|string]: a file name to save the output JSON data
+%        opt.FloatFormat ['%.10g'|string]: format to show each numeric element
+%                         of a 1D/2D array;
+%        opt.ArrayIndent [1|0]: if 1, output explicit data array with
+%                         precedent indentation; if 0, no indentation
+%        opt.ArrayToStruct[0|1]: when set to 0, savejson outputs 1D/2D
+%                         array in JSON array format; if sets to 1, an
+%                         array will be shown as a struct with fields
+%                         "_ArrayType_", "_ArraySize_" and "_ArrayData_"; for
+%                         sparse arrays, the non-zero elements will be
+%                         saved to _ArrayData_ field in triplet-format i.e.
+%                         (ix,iy,val) and "_ArrayIsSparse_" will be added
+%                         with a value of 1; for a complex array, the 
+%                         _ArrayData_ array will include two columns 
+%                         (4 for sparse) to record the real and imaginary 
+%                         parts, and also "_ArrayIsComplex_":1 is added. 
+%        opt.ParseLogical [0|1]: if this is set to 1, logical array elem
+%                         will use true/false rather than 1/0.
+%        opt.SingletArray [0|1]: if this is set to 1, arrays with a single
+%                         numerical element will be shown without a square
+%                         bracket, unless it is the root object; if 0, square
+%                         brackets are forced for any numerical arrays.
+%        opt.SingletCell  [1|0]: if 1, always enclose a cell with "[]" 
+%                         even it has only one element; if 0, brackets
+%                         are ignored when a cell has only 1 element.
+%        opt.ForceRootName [0|1]: when set to 1 and rootname is empty, savejson
+%                         will use the name of the passed obj variable as the 
+%                         root object name; if obj is an expression and 
+%                         does not have a name, 'root' will be used; if this 
+%                         is set to 0 and rootname is empty, the root level 
+%                         will be merged down to the lower level.
+%        opt.Inf ['"$1_Inf_"'|string]: a customized regular expression pattern
+%                         to represent +/-Inf. The matched pattern is '([-+]*)Inf'
+%                         and $1 represents the sign. For those who want to use
+%                         1e999 to represent Inf, they can set opt.Inf to '$11e999'
+%        opt.NaN ['"_NaN_"'|string]: a customized regular expression pattern
+%                         to represent NaN
+%        opt.JSONP [''|string]: to generate a JSONP output (JSON with padding),
+%                         for example, if opt.JSONP='foo', the JSON data is
+%                         wrapped inside a function call as 'foo(...);'
+%        opt.UnpackHex [1|0]: conver the 0x[hex code] output by loadjson 
+%                         back to the string form
+%        opt.SaveBinary [0|1]: 1 - save the JSON file in binary mode; 0 - text mode.
+%        opt.Compact [0|1]: 1- out compact JSON format (remove all newlines and tabs)
+%        opt.Compression  'zlib' or 'gzip': specify array compression
+%                         method; currently only supports 'gzip' or 'zlib'. The
+%                         data compression only applicable to numerical arrays 
+%                         in 3D or higher dimensions, or when ArrayToStruct
+%                         is 1 for 1D or 2D arrays. If one wants to
+%                         compress a long string, one must convert
+%                         it to uint8 or int8 array first. The compressed
+%                         array uses three extra fields
+%                         "_ArrayCompressionMethod_": the opt.Compression value. 
+%                         "_ArrayCompressionSize_": a 1D interger array to
+%                            store the pre-compressed (but post-processed)
+%                            array dimensions, and 
+%                         "_ArrayCompressedData_": the "base64" encoded
+%                             compressed binary array data. 
+%        opt.CompressArraySize [100|int]: only to compress an array if the total 
+%                         element count is larger than this number.
+%        opt can be replaced by a list of ('param',value) pairs. The param 
+%        string is equivallent to a field in opt and is case sensitive.
+% output:
+%      json: a string in the JSON format (see http://json.org)
+%
+% examples:
+%      jsonmesh=struct('MeshNode',[0 0 0;1 0 0;0 1 0;1 1 0;0 0 1;1 0 1;0 1 1;1 1 1],... 
+%               'MeshTetra',[1 2 4 8;1 3 4 8;1 2 6 8;1 5 6 8;1 5 7 8;1 3 7 8],...
+%               'MeshTri',[1 2 4;1 2 6;1 3 4;1 3 7;1 5 6;1 5 7;...
+%                          2 8 4;2 8 6;3 8 4;3 8 7;5 8 6;5 8 7],...
+%               'MeshCreator','FangQ','MeshTitle','T6 Cube',...
+%               'SpecialData',[nan, inf, -inf]);
+%      savejson('jmesh',jsonmesh)
+%      savejson('',jsonmesh,'ArrayIndent',0,'FloatFormat','\t%.5g')
+%
+% license:
+%     BSD, see LICENSE_BSD.txt file for details
+%
+% -- this function is part of JSONLab toolbox (http://iso2mesh.sf.net/cgi-bin/index.cgi?jsonlab)
+%
+
+if(nargin==1)
+   varname=inputname(1);
+   obj=rootname;
+   if(isempty(varname)) 
+      varname='root';
+   end
+   rootname=varname;
+else
+   varname=inputname(2);
+end
+if(length(varargin)==1 && ischar(varargin{1}))
+   opt=struct('filename',varargin{1});
+else
+   opt=varargin2struct(varargin{:});
+end
+opt.IsOctave=exist('OCTAVE_VERSION','builtin');
+
+dozip=jsonopt('Compression','',opt);
+if(~isempty(dozip))
+    if(~(strcmpi(dozip,'gzip') || strcmpi(dozip,'zlib')))
+        error('compression method "%s" is not supported',dozip);
+    end
+    if(exist('zmat')~=3)
+        try
+            error(javachk('jvm'));
+            try
+               base64decode('test');
+            catch
+               matlab.net.base64decode('test');
+            end
+        catch
+            error('java-based compression is not supported');
+        end
+    end    
+    opt.Compression=dozip;
+end
+
+if(isfield(opt,'norowbracket'))
+    warning('Option ''NoRowBracket'' is depreciated, please use ''SingletArray'' and set its value to not(NoRowBracket)');
+    if(~isfield(opt,'singletarray'))
+        opt.singletarray=not(opt.norowbracket);
+    end
+end
+rootisarray=0;
+rootlevel=1;
+forceroot=jsonopt('ForceRootName',0,opt);
+if((isnumeric(obj) || islogical(obj) || ischar(obj) || isstruct(obj) || ...
+        iscell(obj) || isobject(obj)) && isempty(rootname) && forceroot==0)
+    rootisarray=1;
+    rootlevel=0;
+else
+    if(isempty(rootname))
+        rootname=varname;
+    end
+end
+if((isstruct(obj) || iscell(obj))&& isempty(rootname) && forceroot)
+    rootname='root';
+end
+
+whitespaces=struct('tab',sprintf('\t'),'newline',sprintf('\n'),'sep',sprintf(',\n'));
+if(jsonopt('Compact',0,opt)==1)
+    whitespaces=struct('tab','','newline','','sep',',');
+end
+if(~isfield(opt,'whitespaces_'))
+    opt.whitespaces_=whitespaces;
+end
+
+nl=whitespaces.newline;
+
+json=obj2json(rootname,obj,rootlevel,opt);
+if(rootisarray)
+    json=sprintf('%s%s',json,nl);
+else
+    json=sprintf('{%s%s%s}\n',nl,json,nl);
+end
+
+jsonp=jsonopt('JSONP','',opt);
+if(~isempty(jsonp))
+    json=sprintf('%s(%s);%s',jsonp,json,nl);
+end
+
+% save to a file if FileName is set, suggested by Patrick Rapin
+filename=jsonopt('FileName','',opt);
+if(~isempty(filename))
+    if(jsonopt('SaveBinary',0,opt)==1)
+	    fid = fopen(filename, 'wb');
+	    fwrite(fid,json);
+    else
+	    fid = fopen(filename, 'wt');
+	    fwrite(fid,json,'char');
+    end
+    fclose(fid);
+end
+
+%%-------------------------------------------------------------------------
+function txt=obj2json(name,item,level,varargin)
+import jsonlab.*
+
+if(iscell(item) || isa(item,'string'))
+    txt=cell2json(name,item,level,varargin{:});
+elseif(isstruct(item))
+    txt=struct2json(name,item,level,varargin{:});
+elseif(ischar(item))
+    txt=str2json(name,item,level,varargin{:});
+elseif(isobject(item))
+    if(~exist('OCTAVE_VERSION','builtin') && exist('istable','builtin') && istable(item))
+        txt=matlabtable2json(name,item,level,varargin{:});
+    else
+        txt=matlabobject2json(name,item,level,varargin{:});
+    end
+elseif(isa(item,'function_handle'))
+    txt=struct2json(name,functions(item),level,varargin{:});
+else
+    txt=mat2json(name,item,level,varargin{:});
+end
+
+%%-------------------------------------------------------------------------
+function txt=cell2json(name,item,level,varargin)
+import jsonlab.*
+txt={};
+if(~iscell(item) && ~isa(item,'string'))
+        error('input is not a cell or string array');
+end
+
+dim=size(item);
+if(ndims(squeeze(item))>2) % for 3D or higher dimensions, flatten to 2D for now
+    item=reshape(item,dim(1),numel(item)/dim(1));
+    dim=size(item);
+end
+len=numel(item);
+ws=jsonopt('whitespaces_',struct('tab',sprintf('\t'),'newline',sprintf('\n'),'sep',sprintf(',\n')),varargin{:});
+padding0=repmat(ws.tab,1,level);
+padding2=repmat(ws.tab,1,level+1);
+nl=ws.newline;
+bracketlevel=~jsonopt('singletcell',1,varargin{:});
+if(len>bracketlevel)
+    if(~isempty(name))
+        txt={padding0, '"', checkname(name,varargin{:}),'": [', nl}; name=''; 
+    else
+        txt={padding0, '[', nl};
+    end
+elseif(len==0)
+    if(~isempty(name))
+        txt={padding0, '"' checkname(name,varargin{:}) '": []'}; name=''; 
+    else
+        txt={padding0, '[]'};
+    end
+end
+for i=1:dim(1)
+    if(dim(1)>1)
+        txt(end+1:end+3)={padding2,'[',nl};
+    end
+    for j=1:dim(2)
+       txt{end+1}=obj2json(name,item{i,j},level+(dim(1)>1)+(len>bracketlevel),varargin{:});
+       if(j<dim(2))
+           txt(end+1:end+2)={',' nl};
+       end
+    end
+    if(dim(1)>1)
+        txt(end+1:end+3)={nl,padding2,']'};
+    end
+    if(i<dim(1))
+        txt(end+1:end+2)={',' nl};
+    end
+    %if(j==dim(2)) txt=sprintf('%s%s',txt,sprintf(',%s',nl)); end
+end
+if(len>bracketlevel)
+    txt(end+1:end+3)={nl,padding0,']'};
+end
+txt = sprintf('%s',txt{:});
+
+%%-------------------------------------------------------------------------
+function txt=struct2json(name,item,level,varargin)
+import jsonlab.*
+txt={};
+if(~isstruct(item))
+	error('input is not a struct');
+end
+dim=size(item);
+if(ndims(squeeze(item))>2) % for 3D or higher dimensions, flatten to 2D for now
+    item=reshape(item,dim(1),numel(item)/dim(1));
+    dim=size(item);
+end
+len=numel(item);
+forcearray= (len>1 || (jsonopt('SingletArray',0,varargin{:})==1 && level>0));
+ws=struct('tab',sprintf('\t'),'newline',sprintf('\n'));
+ws=jsonopt('whitespaces_',ws,varargin{:});
+padding0=repmat(ws.tab,1,level);
+padding2=repmat(ws.tab,1,level+1);
+padding1=repmat(ws.tab,1,level+(dim(1)>1)+forcearray);
+nl=ws.newline;
+
+if(isempty(item)) 
+    if(~isempty(name)) 
+        txt={padding0, '"', checkname(name,varargin{:}),'": []'};
+    else
+        txt={padding0, '[]'};
+    end
+    txt = sprintf('%s',txt{:});
+    return;
+end
+if(~isempty(name)) 
+    if(forcearray)
+        txt={padding0, '"', checkname(name,varargin{:}),'": [', nl};
+    end
+else
+    if(forcearray)
+        txt={padding0, '[', nl};
+    end
+end
+for j=1:dim(2)
+  if(dim(1)>1)
+      txt(end+1:end+3)={padding2,'[',nl};
+  end
+  for i=1:dim(1)
+    names = fieldnames(item(i,j));
+    if(~isempty(name) && len==1 && ~forcearray)
+        txt(end+1:end+5)={padding1, '"', checkname(name,varargin{:}),'": {', nl};
+    else
+        txt(end+1:end+3)={padding1, '{', nl};
+    end
+    if(~isempty(names))
+      for e=1:length(names)
+	    txt{end+1}=obj2json(names{e},item(i,j).(names{e}),...
+             level+(dim(1)>1)+1+forcearray,varargin{:});
+        if(e<length(names))
+            txt{end+1}=',';
+        end
+        txt{end+1}=nl;
+      end
+    end
+    txt(end+1:end+2)={padding1,'}'};
+    if(i<dim(1))
+        txt(end+1:end+2)={',' nl};
+    end
+  end
+  if(dim(1)>1)
+      txt(end+1:end+3)={nl,padding2,']'};
+  end
+  if(j<dim(2))
+      txt(end+1:end+2)={',' nl};
+  end
+end
+if(forcearray)
+    txt(end+1:end+3)={nl,padding0,']'};
+end
+txt = sprintf('%s',txt{:});
+
+%%-------------------------------------------------------------------------
+function txt=str2json(name,item,level,varargin)
+import jsonlab.*
+txt={};
+if(~ischar(item))
+        error('input is not a string');
+end
+item=reshape(item, max(size(item),[1 0]));
+len=size(item,1);
+ws=struct('tab',sprintf('\t'),'newline',sprintf('\n'),'sep',sprintf(',\n'));
+ws=jsonopt('whitespaces_',ws,varargin{:});
+padding1=repmat(ws.tab,1,level);
+padding0=repmat(ws.tab,1,level+1);
+nl=ws.newline;
+sep=ws.sep;
+
+if(~isempty(name)) 
+    if(len>1)
+        txt={padding1, '"', checkname(name,varargin{:}),'": [', nl};
+    end
+else
+    if(len>1)
+        txt={padding1, '[', nl};
+    end
+end
+for e=1:len
+    val=escapejsonstring(item(e,:));
+    if(len==1)
+        obj=['"' checkname(name,varargin{:}) '": ' '"',val,'"'];
+        if(isempty(name))
+            obj=['"',val,'"'];
+        end
+        txt(end+1:end+2)={padding1, obj};
+    else
+        txt(end+1:end+4)={padding0,'"',val,'"'};
+    end
+    if(e==len)
+        sep='';
+    end
+    txt{end+1}=sep;
+end
+if(len>1)
+    txt(end+1:end+3)={nl,padding1,']'};
+end
+txt = sprintf('%s',txt{:});
+
+%%-------------------------------------------------------------------------
+function txt=mat2json(name,item,level,varargin)
+import jsonlab.*
+if(~isnumeric(item) && ~islogical(item))
+        error('input is not an array');
+end
+ws=struct('tab',sprintf('\t'),'newline',sprintf('\n'),'sep',sprintf(',\n'));
+ws=jsonopt('whitespaces_',ws,varargin{:});
+padding1=repmat(ws.tab,1,level);
+padding0=repmat(ws.tab,1,level+1);
+nl=ws.newline;
+sep=ws.sep;
+
+dozip=jsonopt('Compression','',varargin{:});
+zipsize=jsonopt('CompressArraySize',100,varargin{:});
+
+if(length(size(item))>2 || issparse(item) || ~isreal(item) || ...
+   (isempty(item) && any(size(item))) ||jsonopt('ArrayToStruct',0,varargin{:}) || (~isempty(dozip) && numel(item)>zipsize))
+    if(isempty(name))
+    	txt=sprintf('%s{%s%s"_ArrayType_": "%s",%s%s"_ArraySize_": %s,%s',...
+              padding1,nl,padding0,class(item),nl,padding0,regexprep(mat2str(size(item)),'\s+',','),nl);
+    else
+    	txt=sprintf('%s"%s": {%s%s"_ArrayType_": "%s",%s%s"_ArraySize_": %s,%s',...
+              padding1,checkname(name,varargin{:}),nl,padding0,class(item),nl,padding0,regexprep(mat2str(size(item)),'\s+',','),nl);
+    end
+else
+    if(numel(item)==1 && jsonopt('SingletArray',0,varargin{:})==0 && level>0)
+        numtxt=regexprep(regexprep(matdata2json(item,level+1,varargin{:}),'^\[',''),']$','');
+    else
+        numtxt=matdata2json(item,level+1,varargin{:});
+    end
+    if(isempty(name))
+    	txt=sprintf('%s%s',padding1,numtxt);
+    else
+        if(numel(item)==1 && jsonopt('SingletArray',0,varargin{:})==0)
+           	txt=sprintf('%s"%s": %s',padding1,checkname(name,varargin{:}),numtxt);
+        else
+    	    txt=sprintf('%s"%s": %s',padding1,checkname(name,varargin{:}),numtxt);
+        end
+    end
+    return;
+end
+dataformat='%s%s%s%s%s';
+
+if(issparse(item))
+    [ix,iy]=find(item);
+    data=full(item(find(item)));
+    if(~isreal(item))
+       data=[real(data(:)),imag(data(:))];
+       if(size(item,1)==1)
+           % Kludge to have data's 'transposedness' match item's.
+           % (Necessary for complex row vector handling below.)
+           data=data';
+       end
+       txt=sprintf(dataformat,txt,padding0,'"_ArrayIsComplex_": ','1', sep);
+    end
+    txt=sprintf(dataformat,txt,padding0,'"_ArrayIsSparse_": ','1', sep);
+    if(~isempty(dozip) && numel(data*2)>zipsize)
+        if(size(item,1)==1)
+            % Row vector, store only column indices.
+            fulldata=[iy(:),data'];
+        elseif(size(item,2)==1)
+            % Column vector, store only row indices.
+            fulldata=[ix,data];
+        else
+            % General case, store row and column indices.
+            fulldata=[ix,iy,data];
+        end
+        txt=sprintf(dataformat,txt,padding0,'"_ArrayCompressionSize_": ',regexprep(mat2str(size(fulldata)),'\s+',','), sep);
+        txt=sprintf(dataformat,txt,padding0,'"_ArrayCompressionMethod_": "',dozip, ['"' sep]);
+        if(strcmpi(dozip,'gzip'))
+            txt=sprintf(dataformat,txt,padding0,'"_ArrayCompressedData_": "',base64encode(gzipencode(typecast(fulldata(:),'uint8'))),['"' nl]);
+        elseif(strcmpi(dozip,'zlib'))
+            txt=sprintf(dataformat,txt,padding0,'"_ArrayCompressedData_": "',base64encode(zlibencode(typecast(fulldata(:),'uint8'))),['"' nl]);
+        else
+            error('compression method not supported');
+        end
+    else
+        if(size(item,1)==1)
+            % Row vector, store only column indices.
+            txt=sprintf(dataformat,txt,padding0,'"_ArrayData_": ',...
+               matdata2json([iy(:),data'],level+2,varargin{:}), nl);
+        elseif(size(item,2)==1)
+            % Column vector, store only row indices.
+            txt=sprintf(dataformat,txt,padding0,'"_ArrayData_": ',...
+               matdata2json([ix,data],level+2,varargin{:}), nl);
+        else
+            % General case, store row and column indices.
+            txt=sprintf(dataformat,txt,padding0,'"_ArrayData_": ',...
+               matdata2json([ix,iy,data],level+2,varargin{:}), nl);
+        end
+    end
+else
+    if(~isempty(dozip) && numel(item)>zipsize)
+        if(isreal(item))
+            fulldata=item(:)';
+            if(islogical(fulldata))
+                fulldata=uint8(fulldata);
+            end
+        else
+            txt=sprintf(dataformat,txt,padding0,'"_ArrayIsComplex_": ','1', sep);
+            fulldata=[real(item(:)) imag(item(:))];
+        end
+        txt=sprintf(dataformat,txt,padding0,'"_ArrayCompressionSize_": ',regexprep(mat2str(size(fulldata)),'\s+',','), sep);
+        txt=sprintf(dataformat,txt,padding0,'"_ArrayCompressionMethod_": "',dozip, ['"' sep]);
+        if(strcmpi(dozip,'gzip'))
+            txt=sprintf(dataformat,txt,padding0,'"_ArrayCompressedData_": "',base64encode(gzipencode(typecast(fulldata(:),'uint8'))),['"' nl]);
+        elseif(strcmpi(dozip,'zlib'))
+            txt=sprintf(dataformat,txt,padding0,'"_ArrayCompressedData_": "',base64encode(zlibencode(typecast(fulldata(:),'uint8'))),['"' nl]);
+        else
+            error('compression method not supported');
+        end
+    else
+        if(isreal(item))
+            txt=sprintf(dataformat,txt,padding0,'"_ArrayData_": ',...
+            matdata2json(item(:)',level+2,varargin{:}), nl);
+        else
+            txt=sprintf(dataformat,txt,padding0,'"_ArrayIsComplex_": ','1', sep);
+            txt=sprintf(dataformat,txt,padding0,'"_ArrayData_": ',...
+            matdata2json([real(item(:)) imag(item(:))],level+2,varargin{:}), nl);
+        end
+    end
+end
+txt=sprintf('%s%s%s',txt,padding1,'}');
+
+%%-------------------------------------------------------------------------
+function txt=matlabobject2json(name,item,level,varargin)
+import jsonlab.*
+if numel(item) == 0 %empty object
+    st = struct();
+elseif numel(item) == 1 %
+    st = struct();
+    txt = str2json(name, char(item), level, varargin(:));
+    return
+else
+    % "st = struct(item);" would produce an inmutable warning, because it
+    % make the protected and private properties visible. Instead we get the
+    % visible properties
+    propertynames = properties(item);
+    for p = 1:numel(propertynames)
+        for o = numel(item):-1:1 % aray of objects
+            st(o).(propertynames{p}) = item(o).(propertynames{p});
+        end
+    end
+end
+txt=struct2json(name,st,level,varargin{:});
+
+%%-------------------------------------------------------------------------
+function txt=matlabtable2json(name,item,level,varargin)
+import jsonlab.*
+if numel(item) == 0 %empty object
+    st = struct();
+else
+    % "st = struct(item);" would produce an inmutable warning, because it
+    % make the protected and private properties visible. Instead we get the
+    % visible properties
+    st = struct();
+    propertynames = properties(item);
+    if(isfield(item.Properties,'RowNames') && ~isempty(item.Properties.RowNames))
+        rownames=item.Properties.RowNames;
+        for p = 1:(numel(propertynames)-1)
+            for j = 1:size(item(:,p),1)
+                st.(rownames{j}).(propertynames{p}) = item{j,p};
+            end
+        end
+    else
+        for p = 1:(numel(propertynames)-1)
+            for j = 1:size(item(:,p),1)
+                st(j).(propertynames{p}) = item{j,p};
+            end
+        end
+    end
+end
+txt=struct2json(name,st,level,varargin{:});
+
+%%-------------------------------------------------------------------------
+function txt=matdata2json(mat,level,varargin)
+import jsonlab.*
+
+ws=struct('tab',sprintf('\t'),'newline',sprintf('\n'),'sep',sprintf(',\n'));
+ws=jsonopt('whitespaces_',ws,varargin{:});
+tab=ws.tab;
+nl=ws.newline;
+
+if(size(mat,1)==1)
+    pre='';
+    post='';
+    level=level-1;
+else
+    pre=sprintf('[%s',nl);
+    post=sprintf('%s%s]',nl,repmat(tab,1,level-1));
+end
+
+if(isempty(mat))
+    txt='null';
+    return;
+end
+if(isinteger(mat))
+  floatformat=jsonopt('FloatFormat','%d',varargin{:});
+else
+  floatformat=jsonopt('FloatFormat','%.10g',varargin{:});
+end
+%if(numel(mat)>1)
+    formatstr=['[' repmat([floatformat ','],1,size(mat,2)-1) [floatformat sprintf('],%s',nl)]];
+%else
+%    formatstr=[repmat([floatformat ','],1,size(mat,2)-1) [floatformat sprintf(',\n')]];
+%end
+
+if(nargin>=2 && size(mat,1)>1 && jsonopt('ArrayIndent',1,varargin{:})==1)
+    formatstr=[repmat(tab,1,level) formatstr];
+end
+
+txt=sprintf(formatstr,mat');
+txt(end-length(nl):end)=[];
+if(islogical(mat) && jsonopt('ParseLogical',0,varargin{:})==1)
+   txt=regexprep(txt,'1','true');
+   txt=regexprep(txt,'0','false');
+end
+%txt=regexprep(mat2str(mat),'\s+',',');
+%txt=regexprep(txt,';',sprintf('],\n['));
+% if(nargin>=2 && size(mat,1)>1)
+%     txt=regexprep(txt,'\[',[repmat(sprintf('\t'),1,level) '[']);
+% end
+txt=[pre txt post];
+if(any(isinf(mat(:))))
+    txt=regexprep(txt,'([-+]*)Inf',jsonopt('Inf','"$1_Inf_"',varargin{:}));
+end
+if(any(isnan(mat(:))))
+    txt=regexprep(txt,'NaN',jsonopt('NaN','"_NaN_"',varargin{:}));
+end
+
+%%-------------------------------------------------------------------------
+function newname=checkname(name,varargin)
+import jsonlab.*
+isunpack=jsonopt('UnpackHex',1,varargin{:});
+newname=name;
+if(isempty(regexp(name,'0x([0-9a-fA-F]+)_','once')))
+    return
+end
+if(isunpack)
+    isoct=jsonopt('IsOctave',0,varargin{:});
+    if(~isoct)
+        newname=regexprep(name,'(^x|_){1}0x([0-9a-fA-F]+)_','${native2unicode(hex2dec($2))}');
+    else
+        pos=regexp(name,'(^x|_){1}0x([0-9a-fA-F]+)_','start');
+        pend=regexp(name,'(^x|_){1}0x([0-9a-fA-F]+)_','end');
+        if(isempty(pos))
+            return;
+        end
+        str0=name;
+        pos0=[0 pend(:)' length(name)];
+        newname='';
+        for i=1:length(pos)
+            newname=[newname str0(pos0(i)+1:pos(i)-1) char(hex2dec(str0(pos(i)+3:pend(i)-1)))];
+        end
+        if(pos(end)~=length(name))
+            newname=[newname str0(pos0(end-1)+1:pos0(end))];
+        end
+    end
+end
+
+%%-------------------------------------------------------------------------
+function newstr=escapejsonstring(str)
+import jsonlab.*
+newstr=str;
+isoct=exist('OCTAVE_VERSION','builtin');
+if(isoct)
+   vv=sscanf(OCTAVE_VERSION,'%f');
+   if(vv(1)>=3.8)
+       isoct=0;
+   end
+end
+if(isoct)
+  escapechars={'\\','\"','\/','\a','\f','\n','\r','\t','\v'};
+  for i=1:length(escapechars);
+    newstr=regexprep(newstr,escapechars{i},escapechars{i});
+  end
+  newstr=regexprep(newstr,'\\\\(u[0-9a-fA-F]{4}[^0-9a-fA-F]*)','\$1');
+else
+  escapechars={'\\','\"','\/','\a','\b','\f','\n','\r','\t','\v'};
+  for i=1:length(escapechars);
+    newstr=regexprep(newstr,escapechars{i},regexprep(escapechars{i},'\\','\\\\'));
+  end
+  newstr=regexprep(newstr,'\\\\(u[0-9a-fA-F]{4}[^0-9a-fA-F]*)','\\$1');
+end

matlab/+jsonlab/saveubjson.m

@@ -0,0 +1,663 @@
+function json=saveubjson(rootname,obj,varargin)
+import jsonlab.*
+%
+% json=saveubjson(rootname,obj,filename)
+%    or
+% json=saveubjson(rootname,obj,opt)
+% json=saveubjson(rootname,obj,'param1',value1,'param2',value2,...)
+%
+% convert a MATLAB object (cell, struct or array) into a Universal 
+% Binary JSON (UBJSON) binary string
+%
+% author: Qianqian Fang (q.fang <at> neu.edu)
+% created on 2013/08/17
+%
+% $Id$
+%
+% input:
+%      rootname: the name of the root-object, when set to '', the root name
+%        is ignored, however, when opt.ForceRootName is set to 1 (see below),
+%        the MATLAB variable name will be used as the root name.
+%      obj: a MATLAB object (array, cell, cell array, struct, struct array,
+%      class instance)
+%      filename: a string for the file name to save the output UBJSON data
+%      opt: a struct for additional options, ignore to use default values.
+%        opt can have the following fields (first in [.|.] is the default)
+%
+%        opt.FileName [''|string]: a file name to save the output JSON data
+%        opt.ArrayToStruct[0|1]: when set to 0, saveubjson outputs 1D/2D
+%                         array in JSON array format; if sets to 1, an
+%                         array will be shown as a struct with fields
+%                         "_ArrayType_", "_ArraySize_" and "_ArrayData_"; for
+%                         sparse arrays, the non-zero elements will be
+%                         saved to _ArrayData_ field in triplet-format i.e.
+%                         (ix,iy,val) and "_ArrayIsSparse_" will be added
+%                         with a value of 1; for a complex array, the 
+%                         _ArrayData_ array will include two columns 
+%                         (4 for sparse) to record the real and imaginary 
+%                         parts, and also "_ArrayIsComplex_":1 is added. 
+%        opt.ParseLogical [1|0]: if this is set to 1, logical array elem
+%                         will use true/false rather than 1/0.
+%        opt.SingletArray [0|1]: if this is set to 1, arrays with a single
+%                         numerical element will be shown without a square
+%                         bracket, unless it is the root object; if 0, square
+%                         brackets are forced for any numerical arrays.
+%        opt.SingletCell  [1|0]: if 1, always enclose a cell with "[]" 
+%                         even it has only one element; if 0, brackets
+%                         are ignored when a cell has only 1 element.
+%        opt.ForceRootName [0|1]: when set to 1 and rootname is empty, saveubjson
+%                         will use the name of the passed obj variable as the 
+%                         root object name; if obj is an expression and 
+%                         does not have a name, 'root' will be used; if this 
+%                         is set to 0 and rootname is empty, the root level 
+%                         will be merged down to the lower level.
+%        opt.JSONP [''|string]: to generate a JSONP output (JSON with padding),
+%                         for example, if opt.JSON='foo', the JSON data is
+%                         wrapped inside a function call as 'foo(...);'
+%        opt.UnpackHex [1|0]: conver the 0x[hex code] output by loadjson 
+%                         back to the string form
+%        opt.Compression  'zlib' or 'gzip': specify array compression
+%                         method; currently only supports 'gzip' or 'zlib'. The
+%                         data compression only applicable to numerical arrays 
+%                         in 3D or higher dimensions, or when ArrayToStruct
+%                         is 1 for 1D or 2D arrays. If one wants to
+%                         compress a long string, one must convert
+%                         it to uint8 or int8 array first. The compressed
+%                         array uses three extra fields
+%                         "_ArrayCompressionMethod_": the opt.Compression value. 
+%                         "_ArrayCompressionSize_": a 1D interger array to
+%                            store the pre-compressed (but post-processed)
+%                            array dimensions, and 
+%                         "_ArrayCompressedData_": the binary stream of
+%                            the compressed binary array data WITHOUT
+%                            'base64' encoding
+%        opt.CompressArraySize [100|int]: only to compress an array if the total 
+%                         element count is larger than this number.
+%
+%        opt can be replaced by a list of ('param',value) pairs. The param 
+%        string is equivallent to a field in opt and is case sensitive.
+% output:
+%      json: a binary string in the UBJSON format (see http://ubjson.org)
+%
+% examples:
+%      jsonmesh=struct('MeshNode',[0 0 0;1 0 0;0 1 0;1 1 0;0 0 1;1 0 1;0 1 1;1 1 1],... 
+%               'MeshTetra',[1 2 4 8;1 3 4 8;1 2 6 8;1 5 6 8;1 5 7 8;1 3 7 8],...
+%               'MeshTri',[1 2 4;1 2 6;1 3 4;1 3 7;1 5 6;1 5 7;...
+%                          2 8 4;2 8 6;3 8 4;3 8 7;5 8 6;5 8 7],...
+%               'MeshCreator','FangQ','MeshTitle','T6 Cube',...
+%               'SpecialData',[nan, inf, -inf]);
+%      saveubjson('jsonmesh',jsonmesh)
+%      saveubjson('jsonmesh',jsonmesh,'meshdata.ubj')
+%
+% license:
+%     BSD, see LICENSE_BSD.txt file for details
+%
+% -- this function is part of JSONLab toolbox (http://iso2mesh.sf.net/cgi-bin/index.cgi?jsonlab)
+%
+
+if(nargin==1)
+   varname=inputname(1);
+   obj=rootname;
+   if(isempty(varname)) 
+      varname='root';
+   end
+   rootname=varname;
+else
+   varname=inputname(2);
+end
+if(length(varargin)==1 && ischar(varargin{1}))
+   opt=struct('filename',varargin{1});
+else
+   opt=varargin2struct(varargin{:});
+end
+opt.IsOctave=exist('OCTAVE_VERSION','builtin');
+
+dozip=jsonopt('Compression','',opt);
+if(~isempty(dozip))
+    if(~(strcmpi(dozip,'gzip') || strcmpi(dozip,'zlib')))
+        error('compression method "%s" is not supported',dozip);
+    end
+    if(exist('zmat')~=3)
+        try
+            error(javachk('jvm'));
+            try
+               base64decode('test');
+            catch
+               matlab.net.base64decode('test');
+            end
+        catch
+            error('java-based compression is not supported');
+        end
+    end    
+    opt.Compression=dozip;
+end
+
+if(isfield(opt,'norowbracket'))
+    warning('Option ''NoRowBracket'' is depreciated, please use ''SingletArray'' and set its value to not(NoRowBracket)');
+    if(~isfield(opt,'singletarray'))
+        opt.singletarray=not(opt.norowbracket);
+    end
+end
+rootisarray=0;
+rootlevel=1;
+forceroot=jsonopt('ForceRootName',0,opt);
+if((isnumeric(obj) || islogical(obj) || ischar(obj) || isstruct(obj) || ...
+        iscell(obj) || isobject(obj)) && isempty(rootname) && forceroot==0)
+    rootisarray=1;
+    rootlevel=0;
+else
+    if(isempty(rootname))
+        rootname=varname;
+    end
+end
+if((isstruct(obj) || iscell(obj))&& isempty(rootname) && forceroot)
+    rootname='root';
+end
+json=obj2ubjson(rootname,obj,rootlevel,opt);
+if(~rootisarray)
+    json=['{' json '}'];
+end
+
+jsonp=jsonopt('JSONP','',opt);
+if(~isempty(jsonp))
+    json=[jsonp '(' json ')'];
+end
+
+% save to a file if FileName is set, suggested by Patrick Rapin
+filename=jsonopt('FileName','',opt);
+if(~isempty(filename))
+    fid = fopen(filename, 'wb');
+    fwrite(fid,json);
+    fclose(fid);
+end
+
+%%-------------------------------------------------------------------------
+function txt=obj2ubjson(name,item,level,varargin)
+import jsonlab.*
+
+if(iscell(item))
+    txt=cell2ubjson(name,item,level,varargin{:});
+elseif(isstruct(item))
+    txt=struct2ubjson(name,item,level,varargin{:});
+elseif(ischar(item))
+    txt=str2ubjson(name,item,level,varargin{:});
+elseif(isobject(item)) 
+    txt=matlabobject2ubjson(name,item,level,varargin{:});
+else
+    txt=mat2ubjson(name,item,level,varargin{:});
+end
+
+%%-------------------------------------------------------------------------
+function txt=cell2ubjson(name,item,level,varargin)
+import jsonlab.*
+txt='';
+if(~iscell(item))
+        error('input is not a cell');
+end
+
+dim=size(item);
+if(ndims(squeeze(item))>2) % for 3D or higher dimensions, flatten to 2D for now
+    item=reshape(item,dim(1),numel(item)/dim(1));
+    dim=size(item);
+end
+bracketlevel=~jsonopt('singletcell',1,varargin{:});
+len=numel(item); % let's handle 1D cell first
+if(len>bracketlevel) 
+    if(~isempty(name))
+        txt=[N_(checkname(name,varargin{:})) '[']; name=''; 
+    else
+        txt='['; 
+    end
+elseif(len==0)
+    if(~isempty(name))
+        txt=[N_(checkname(name,varargin{:})) 'Z']; name=''; 
+    else
+        txt='Z'; 
+    end
+end
+for j=1:dim(2)
+    if(dim(1)>1)
+        txt=[txt '['];
+    end
+    for i=1:dim(1)
+       txt=[txt obj2ubjson(name,item{i,j},level+(len>bracketlevel),varargin{:})];
+    end
+    if(dim(1)>1)
+        txt=[txt ']'];
+    end
+end
+if(len>bracketlevel)
+    txt=[txt ']'];
+end
+
+%%-------------------------------------------------------------------------
+function txt=struct2ubjson(name,item,level,varargin)
+import jsonlab.*
+txt='';
+if(~isstruct(item))
+	error('input is not a struct');
+end
+dim=size(item);
+if(ndims(squeeze(item))>2) % for 3D or higher dimensions, flatten to 2D for now
+    item=reshape(item,dim(1),numel(item)/dim(1));
+    dim=size(item);
+end
+len=numel(item);
+forcearray= (len>1 || (jsonopt('SingletArray',0,varargin{:})==1 && level>0));
+
+if(~isempty(name)) 
+    if(forcearray)
+        txt=[N_(checkname(name,varargin{:})) '['];
+    end
+else
+    if(forcearray)
+        txt='[';
+    end
+end
+for j=1:dim(2)
+  if(dim(1)>1)
+      txt=[txt '['];
+  end
+  for i=1:dim(1)
+     names = fieldnames(item(i,j));
+     if(~isempty(name) && len==1 && ~forcearray)
+        txt=[txt N_(checkname(name,varargin{:})) '{']; 
+     else
+        txt=[txt '{']; 
+     end
+     if(~isempty(names))
+       for e=1:length(names)
+	     txt=[txt obj2ubjson(names{e},item(i,j).(names{e}),...
+             level+(dim(1)>1)+1+forcearray,varargin{:})];
+       end
+     end
+     txt=[txt '}'];
+  end
+  if(dim(1)>1)
+      txt=[txt ']'];
+  end
+end
+if(forcearray)
+    txt=[txt ']'];
+end
+
+%%-------------------------------------------------------------------------
+function txt=str2ubjson(name,item,level,varargin)
+import jsonlab.*
+txt='';
+if(~ischar(item))
+        error('input is not a string');
+end
+item=reshape(item, max(size(item),[1 0]));
+len=size(item,1);
+
+if(~isempty(name)) 
+    if(len>1)
+        txt=[N_(checkname(name,varargin{:})) '['];
+    end
+else
+    if(len>1)
+        txt='[';
+    end
+end
+for e=1:len
+    val=item(e,:);
+    if(len==1)
+        obj=[N_(checkname(name,varargin{:})) '' '',S_(val),''];
+        if(isempty(name))
+            obj=['',S_(val),''];
+        end
+        txt=[txt,'',obj];
+    else
+        txt=[txt,'',['',S_(val),'']];
+    end
+end
+if(len>1)
+    txt=[txt ']'];
+end
+
+%%-------------------------------------------------------------------------
+function txt=mat2ubjson(name,item,level,varargin)
+import jsonlab.*
+if(~isnumeric(item) && ~islogical(item))
+        error('input is not an array');
+end
+
+dozip=jsonopt('Compression','',varargin{:});
+zipsize=jsonopt('CompressArraySize',100,varargin{:});
+
+if(length(size(item))>2 || issparse(item) || ~isreal(item) || ...
+   (isempty(item) && any(size(item))) ||jsonopt('ArrayToStruct',0,varargin{:}) || (~isempty(dozip) && numel(item)>zipsize))
+      cid=I_(uint32(max(size(item))));
+      if(isempty(name))
+    	txt=['{' N_('_ArrayType_'),S_(class(item)),N_('_ArraySize_'),I_a(size(item),cid(1)) ];
+      else
+          if(isempty(item))
+              txt=[N_(checkname(name,varargin{:})),'Z'];
+              return;
+          else
+    	      txt=[N_(checkname(name,varargin{:})),'{',N_('_ArrayType_'),S_(class(item)),N_('_ArraySize_'),I_a(size(item),cid(1))];
+          end
+      end
+else
+    if(isempty(name))
+    	txt=matdata2ubjson(item,level+1,varargin{:});
+    else
+        if(numel(item)==1 && jsonopt('SingletArray',0,varargin{:})==0)
+            numtxt=regexprep(regexprep(matdata2ubjson(item,level+1,varargin{:}),'^\[',''),']$','');
+           	txt=[N_(checkname(name,varargin{:})) numtxt];
+        else
+    	    txt=[N_(checkname(name,varargin{:})),matdata2ubjson(item,level+1,varargin{:})];
+        end
+    end
+    return;
+end
+if(issparse(item))
+    [ix,iy]=find(item);
+    data=full(item(find(item)));
+    if(~isreal(item))
+       data=[real(data(:)),imag(data(:))];
+       if(size(item,1)==1)
+           % Kludge to have data's 'transposedness' match item's.
+           % (Necessary for complex row vector handling below.)
+           data=data';
+       end
+       txt=[txt,N_('_ArrayIsComplex_'),'T'];
+    end
+    txt=[txt,N_('_ArrayIsSparse_'),'T'];
+    if(~isempty(dozip) && numel(data*2)>zipsize)
+        if(size(item,1)==1)
+            % Row vector, store only column indices.
+            fulldata=[iy(:),data'];
+        elseif(size(item,2)==1)
+            % Column vector, store only row indices.
+            fulldata=[ix,data];
+        else
+            % General case, store row and column indices.
+            fulldata=[ix,iy,data];
+        end
+        cid=I_(uint32(max(size(fulldata))));
+        txt=[txt, N_('_ArrayCompressionSize_'),I_a(size(fulldata),cid(1))];
+        txt=[txt, N_('_ArrayCompressionMethod_'),S_(dozip)];
+        if(strcmpi(dozip,'gzip'))
+            txt=[txt,N_('_ArrayCompressedData_'), I_a(gzipencode(typecast(fulldata(:),'uint8')),'U')];
+        elseif(strcmpi(dozip,'zlib'))
+            txt=[txt,N_('_ArrayCompressedData_'), I_a(zlibencode(typecast(fulldata(:),'uint8')),'U')];
+        else
+            error('compression method not supported');
+        end
+    else
+        if(size(item,1)==1)
+            % Row vector, store only column indices.
+            txt=[txt,N_('_ArrayData_'),...
+               matdata2ubjson([iy(:),data'],level+2,varargin{:})];
+        elseif(size(item,2)==1)
+            % Column vector, store only row indices.
+            txt=[txt,N_('_ArrayData_'),...
+               matdata2ubjson([ix,data],level+2,varargin{:})];
+        else
+            % General case, store row and column indices.
+            txt=[txt,N_('_ArrayData_'),...
+               matdata2ubjson([ix,iy,data],level+2,varargin{:})];
+        end
+    end
+else
+    if(~isempty(dozip) && numel(item)>zipsize)
+        if(isreal(item))
+            fulldata=item(:)';
+            if(islogical(fulldata))
+                fulldata=uint8(fulldata);
+            end
+        else
+            txt=[txt,N_('_ArrayIsComplex_'),'T'];
+            fulldata=[real(item(:)) imag(item(:))];
+        end
+        cid=I_(uint32(max(size(fulldata))));
+        txt=[txt, N_('_ArrayCompressionSize_'),I_a(size(fulldata),cid(1))];
+        txt=[txt, N_('_ArrayCompressionMethod_'),S_(dozip)];
+        if(strcmpi(dozip,'gzip'))
+            txt=[txt,N_('_ArrayCompressedData_'), I_a(gzipencode(typecast(fulldata(:),'uint8')),'U')];
+        elseif(strcmpi(dozip,'zlib'))
+            txt=[txt,N_('_ArrayCompressedData_'), I_a(zlibencode(typecast(fulldata(:),'uint8')),'U')];
+        else
+            error('compression method not supported');
+        end
+    else
+        if(isreal(item))
+            txt=[txt,N_('_ArrayData_'),...
+                matdata2ubjson(item(:)',level+2,varargin{:})];
+        else
+            txt=[txt,N_('_ArrayIsComplex_'),'T'];
+            txt=[txt,N_('_ArrayData_'),...
+                matdata2ubjson([real(item(:)) imag(item(:))],level+2,varargin{:})];
+        end
+    end
+end
+txt=[txt,'}'];
+
+%%-------------------------------------------------------------------------
+function txt=matlabobject2ubjson(name,item,level,varargin)
+import jsonlab.*
+st = struct();
+if numel(item) > 0 %non-empty object
+    % "st = struct(item);" would produce an inmutable warning, because it
+    % make the protected and private properties visible. Instead we get the
+    % visible properties
+    propertynames = properties(item);
+    for p = 1:numel(propertynames)
+        for o = numel(item):-1:1 % aray of objects
+            st(o).(propertynames{p}) = item(o).(propertynames{p});
+        end
+    end
+end
+txt=struct2ubjson(name,st,level,varargin{:});
+
+%%-------------------------------------------------------------------------
+function txt=matdata2ubjson(mat,level,varargin)
+import jsonlab.*
+if(isempty(mat))
+    txt='Z';
+    return;
+end
+type='';
+hasnegtive=(mat<0);
+if(isa(mat,'integer') || isinteger(mat) || (isfloat(mat) && all(mod(mat(:),1) == 0)))
+    if(isempty(hasnegtive))
+       if(max(mat(:))<=2^8)
+           type='U';
+       end
+    end
+    if(isempty(type))
+        % todo - need to consider negative ones separately
+        id= histc(abs(max(double(mat(:)))),[0 2^7 2^15 2^31 2^63]);
+        if(isempty(id~=0))
+            error('high-precision data is not yet supported');
+        end
+        key='iIlL';
+	type=key(id~=0);
+    end
+    txt=[I_a(mat(:),type,size(mat))];
+elseif(islogical(mat))
+    logicalval='FT';
+    if(numel(mat)==1)
+        txt=logicalval(mat+1);
+    else
+        txt=['[$U#' I_a(size(mat),'l') typecast(swapbytes(uint8(mat(:)')),'uint8')];
+    end
+else
+    if(numel(mat)==1)
+        txt=['[' D_(mat) ']'];
+    else
+        txt=D_a(mat(:),'D',size(mat));
+    end
+end
+
+%txt=regexprep(mat2str(mat),'\s+',',');
+%txt=regexprep(txt,';',sprintf('],['));
+% if(nargin>=2 && size(mat,1)>1)
+%     txt=regexprep(txt,'\[',[repmat(sprintf('\t'),1,level) '[']);
+% end
+if(any(isinf(mat(:))))
+    txt=regexprep(txt,'([-+]*)Inf',jsonopt('Inf','"$1_Inf_"',varargin{:}));
+end
+if(any(isnan(mat(:))))
+    txt=regexprep(txt,'NaN',jsonopt('NaN','"_NaN_"',varargin{:}));
+end
+
+%%-------------------------------------------------------------------------
+function newname=checkname(name,varargin)
+import jsonlab.*
+isunpack=jsonopt('UnpackHex',1,varargin{:});
+newname=name;
+if(isempty(regexp(name,'0x([0-9a-fA-F]+)_','once')))
+    return
+end
+if(isunpack)
+    isoct=jsonopt('IsOctave',0,varargin{:});
+    if(~isoct)
+        newname=regexprep(name,'(^x|_){1}0x([0-9a-fA-F]+)_','${native2unicode(hex2dec($2))}');
+    else
+        pos=regexp(name,'(^x|_){1}0x([0-9a-fA-F]+)_','start');
+        pend=regexp(name,'(^x|_){1}0x([0-9a-fA-F]+)_','end');
+        if(isempty(pos))
+            return;
+        end
+        str0=name;
+        pos0=[0 pend(:)' length(name)];
+        newname='';
+        for i=1:length(pos)
+            newname=[newname str0(pos0(i)+1:pos(i)-1) char(hex2dec(str0(pos(i)+3:pend(i)-1)))];
+        end
+        if(pos(end)~=length(name))
+            newname=[newname str0(pos0(end-1)+1:pos0(end))];
+        end
+    end
+end
+%%-------------------------------------------------------------------------
+function val=N_(str)
+import jsonlab.*
+val=[I_(int32(length(str))) str];
+%%-------------------------------------------------------------------------
+function val=S_(str)
+import jsonlab.*
+if(length(str)==1)
+  val=['C' str];
+else
+  val=['S' I_(int32(length(str))) str];
+end
+%%-------------------------------------------------------------------------
+function val=I_(num)
+import jsonlab.*
+if(~isinteger(num))
+    error('input is not an integer');
+end
+if(num>=0 && num<255)
+   val=['U' data2byte(swapbytes(cast(num,'uint8')),'uint8')];
+   return;
+end
+key='iIlL';
+cid={'int8','int16','int32','int64'};
+for i=1:4
+  if((num>0 && num<2^(i*8-1)) || (num<0 && num>=-2^(i*8-1)))
+    val=[key(i) data2byte(swapbytes(cast(num,cid{i})),'uint8')];
+    return;
+  end
+end
+error('unsupported integer');
+
+%%-------------------------------------------------------------------------
+function val=D_(num)
+import jsonlab.*
+if(~isfloat(num))
+    error('input is not a float');
+end
+
+if(isa(num,'single'))
+  val=['d' data2byte(swapbytes(num),'uint8')];
+else
+  val=['D' data2byte(swapbytes(num),'uint8')];
+end
+%%-------------------------------------------------------------------------
+function data=I_a(num,type,dim,format)
+import jsonlab.*
+id=find(ismember('iUIlL',type));
+
+if(id==0)
+  error('unsupported integer array');
+end
+
+% based on UBJSON specs, all integer types are stored in big endian format
+
+if(id==1)
+  data=data2byte(swapbytes(int8(num)),'uint8');
+  blen=1;
+elseif(id==2)
+  data=data2byte(swapbytes(uint8(num)),'uint8');
+  blen=1;
+elseif(id==3)
+  data=data2byte(swapbytes(int16(num)),'uint8');
+  blen=2;
+elseif(id==4)
+  data=data2byte(swapbytes(int32(num)),'uint8');
+  blen=4;
+elseif(id==5)
+  data=data2byte(swapbytes(int64(num)),'uint8');
+  blen=8;
+end
+
+if(nargin>=3 && length(dim)>=2 && prod(dim)~=dim(2))
+  format='opt';
+end
+if((nargin<4 || strcmp(format,'opt')) && numel(num)>1)
+  if(nargin>=3 && (length(dim)==1 || (length(dim)>=2 && prod(dim)~=dim(2))))
+      cid=I_(uint32(max(dim)));
+      data=['$' type '#' I_a(dim,cid(1)) data(:)'];
+  else
+      data=['$' type '#' I_(int32(numel(data)/blen)) data(:)'];
+  end
+  data=['[' data(:)'];
+else
+  data=reshape(data,blen,numel(data)/blen);
+  data(2:blen+1,:)=data;
+  data(1,:)=type;
+  data=data(:)';
+  data=['[' data(:)' ']'];
+end
+%%-------------------------------------------------------------------------
+function data=D_a(num,type,dim,format)
+import jsonlab.*
+id=find(ismember('dD',type));
+
+if(id==0)
+  error('unsupported float array');
+end
+
+if(id==1)
+  data=data2byte(swapbytes(single(num)),'uint8');
+elseif(id==2)
+  data=data2byte(swapbytes(double(num)),'uint8');
+end
+
+if(nargin>=3 && length(dim)>=2 && prod(dim)~=dim(2))
+  format='opt';
+end
+if((nargin<4 || strcmp(format,'opt')) && numel(num)>1)
+  if(nargin>=3 && (length(dim)==1 || (length(dim)>=2 && prod(dim)~=dim(2))))
+      cid=I_(uint32(max(dim)));
+      data=['$' type '#' I_a(dim,cid(1)) data(:)'];
+  else
+      data=['$' type '#' I_(int32(numel(data)/(id*4))) data(:)'];
+  end
+  data=['[' data];
+else
+  data=reshape(data,(id*4),length(data)/(id*4));
+  data(2:(id*4+1),:)=data;
+  data(1,:)=type;
+  data=data(:)';
+  data=['[' data(:)' ']'];
+end
+%%-------------------------------------------------------------------------
+function bytes=data2byte(varargin)
+import jsonlab.*
+bytes=typecast(varargin{:});
+bytes=char(bytes(:)');

matlab/+jsonlab/struct2jdata.m

@@ -0,0 +1,126 @@
+function newdata=struct2jdata(data,varargin)
+import jsonlab.*
+%
+% newdata=struct2jdata(data,opt,...)
+%
+% convert a JData object (in the form of a struct array) into an array
+%
+% authors:Qianqian Fang (q.fang <at> neu.edu)
+%
+% input:
+%      data: a struct array. If data contains JData keywords in the first
+%            level children, these fields are parsed and regrouped into a
+%            data object (arrays, trees, graphs etc) based on JData 
+%            specification. The JData keywords are
+%               "_ArrayType_", "_ArraySize_", "_ArrayData_"
+%               "_ArrayIsSparse_", "_ArrayIsComplex_", 
+%               "_ArrayCompressionMethod_", "_ArrayCompressionSize",
+%               "_ArrayCompressedData_"
+%      opt: (optional) a list of 'Param',value pairs for additional options 
+%           The supported options include
+%               'Recursive', if set to 1, will apply the conversion to 
+%                            every child; 0 to disable
+%               'Base64'. if set to 1, _ArrayCompressedData_ is assumed to
+%                         be encoded with base64 format and need to be
+%                         decoded first. This is needed for JSON but not
+%                         UBJSON data
+%
+% output:
+%      newdata: the covnerted data if the input data does contain a JData 
+%               structure; otherwise, the same as the input.
+%
+% examples:
+%      obj=struct('_ArrayType_','double','_ArraySize_',[2 3],
+%                 '_ArrayIsSparse_',1 ,'_ArrayData_',null);
+%      ubjdata=struct2jdata(obj);
+%
+% license:
+%     BSD, see LICENSE_BSD.txt file for details 
+%
+% -- this function is part of JSONLab toolbox (http://iso2mesh.sf.net/cgi-bin/index.cgi?jsonlab)
+%
+
+fn=fieldnames(data);
+newdata=data;
+len=length(data);
+needbase64=jsonopt('Base64',1,varargin{:});
+if(jsonopt('Recursive',0,varargin{:})==1)
+  for i=1:length(fn) % depth-first
+    for j=1:len
+        if(isstruct(getfield(data(j),fn{i})))
+            newdata(j)=setfield(newdata(j),fn{i},jstruct2array(getfield(data(j),fn{i})));
+        end
+    end
+  end
+end
+if(~isempty(strmatch('x0x5F_ArrayType_',fn)) && (~isempty(strmatch('x0x5F_ArrayData_',fn)) || ~isempty(strmatch('x0x5F_ArrayCompressedData_',fn))))
+  newdata=cell(len,1);
+  for j=1:len
+    if(~isempty(strmatch('x0x5F_ArrayCompressionSize_',fn)) && ~isempty(strmatch('x0x5F_ArrayCompressedData_',fn)))
+        zipmethod='zip';
+        if(~isempty(strmatch('x0x5F_ArrayCompressionMethod_',fn)))
+            zipmethod=data(j).x0x5F_ArrayCompressionMethod_;
+        end
+        if(strcmpi(zipmethod,'gzip'))
+            if(needbase64)
+                ndata=reshape(typecast(gzipdecode(base64decode(data(j).x0x5F_ArrayCompressedData_)),data(j).x0x5F_ArrayType_),data(j).x0x5F_ArrayCompressionSize_);
+            else
+                ndata=reshape(typecast(gzipdecode(data(j).x0x5F_ArrayCompressedData_),data(j).x0x5F_ArrayType_),data(j).x0x5F_ArrayCompressionSize_);
+            end
+        elseif(strcmpi(zipmethod,'zlib'))
+            if(needbase64)
+                ndata=reshape(typecast(zlibdecode(base64decode(data(j).x0x5F_ArrayCompressedData_)),data(j).x0x5F_ArrayType_),data(j).x0x5F_ArrayCompressionSize_);
+            else
+                ndata=reshape(typecast(zlibdecode(data(j).x0x5F_ArrayCompressedData_),data(j).x0x5F_ArrayType_),data(j).x0x5F_ArrayCompressionSize_);
+            end
+        else
+            error('compression method is not supported');
+        end
+    else
+        ndata=cast(data(j).x0x5F_ArrayData_,data(j).x0x5F_ArrayType_);
+    end
+    iscpx=0;
+    if(~isempty(strmatch('x0x5F_ArrayIsComplex_',fn)))
+        if(data(j).x0x5F_ArrayIsComplex_)
+           iscpx=1;
+        end
+    end
+    if(~isempty(strmatch('x0x5F_ArrayIsSparse_',fn)))
+        if(data(j).x0x5F_ArrayIsSparse_)
+            if(~isempty(strmatch('x0x5F_ArraySize_',fn)))
+                dim=double(data(j).x0x5F_ArraySize_);
+                if(iscpx && size(ndata,2)==4-any(dim==1))
+                    ndata(:,end-1)=complex(ndata(:,end-1),ndata(:,end));
+                end
+                if isempty(ndata)
+                    % All-zeros sparse
+                    ndata=sparse(dim(1),prod(dim(2:end)));
+                elseif dim(1)==1
+                    % Sparse row vector
+                    ndata=sparse(1,ndata(:,1),ndata(:,2),dim(1),prod(dim(2:end)));
+                elseif dim(2)==1
+                    % Sparse column vector
+                    ndata=sparse(ndata(:,1),1,ndata(:,2),dim(1),prod(dim(2:end)));
+                else
+                    % Generic sparse array.
+                    ndata=sparse(ndata(:,1),ndata(:,2),ndata(:,3),dim(1),prod(dim(2:end)));
+                end
+            else
+                if(iscpx && size(ndata,2)==4)
+                    ndata(:,3)=complex(ndata(:,3),ndata(:,4));
+                end
+                ndata=sparse(ndata(:,1),ndata(:,2),ndata(:,3));
+            end
+        end
+    elseif(~isempty(strmatch('x0x5F_ArraySize_',fn)))
+        if(iscpx && size(ndata,2)==2)
+             ndata=complex(ndata(:,1),ndata(:,2));
+        end
+        ndata=reshape(ndata(:),data(j).x0x5F_ArraySize_);
+    end
+    newdata{j}=ndata;
+  end
+  if(len==1)
+      newdata=newdata{1};
+  end
+end

matlab/+jsonlab/varargin2struct.m

@@ -0,0 +1,41 @@
+function opt=varargin2struct(varargin)
+import jsonlab.*
+%
+% opt=varargin2struct('param1',value1,'param2',value2,...)
+%   or
+% opt=varargin2struct(...,optstruct,...)
+%
+% convert a series of input parameters into a structure
+%
+% authors:Qianqian Fang (q.fang <at> neu.edu)
+% date: 2012/12/22
+%
+% input:
+%      'param', value: the input parameters should be pairs of a string and a value
+%       optstruct: if a parameter is a struct, the fields will be merged to the output struct
+%
+% output:
+%      opt: a struct where opt.param1=value1, opt.param2=value2 ...
+%
+% license:
+%     BSD, see LICENSE_BSD.txt file for details 
+%
+% -- this function is part of jsonlab toolbox (http://iso2mesh.sf.net/cgi-bin/index.cgi?jsonlab)
+%
+
+len=length(varargin);
+opt=struct;
+if(len==0) return; end
+i=1;
+while(i<=len)
+    if(isstruct(varargin{i}))
+        opt=mergestruct(opt,varargin{i});
+    elseif(ischar(varargin{i}) && i<len)
+        opt=setfield(opt,lower(varargin{i}),varargin{i+1});
+        i=i+1;
+    else
+        error('input must be in the form of ...,''name'',value,... pairs or structs');
+    end
+    i=i+1;
+end
+

matlab/+jsonlab/zlibdecode.m

@@ -0,0 +1,41 @@
+function output = zlibdecode(input)
+import jsonlab.*
+%ZLIBDECODE Decompress input bytes using ZLIB.
+%
+%    output = zlibdecode(input)
+%
+% The function takes a compressed byte array INPUT and returns inflated
+% bytes OUTPUT. The INPUT is a result of GZIPENCODE function. The OUTPUT
+% is always an 1-by-N uint8 array. JAVA must be enabled to use the function.
+%
+% See also zlibencode typecast
+%
+% Copyright (c) 2012, Kota Yamaguchi
+% URL: https://www.mathworks.com/matlabcentral/fileexchange/39526-byte-encoding-utilities
+% License : BSD, see LICENSE_*.txt
+%
+
+if(nargin==0)
+    error('you must provide at least 1 input');
+end
+if(exist('zmat')==3)
+    output=zmat(uint8(input),0,'zlib');
+    return;
+end
+error(javachk('jvm'));
+if ischar(input)
+  warning('zlibdecode:inputTypeMismatch', ...
+          'Input is char, but treated as uint8.');
+  input = uint8(input);
+end
+if ~isa(input, 'int8') && ~isa(input, 'uint8')
+    error('Input must be either int8 or uint8.');
+end
+
+buffer = java.io.ByteArrayOutputStream();
+zlib = java.util.zip.InflaterOutputStream(buffer);
+zlib.write(input, 0, numel(input));
+zlib.close();
+output = typecast(buffer.toByteArray(), 'uint8')';
+
+end

matlab/+jsonlab/zlibencode.m

@@ -0,0 +1,40 @@
+function output = zlibencode(input)
+import jsonlab.*
+%ZLIBENCODE Compress input bytes with ZLIB.
+%
+%    output = zlibencode(input)
+%
+% The function takes a char, int8, or uint8 array INPUT and returns
+% compressed bytes OUTPUT as a uint8 array. Note that the compression
+% doesn't preserve input dimensions. JAVA must be enabled to use the
+% function.
+%
+% See also zlibdecode typecast
+%
+% Copyright (c) 2012, Kota Yamaguchi
+% URL: https://www.mathworks.com/matlabcentral/fileexchange/39526-byte-encoding-utilities
+% License : BSD, see LICENSE_*.txt
+%
+
+
+if(nargin==0)
+    error('you must provide at least 1 input');
+end
+if(exist('zmat')==3)
+    output=zmat(input,1,'zlib');
+    return;
+end
+error(javachk('jvm'));
+if ischar(input), input = uint8(input); end
+if ~isa(input, 'int8') && ~isa(input, 'uint8')
+    error('Input must be either char, int8 or uint8.');
+end
+
+buffer = java.io.ByteArrayOutputStream();
+zlib = java.util.zip.DeflaterOutputStream(buffer);
+zlib.write(input, 0, numel(input));
+zlib.close();
+output = typecast(buffer.toByteArray(), 'uint8')';
+
+end
+

matlab/+msgpack/README.md

@@ -0,0 +1,89 @@
+# A MessagePack implementation for Matlab and Octave
+
+The code is written in pure Matlab, and has no dependencies beyond Matlab itself. And it works in recent versions of Octave, too.
+
+The files in this repository are taken from [Transplant](https://github.com/bastibe/transplant).
+
+## Basic Usage:
+```matlab
+data = {'life, the universe, and everything', struct('the_answer', 42)};
+bytes = dumpmsgpack(data)
+data = parsemsgpack(bytes)
+% returns: {'life, the universe, and everything', containers.Map('the_answer', 42)}
+```
+
+## Converting Matlab to MsgPack:
+
+| Matlab         | MsgPack                   |
+| -------------- | ------------------------- |
+| string         | string                    |
+| scalar         | number                    |
+| logical        | `true`/`false`            |
+| vector         | array of numbers          |
+| uint8 vector   | bin                       |
+| matrix         | array of array of numbers |
+| empty matrix   | nil                       |
+| cell array     | array                     |
+| cell matrix    | array of arrays           |
+| struct         | map                       |
+| containers.Map | map                       |
+| struct array   | array of maps             |
+| handles        | raise error               |
+
+There is no way of encoding exts
+
+## Converting MsgPack to Matlab
+
+| MsgPack        | Matlab         |
+| -------------- | -------------- |
+| string         | string         |
+| number         | scalar         |
+| `true`/`false` | logical        |
+| nil            | empty matrix   |
+| array          | cell array     |
+| map            | containers.Map |
+| bin            | uint8          |
+| ext            | uint8          |
+
+Note that since `structs` don't support arbitrary field names, they can't be used for representing `maps`. We use `containers.Map` instead.
+
+## Tests
+ ```matlab
+ runtests()
+ ```
+
+## License
+
+MATLAB (R) is copyright of the Mathworks
+
+Copyright (c) 2014 Bastian Bechtold
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the
+   distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived
+   from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+

matlab/+msgpack/dumpmsgpack.m

@@ -0,0 +1,226 @@
+%DUMPMSGPACK dumps Matlab data structures as a msgpack data
+% DUMPMSGPACK(DATA)
+%    recursively walks through DATA and creates a msgpack byte buffer from it.
+%    - strings are converted to strings
+%    - scalars are converted to numbers
+%    - logicals are converted to `true` and `false`
+%    - arrays are converted to arrays of numbers
+%    - matrices are converted to arrays of arrays of numbers
+%    - empty matrices are converted to nil
+%    - cell arrays are converted to arrays
+%    - cell matrices are converted to arrays of arrays
+%    - struct arrays are converted to arrays of maps
+%    - structs and container.Maps are converted to maps
+%    - function handles and matlab objects will raise an error.
+%
+%    There is no way of encoding bins or exts
+
+% (c) 2016 Bastian Bechtold
+% This code is licensed under the BSD 3-clause license
+
+function msgpack = dumpmsgpack(data)
+    msgpack = dump(data);
+    % collect all parts in a cell array to avoid frequent uint8
+    % concatenations.
+    msgpack = [msgpack{:}];
+end
+
+function msgpack = dump(data)
+    % convert numeric matrices to cell matrices since msgpack doesn't know matrices
+    if (isnumeric(data) || islogical(data)) && ...
+       ~(isvector(data) && isa(data, 'uint8')) && ~isscalar(data) && ~isempty(data)
+        data = num2cell(data);
+    end
+    % convert character matrices to cell of strings or cell matrices
+    if ischar(data) && ~(isvector(data)||isempty(data)) && ndims(data) == 2
+        data = cellstr(data);
+    elseif ischar(data) && ~isvector(data)
+        data = num2cell(data);
+    end
+    % convert struct arrays to cell of structs
+    if isstruct(data) && ~isscalar(data)
+        data = num2cell(data);
+    end
+    % standardize on always using maps instead of structs
+    if isstruct(data)
+        if ~isempty(fieldnames(data))
+            data = containers.Map(fieldnames(data), struct2cell(data));
+        else
+            data = containers.Map();
+        end
+    end
+
+    if isnumeric(data) && isempty(data)
+        msgpack = {uint8(192)}; % encode nil
+    elseif isa(data, 'uint8') && numel(data) > 1
+        msgpack = dumpbin(data);
+    elseif islogical(data)
+        if data
+            msgpack = {uint8(195)}; % encode true
+        else
+            msgpack = {uint8(194)}; % encode false
+        end
+    elseif isinteger(data)
+        msgpack = {dumpinteger(data)};
+    elseif isnumeric(data)
+        msgpack = {dumpfloat(data)};
+    elseif ischar(data)
+        msgpack = dumpstring(data);
+    elseif iscell(data)
+        msgpack = dumpcell(data);
+    elseif isa(data, 'containers.Map')
+        msgpack = dumpmap(data);
+    else
+        error('transplant:dumpmsgpack:unknowntype', ...
+              ['Unknown type "' class(data) '"']);
+    end
+end
+
+function bytes = scalar2bytes(value)
+    % reverse byte order to convert from little endian to big endian
+    bytes = typecast(swapbytes(value), 'uint8');
+end
+
+function msgpack = dumpinteger(value)
+    % if the values are small enough, encode as fixnum:
+    if value >= 0 && value < 128
+        % first bit is 0, last 7 bits are value
+        msgpack = uint8(value);
+        return
+    elseif value < 0 && value > -32
+        % first three bits are 111, last 5 bytes are value
+        msgpack = typecast(int8(value), 'uint8');
+        return
+    end
+
+    % otherwise, encode by type:
+    switch class(value)
+        case 'uint8' % encode as uint8
+            msgpack = uint8([204, value]);
+        case 'uint16' % encode as uint16
+            msgpack = uint8([205, scalar2bytes(value)]);
+        case 'uint32' % encode as uint32
+            msgpack = uint8([206, scalar2bytes(value)]);
+        case 'uint64' % encode as uint64
+            msgpack = uint8([207, scalar2bytes(value)]);
+        case 'int8' % encode as int8
+            msgpack = uint8([208, scalar2bytes(value)]);
+        case 'int16' % encode as int16
+            msgpack = uint8([209, scalar2bytes(value)]);
+        case 'int32' % encode as int32
+            msgpack = uint8([210, scalar2bytes(value)]);
+        case 'int64' % encode as int64
+            msgpack = uint8([211, scalar2bytes(value)]);
+        otherwise
+            error('transplant:dumpmsgpack:unknowninteger', ...
+                  ['Unknown integer type "' class(value) '"']);
+    end
+end
+
+function msgpack = dumpfloat(value)
+    % do double first, as it is more common in Matlab
+    if isa(value, 'double') % encode as float64
+        msgpack = uint8([203, scalar2bytes(value)]);
+    elseif isa(value, 'single') % encode as float32
+        msgpack = uint8([202, scalar2bytes(value)]);
+    else
+        error('transplant:dumpmsgpack:unknownfloat', ...
+              ['Unknown float type "' class(value) '"']);
+    end
+end
+
+function msgpack = dumpstring(value)
+    b10100000 = 160;
+
+    encoded = unicode2native(value, 'utf-8');
+    len = length(encoded);
+
+    if len < 32 % encode as fixint:
+        % first three bits are 101, last 5 are length:
+        msgpack = {uint8(bitor(len, b10100000)), encoded};
+    elseif len < 256 % encode as str8
+        msgpack = {uint8([217, len]), encoded};
+    elseif len < 2^16 % encode as str16
+        msgpack = {uint8(218), scalar2bytes(uint16(len)), encoded};
+    elseif len < 2^32 % encode as str32
+        msgpack = {uint8(219), scalar2bytes(uint32(len)), encoded};
+    else
+        error('transplant:dumpmsgpack:stringtoolong', ...
+              sprintf('String is too long (%d bytes)', len));
+    end
+end
+
+function msgpack = dumpbin(value)
+    len = length(value);
+    if len < 256 % encode as bin8
+        msgpack = {uint8([196, len]) value(:)'};
+    elseif len < 2^16 % encode as bin16
+        msgpack = {uint8(197), scalar2bytes(uint16(len)), value(:)'};
+    elseif len < 2^32 % encode as bin32
+        msgpack = {uint8(198), scalar2bytes(uint32(len)), value(:)'};
+    else
+        error('transplant:dumpmsgpack:bintoolong', ...
+              sprintf('Bin is too long (%d bytes)', len));
+    end
+end
+
+function msgpack = dumpcell(value)
+    b10010000 = 144;
+
+    % Msgpack can only work with 1D-arrays. Thus, Convert a
+    % multidimensional AxBxC array into a cell-of-cell-of-cell, so
+    % that indexing value{a, b, c} becomes value{a}{b}{c}.
+    if length(value) ~= prod(size(value))
+        for n=ndims(value):-1:2
+            value = cellfun(@squeeze, num2cell(value, n), ...
+                            'uniformoutput', false);
+        end
+    end
+
+    % write header
+    len = length(value);
+    if len < 16 % encode as fixarray
+        % first four bits are 1001, last 4 are length
+        msgpack = {uint8(bitor(len, b10010000))};
+    elseif len < 2^16 % encode as array16
+        msgpack = {uint8(220), scalar2bytes(uint16(len))};
+    elseif len < 2^32 % encode as array32
+        msgpack = {uint8(221), scalar2bytes(uint32(len))};
+    else
+        error('transplant:dumpmsgpack:arraytoolong', ...
+              sprintf('Array is too long (%d elements)', len));
+    end
+
+    % write values
+    for n=1:len
+        stuff = dump(value{n});
+        msgpack = [msgpack stuff{:}];
+    end
+end
+
+function msgpack = dumpmap(value)
+    b10000000 = 128;
+
+    % write header
+    len = length(value);
+    if len < 16 % encode as fixmap
+        % first four bits are 1000, last 4 are length
+        msgpack = {uint8(bitor(len, b10000000))};
+    elseif len < 2^16 % encode as map16
+        msgpack = {uint8(222), scalar2bytes(uint16(len))};
+    elseif len < 2^32 % encode as map32
+        msgpack = {uint8(223), scalar2bytes(uint32(len))};
+    else
+        error('transplant:dumpmsgpack:maptoolong', ...
+              sprintf('Map is too long (%d elements)', len));
+    end
+
+    % write key-value pairs
+    keys = value.keys();
+    values = value.values();
+    for n=1:len
+        keystuff = dump(keys{n});
+        valuestuff = dump(values{n});
+        msgpack = [msgpack, keystuff{:}, valuestuff{:}];
+    end
+end

matlab/+msgpack/dumpmsgpack_test.m

@@ -0,0 +1,96 @@
+%% positive integer dumping
+if dumpmsgpack(int8(0)) ~= uint8(0)
+    error('Dumping 0 failed')
+end
+if any(dumpmsgpack(uint8(1)) ~= uint8(1))
+    error('Dumping positive fixnum failed')
+end
+if any(dumpmsgpack(uint8(128)) ~= uint8([204, 128]))
+    error('Dumping uint8  failed')
+end
+if any(dumpmsgpack(uint16(256)) ~= uint8([205, 1, 0]))
+    error('Dumping uint16 failed')
+end
+if any(dumpmsgpack(uint32(2^16)) ~= uint8([206, 0, 1, 0, 0]))
+    error('Dumping uint32 failed')
+end
+if any(dumpmsgpack(uint64(2^32)) ~= uint8([207, 0, 0, 0, 1, 0, 0, 0, 0]))
+    error('Dumping uint64 failed')
+end
+
+%% negative integer dumping
+if dumpmsgpack(int8(-1)) ~= uint8(255)
+    error('Dumping negative fixnum failed')
+end
+if any(dumpmsgpack(int8(-128)) ~= uint8([208, 128]))
+    error('Dumping int8  failed')
+end
+if any(dumpmsgpack(int16(-256)) ~= uint8([209, 255, 0]))
+    error('Dumping int16 failed')
+end
+if any(dumpmsgpack(int32(-2^16)) ~= uint8([210, 255, 255, 0, 0]))
+    error('Dumping int32 failed')
+end
+if any(dumpmsgpack(int64(-2^32)) ~= uint8([211, 255, 255, 255, 255, 0, 0, 0, 0]))
+    error('Dumping int64 failed')
+end
+
+%% float dumping
+if any(dumpmsgpack(single(1.5)) ~= uint8([202, 63, 192, 0, 0]))
+    error('Dumping float32 failed')
+end
+if any(dumpmsgpack(double(1.5)) ~= uint8([203, 63, 248, 0, 0, 0, 0, 0, 0]))
+    error('Dumping float64 failed')
+end
+
+%% string dumping
+if any(dumpmsgpack('foo') ~= uint8([163, 102, 111, 111]))
+    error('Dumping fixstr failed')
+end
+if any(dumpmsgpack(repmat('a', [1, 32])) ~= uint8([217, 32, ones(1, 32)*'a']))
+    error('Dumping str8 failed')
+end
+if any(dumpmsgpack(repmat('a', [1, 2^8])) ~= uint8([218, 1, 0, ones(1, 2^8)*'a']))
+    error('Dumping str16 failed')
+end
+if any(dumpmsgpack(repmat('a', [1, 2^16])) ~= uint8([219, 0, 1, 0, 0, ones(1, 2^16)*'a']))
+    error('Dumping str16 failed')
+end
+
+%% bin dumping
+if any(dumpmsgpack(repmat(uint8(42), [1, 32])) ~= uint8([196, 32, ones(1, 32)*42]))
+    error('Dumping str8 failed')
+end
+if any(dumpmsgpack(repmat(uint8(42), [1, 2^8])) ~= uint8([197, 1, 0, ones(1, 2^8)*42]))
+    error('Dumping str16 failed')
+end
+if any(dumpmsgpack(repmat(uint8(42), [1, 2^16])) ~= uint8([198, 0, 1, 0, 0, ones(1, 2^16)*42]))
+    error('Dumping str16 failed')
+end
+
+%% array dumping
+if any(dumpmsgpack({uint8(1), uint8(2)}) ~= uint8([146, 1, 2]))
+    error('Dumping fixarray failed')
+end
+if any(dumpmsgpack(num2cell(repmat(uint8(42), [1, 16]))) ~= uint8([220, 0, 16, repmat(42, [1, 16])]))
+    error('Dumping array16 failed')
+end
+% takes too long:
+% if any(dumpmsgpack(num2cell(repmat(uint8(42), [1, 2^16]))) ~= uint8([221, 0, 1, 0, 0 repmat(42, [1, 2^16])]))
+%     error('Dumping array32 failed')
+% end
+
+%% map dumping
+if any(dumpmsgpack(struct('one', uint8(1), 'two', uint8(2))) ~= uint8([130, dumpmsgpack('one'), 1, dumpmsgpack('two'), 2]))
+    error('Dumping fixmap failed')
+end
+data = struct();
+msgpack = uint8([]);
+for n=[1 10 11 12 13 14 15 16 2 3 4 5 6 7 8 9] % default struct field order
+    data.(['x' num2str(n)]) = uint8(n);
+    msgpack = [msgpack dumpmsgpack(['x' num2str(n)]) uint8(n)];
+end
+if any(dumpmsgpack(data) ~= uint8([222, 0, 16, msgpack]))
+    error('Dumping map16 failed')
+end
+% map32 takes too long

matlab/+msgpack/parsemsgpack.m

@@ -0,0 +1,194 @@
+%PARSEMSGPACK parses a msgpack byte buffer into Matlab data structures
+% PARSEMSGPACK(BYTES)
+%    reads BYTES as msgpack data, and creates Matlab data structures
+%    from it. The number of bytes consumed by the parsemsgpack call 
+%    is returned in the variable IDX.
+%    - strings are converted to strings
+%    - numbers are converted to appropriate numeric values
+%    - true, false are converted to logical 1, 0
+%    - nil is converted to []
+%    - arrays are converted to cell arrays
+%    - maps are converted to containers.Map
+
+% (c) 2016 Bastian Bechtold
+% This code is licensed under the BSD 3-clause license
+
+function [obj, idx] = parsemsgpack(bytes)
+    [obj, idx] = parse(uint8(bytes(:)), 1);
+end
+
+function [obj, idx] = parse(bytes, idx)
+    % masks:
+    b10000000 = 128;
+    b01111111 = 127;
+    b11000000 = 192;
+    b00111111 = 63;
+    b11100000 = 224;
+    b00011111 = 31;
+    b11110000 = 240;
+    b00001111 = 15;
+    % values:
+    b00000000 = 0;
+    b10010000 = 144;
+    b10100000 = 160;
+
+    currentbyte = bytes(idx);
+
+    if bitand(b10000000, currentbyte) == b00000000
+        % decode positive fixint
+        obj = int8(currentbyte);
+        idx = idx + 1;
+        return
+    elseif bitand(b11100000, currentbyte) == b11100000
+        % decode negative fixint
+        obj = typecast(currentbyte, 'int8');
+        idx = idx + 1;
+        return
+    elseif bitand(b11110000, currentbyte) == b10000000
+        % decode fixmap
+        len = double(bitand(b00001111, currentbyte));
+        [obj, idx] = parsemap(len, bytes, idx+1);
+        return
+    elseif bitand(b11110000, currentbyte) == b10010000
+        % decode fixarray
+        len = double(bitand(b00001111, currentbyte));
+        [obj, idx] = parsearray(len, bytes, idx+1);
+        return
+    elseif bitand(b11100000, currentbyte) == b10100000
+        % decode fixstr
+        len = double(bitand(b00011111, currentbyte));
+        [obj, idx] = parsestring(len, bytes, idx + 1);
+        return
+    end
+
+    switch currentbyte
+        case 192 % nil
+            obj = [];
+            idx = idx+1;
+      % case 193 % unused
+        case 194 % false
+            obj = false;
+            idx = idx+1;
+        case 195 % true
+            obj = true;
+            idx = idx+1;
+        case 196 % bin8
+            len = double(bytes(idx+1));
+            [obj, idx] = parsebytes(len, bytes, idx+2);
+        case 197 % bin16
+            len = double(bytes2scalar(bytes(idx+1:idx+2), 'uint16'));
+            [obj, idx] = parsebytes(len, bytes, idx+3);
+        case 198 % bin32
+            len = double(bytes2scalar(bytes(idx+1:idx+4), 'uint32'));
+            [obj, idx] = parsebytes(len, bytes, idx+5);
+        case 199 % ext8
+            len = double(bytes(idx+1));
+            [obj, idx] = parseext(len, bytes, idx+2);
+        case 200 % ext16
+            len = double(bytes2scalar(bytes(idx+1:idx+2), 'uint16'));
+            [obj, idx] = parseext(len, bytes, idx+3);
+        case 201 % ext32
+            len = double(bytes2scalar(bytes(idx+1:idx+4), 'uint32'));
+            [obj, idx] = parseext(len, bytes, idx+5);
+        case 202 % float32
+            obj = bytes2scalar(bytes(idx+1:idx+4), 'single');
+            idx = idx+5;
+        case 203 % float64
+            obj = bytes2scalar(bytes(idx+1:idx+8), 'double');
+            idx = idx+9;
+        case 204 % uint8
+            obj = bytes(idx+1);
+            idx = idx+2;
+        case 205 % uint16
+            obj = bytes2scalar(bytes(idx+1:idx+2), 'uint16');
+            idx = idx+3;
+        case 206 % uint32
+            obj = bytes2scalar(bytes(idx+1:idx+4), 'uint32');
+            idx = idx+5;
+        case 207 % uint64
+            obj = bytes2scalar(bytes(idx+1:idx+8), 'uint64');
+            idx = idx+9;
+        case 208 % int8
+            obj = bytes2scalar(bytes(idx+1), 'int8');
+            idx = idx+2;
+        case 209 % int16
+            obj = bytes2scalar(bytes(idx+1:idx+2), 'int16');
+            idx = idx+3;
+        case 210 % int32
+            obj = bytes2scalar(bytes(idx+1:idx+4), 'int32');
+            idx = idx+5;
+        case 211 % int64
+            obj = bytes2scalar(bytes(idx+1:idx+8), 'int64');
+            idx = idx+9;
+        case 212 % fixext1
+            [obj, idx] = parseext(1, bytes, idx+1);
+        case 213 % fixext2
+            [obj, idx] = parseext(2, bytes, idx+1);
+        case 214 % fixext4
+            [obj, idx] = parseext(4, bytes, idx+1);
+        case 215 % fixext8
+            [obj, idx] = parseext(8, bytes, idx+1);
+        case 216 % fixext16
+            [obj, idx] = parseext(16, bytes, idx+1);
+        case 217 % str8
+            len = double(bytes(idx+1));
+            [obj, idx] = parsestring(len, bytes, idx+2);
+        case 218 % str16
+            len = double(bytes2scalar(bytes(idx+1:idx+2), 'uint16'));
+            [obj, idx] = parsestring(len, bytes, idx+3);
+        case 219 % str32
+            len = double(bytes2scalar(bytes(idx+1:idx+4), 'uint32'));
+            [obj, idx] = parsestring(len, bytes, idx+5);
+        case 220 % array16
+            len = double(bytes2scalar(bytes(idx+1:idx+2), 'uint16'));
+            [obj, idx] = parsearray(len, bytes, idx+3);
+        case 221 % array32
+            len = double(bytes2scalar(bytes(idx+1:idx+4), 'uint32'));
+            [obj, idx] = parsearray(len, bytes, idx+5);
+        case 222 % map16
+            len = double(bytes2scalar(bytes(idx+1:idx+2), 'uint16'));
+            [obj, idx] = parsemap(len, bytes, idx+3);
+        case 223 % map32
+            len = double(bytes2scalar(bytes(idx+1:idx+4), 'uint32'));
+            [obj, idx] = parsemap(len, bytes, idx+5);
+        otherwise
+            error('transplant:parsemsgpack:unknowntype', ...
+                  ['Unknown type "' dec2bin(currentbyte) '"']);
+    end
+end
+
+function value = bytes2scalar(bytes, type)
+    % reverse byte order to convert from little-endian to big-endian
+    value = typecast(bytes(end:-1:1), type);
+end
+
+function [str, idx] = parsestring(len, bytes, idx)
+    str = native2unicode(bytes(idx:idx+len-1)', 'utf-8');
+    idx = idx + len;
+end
+
+function [out, idx] = parsebytes(len, bytes, idx)
+    out = bytes(idx:idx+len-1);
+    idx = idx + len;
+end
+
+function [out, idx] = parseext(len, bytes, idx)
+    out.type = bytes(idx);
+    out.data = bytes(idx+1:idx+len);
+    idx = idx + len + 1;
+end
+
+function [out, idx] = parsearray(len, bytes, idx)
+    out = cell(1, len);
+    for n=1:len
+        [out{n}, idx] = parse(bytes, idx);
+    end
+end
+
+function [out, idx] = parsemap(len, bytes, idx)
+    out = containers.Map();
+    for n=1:len
+        [key, idx] = parse(bytes, idx);
+        [out(key), idx] = parse(bytes, idx);
+    end
+end

matlab/+msgpack/parsemsgpack_test.m

@@ -0,0 +1,111 @@
+%% positive integer parsing
+if parsemsgpack(uint8(0)) ~= uint8(0)
+    error('Parsing 0 failed')
+end
+if any(parsemsgpack(uint8(1)) ~= uint8(1))
+    error('Parsing positive fixnum failed')
+end
+if any(parsemsgpack(uint8([204, 128])) ~= uint8(128))
+    error('Parsing uint8  failed')
+end
+if any(parsemsgpack(uint8([205, 1, 0])) ~= uint16(256))
+    error('Parsing uint16 failed')
+end
+if any(parsemsgpack(uint8([206, 0, 1, 0, 0])) ~= uint32(2^16))
+    error('Parsing uint32 failed')
+end
+if any(parsemsgpack(uint8([207, 0, 0, 0, 1, 0, 0, 0, 0])) ~= uint64(2^32))
+    error('Parsing uint64 failed')
+end
+
+%% negative integer parsing
+if any(parsemsgpack(uint8(255)) ~= int8(-1))
+    error('Parsing negative fixnum failed')
+end
+if any(parsemsgpack(uint8([208, 128])) ~= int8(-128))
+    error('Parsing int8 failed')
+end
+if any(parsemsgpack(uint8([209, 255, 0])) ~= int16(-256))
+    error('Parsing int16 failed')
+end
+if any(parsemsgpack(uint8([210, 255, 255, 0, 0])) ~= int32(-2^16))
+    error('Parsing int32 failed')
+end
+if any(parsemsgpack(uint8([211, 255, 255, 255, 255, 0, 0, 0, 0])) ~= int64(-2^32))
+    error('Parsing int64 failed')
+end
+
+%% float parsing
+if any(parsemsgpack(uint8([202, 63, 192, 0, 0])) ~= single(1.5))
+    error('Parsing float32 failed')
+end
+if any(parsemsgpack(uint8([203, 63, 248, 0, 0, 0, 0, 0, 0])) ~= double(1.5))
+    error('Parsing float64 failed')
+end
+
+%% string parsing
+if any(parsemsgpack(uint8([163, 102, 111, 111])) ~= 'foo')
+    error('Parsing fixstr failed')
+end
+if any(parsemsgpack(uint8([217, 32, ones(1, 32)*'a'])) ~= repmat('a', [1, 32]))
+    error('Parsing str8 failed')
+end
+if any(parsemsgpack(uint8([218, 1, 0, ones(1, 2^8)*'a'])) ~= repmat('a', [1, 2^8]))
+    error('Parsing str16 failed')
+end
+if any(parsemsgpack(uint8([219, 0, 1, 0, 0, ones(1, 2^16)*'a'])) ~= repmat('a', [1, 2^16]))
+    error('Parsing str16 failed')
+end
+
+%% bin parsing
+if any(parsemsgpack(uint8([196, 32, ones(1, 32)*42])) ~= repmat(uint8(42), [1, 32]))
+    error('Parsing str8 failed')
+end
+if any(parsemsgpack(uint8([197, 1, 0, ones(1, 2^8)*42])) ~= repmat(uint8(42), [1, 2^8]))
+    error('Parsing str16 failed')
+end
+if any(parsemsgpack(uint8([198, 0, 1, 0, 0, ones(1, 2^16)*42])) ~= repmat(uint8(42), [1, 2^16]))
+    error('Parsing str16 failed')
+end
+
+%% array parsing
+c = parsemsgpack(uint8([146, 1, 2]));
+d = {uint8(1), uint8(2)};
+for n=1:max([length(c), length(d)])
+    if c{n} ~= d{n}
+        error('Parsing fixarray failed')
+    end
+end
+c = parsemsgpack(uint8([220, 0, 16, repmat(42, [1, 16])]));
+d = num2cell(repmat(uint8(42), [1, 16]));
+for n=1:max([length(c), length(d)])
+    if c{n} ~= d{n}
+        error('Parsing array16 failed')
+    end
+end
+% array32 takes too long
+
+%% map parsing
+c = parsemsgpack(uint8([130, dumpmsgpack('one'), 1, dumpmsgpack('two'), 2]));
+d = struct('one', uint8(1), 'two', uint8(2));
+f = [fieldnames(d)' c.keys()];
+for n=1:length(f)
+    if c(f{n}) ~= d.(f{n})
+        error('Parsing fixmap failed')
+    end
+end
+data = struct();
+msgpack = uint8([222, 0, 16]);
+for n=[1 10 11 12 13 14 15 16 2 3 4 5 6 7 8 9] % default struct field order
+    data.(['x' num2str(n)]) = uint8(n);
+    msgpack = [msgpack dumpmsgpack(['x' num2str(n)]) uint8(n)];
+end
+c = parsemsgpack(msgpack);
+d = data;
+f = [fieldnames(d)' c.keys()];
+for n=1:length(f)
+    if c(f{n}) ~= d.(f{n})
+        error('Parsing map16 failed')
+    end
+end
+% map32 takes too long

matlab/@ini/ini.m

@@ -0,0 +1,244 @@
+classdef ini < handle
+  % Low-level utilities for INI files.
+  properties (Access = public)
+    File         % file name
+    IOMode       % file opened in read-only or read-write mode?
+    FileSize     % file size
+    NumSection   % maximum number of datasets in this file (over all time steps)
+    Verbosity    % verbose output?
+  end
+  %%
+  properties (Access = private)
+    % File info
+    fileID
+    fileBeg
+    fileEnd
+    ioflag
+    tarflag
+    content
+  end
+  %% ------------------------------------------------------------------------%%
+  %%                      CONSTRUCORS/DESTRUCTORS                            %%
+  %% ------------------------------------------------------------------------%%
+  methods (Access = public)
+    function obj = ini(varargin)
+      % [obj] = ini(varargin)
+      % Default contructor
+      % Input 
+      % ? verbosity         verbose output? (default: no)
+      par = inputParser;
+      addParamValue(par,'verbosity',0,@isnumeric);
+      parse(par,varargin{:});
+      obj.resetPublicProperties();
+      obj.resetPrivateProperties();
+      obj.fileID  = -1;
+      obj.tarflag = false;
+      obj.setVerbosity(par.Results.verbosity);
+    end
+    function delete(obj)
+      % obj.delete()
+      % Object destructor
+      obj.close();
+    end
+  end
+  %% ------------------------------------------------------------------------%%
+  %%                     INITIALIZATION METHODS                              %%
+  %% ------------------------------------------------------------------------%%
+  methods(Access=public)
+    function open(obj,fname)
+      % obj.open()
+      % Opens an INI file.
+      % Input
+      %   fname         file name
+      obj.File    = fname;
+      obj.IOMode  = 'read';
+      obj.ioflag  = 'r';
+      obj.tarflag = false;
+      obj.fileID  = fopen(obj.File,obj.ioflag);
+      obj.fileBeg = 0;
+      fseek(obj.fileID,0,'eof');
+      obj.fileEnd = ftell(obj.fileID);
+      obj.FileSize = obj.fileEnd-obj.fileBeg;
+      obj.parse();
+    end
+    function opentar(obj,ptr)
+      % obj.opentar(ptr)
+      % Opens a subfile from tar in read-only mode.
+      % Input
+      %   ptr             pointer: [fid,first byte,number of bytes]
+      obj.File    = 'tar-mode';                      % Set generic file name
+      obj.IOMode  = 'read';                          % abstract IO mode
+      obj.ioflag  = 'r';                             % internal IO mode
+      obj.tarflag = true;                            % reading from tar archive?
+      % Set file ID to tar file ID
+      obj.fileID = ptr(1);      
+      if obj.fileID<0
+        error('Unable to access file: %s',obj.File);
+      end
+      obj.fileBeg = ptr(2);
+      obj.fileEnd = ptr(2)+ptr(3);
+      obj.FileSize = ptr(3);
+      obj.parse();
+    end
+  end
+  methods(Access=public)
+    function close(obj)
+      % obj.close()
+      % Closes a file.
+      obj.resetPublicProperties();
+      obj.resetPrivateProperties();
+      if obj.tarflag
+        obj.tarflag = false;
+        obj.fileID = -1;
+        return;
+      end
+      if obj.fileID<0
+        return;
+      end
+      status = fclose(obj.fileID);
+      if status<0
+        warning('Error while closing: %d',status);
+        return;
+      end
+      obj.fileID  = -1;
+    end
+    function setVerbosity(obj,flag)
+      % obj.setVerbosity(flag)
+      % Sets verbosity
+      % Input 
+      %   flag              '1' -> verbose output
+      obj.Verbosity = flag;
+    end
+    function [data] = getContent(obj)
+      % [data] = obj.getContent()
+      % Get the entire content of the INI file.
+      % Output
+      %   data            entire content of file as structure
+      data = obj.content;
+    end
+    function [data] = getSection(obj,section)
+      % [data] = obj.getSection(section)
+      % Get the content of a single section of the INI file.
+      % Input
+      %   section         name of the requested section
+      % Output
+      %   data            content of one section of file as structure
+      if isfield(obj.content,section)
+        data = obj.content.(section);
+      else
+        error('Section not found: %s',name);
+      end
+    end
+    function [data] = getEntry(obj,section,entry)
+      % [data] = obj.getEntry(section,entry)
+      % Get the value of a single entry within the INI file.
+      % Input
+      %   section         name of the requested section
+      %   entry           name of the entry within requested section
+      % Output
+      %   data            value of the requested entry
+      if ~isfield(obj.content,section)
+        error('Section not found: %s',section);
+      elseif ~isfield(obj.content.(section),entry)
+        error('Entry not found: %s',entry);
+      else
+        data = obj.content.(section).(entry);
+      end
+    end
+    function [data] = listSections(obj)
+      % [data] = obj.listSections()
+      % Get a list of section names.
+      % Output
+      %   data              cell array with section names
+      data = fieldnames(obj.content);
+    end
+    function [data] = listEntries(obj,section)
+      % [data] = obj.listEntries(section)
+      % Get a list of entry names within a section..
+      % Input
+      %   section           name of the requested section
+      % Output
+      %   data              cell array with section names
+      data = fieldnames(obj.content.(section));
+    end
+  end
+  %% ------------------------------------------------------------------------%%
+  %%                      PRIVATE STATIC METHODS                             %%
+  %% ------------------------------------------------------------------------%%
+  methods(Access=private)
+    function parse(obj)
+      fseek(obj.fileID,obj.fileBeg,'bof');
+      % Construct a struct with content of file
+      obj.content = [];
+      curSection = '';
+      while ftell(obj.fileID)<obj.fileEnd
+        str = strtrim(fgetl(obj.fileID));
+        % Check for empty lines and comments (; or #)
+        if isempty(str)
+            continue;
+        end
+        if (str(1)==';')
+            continue;
+        end
+        if (str(1)=='#')
+            continue;
+        end
+        % Check for section
+        if (str(1)=='[') && (str(end)==']')
+          % This is a section: create a structure
+          curSection = ini.getUniqueVarname(str(2:end-1));
+          obj.content.(curSection) = [];
+        else
+          % This is not a section: create structure entry
+          [curMember,curValue] = strtok(str,'=');
+          curValue  = ini.trimValue(curValue);
+          curMember = ini.getUniqueVarname(curMember);
+          % We now have the field name and the corresponding value as strings.
+          % Convert the value to a numeric value if possible.
+          [tmp,flag] = str2num(curValue);
+          if flag
+              curValue = tmp;
+          end
+          % Add this structure to the current section
+          if ~isempty(curSection)
+              obj.content.(curSection).(curMember) = curValue;
+          else
+              obj.content.(curMember) = curValue;
+          end
+        end
+      end
+      obj.NumSection = numel(fieldnames(obj.content));
+    end
+    function resetPublicProperties(obj)
+      obj.File       = [];
+      obj.FileSize   = [];
+      obj.IOMode     = [];
+      obj.NumSection = [];
+      obj.Verbosity  = [];
+    end
+    function resetPrivateProperties(obj)
+      obj.fileBeg = [];
+      obj.fileEnd = [];
+      obj.ioflag  = [];
+      obj.content = [];
+    end
+  end
+  %% ------------------------------------------------------------------------%%
+  %%                      PRIVATE STATIC METHODS                             %%
+  %% ------------------------------------------------------------------------%%
+  methods(Access=private,Static)
+    function str = trimValue(str)
+        str = strtrim(str);
+        if strcmpi(str(1),'=')
+            str(1)=[];
+        end
+        str = strtrim(str);
+    end
+    function varname = getUniqueVarname(varname)
+        %tmp = matlab.lang.makeValidName(varname);
+        %varname = matlab.lang.makeUniqueStrings(tmp,who,namelengthmax);
+        varname = strrep(varname,'-','_');
+        varname = genvarname(varname);
+    end
+  end
+end

matlab/@ucf/ucf.m

@@ -0,0 +1,808 @@
+classdef ucf < handle
+  % Low-level utilities for UCF files.
+  properties (Access = public)
+    File         % file name
+    Type         % file type
+    Class        % file class
+    Endian       % endianess
+    CodeVersion  % version of the simulation code
+    UCFVersion   % version of the data format ("unified container format")
+    NumDataset   % maximum number of datasets in this file (over all time steps)
+    NumTimestep  % number of time steps in this file
+    FileSize     % file size
+    CreationTime % time of creation
+    IOMode       % file opened in read-only or read-write mode?
+    IORank       % rank of processor + col,row,pln
+    Verbosity    % verbose output?
+    Debug        % debug information?
+  end
+  properties (Access = private)
+    % File info
+    fileID
+    fileBeg
+    fileEnd
+    typeID
+    ioflag  
+    tarflag
+    creationTimeUnix
+    versionMajor
+    versionMinor
+    versionPatch
+    versionFile
+    % Step pointers and info
+    posStep
+    numSetPerStep
+    timeStep
+    % Write-mode
+    isFileHeaderWritten
+    isStepHeaderWritten
+    % Current step information (should be resetable by resetCurrentStep())
+    currentStep
+    currentStepPosHeader
+    currentStepPosData
+    currentStepSize
+    currentStepTime
+    currentStepNumSet
+    % Current set information (should be resetable by resetCurrentSet())
+    currentSet
+    currentSetPosHeader
+    currentSetPosData
+    currentSetSize
+    currentSetDatatype
+    currentSetDatatypeNumeric
+    currentSetSizeof
+    currentSetNumParams
+    currentSetParams
+    currentSetNumElements
+    % Constants
+    magicFile            = int64(81985529216486895);
+    magicStep            = int64(11944304052957);
+    magicSet             = int64(240217520921210);
+    nHeaderFile          = 8;
+    nHeaderStep          = 4;
+    nHeaderSet           = 4;
+    nByteHeaderFile      = 8;
+    nByteHeaderStep      = 8;
+    nByteHeaderSet       = 8;
+    nSetParamsField      = 10;
+    nSetParamsParticle   = 16;
+    factorMajor          = 1000000000;
+    factorMinor          = 1000000;
+    factorPatch          = 1000;
+    factorTypeIDClass    = 1000;
+    factorTypeIDKind     = 10;
+    typeIDmatlabField    = 1999;
+    typeIDmatlabParticle = 2999;
+    scanBuffSize         = 4096;
+  end
+  %% ------------------------------------------------------------------------%%
+  %%                      CONSTRUCORS/DESTRUCTORS                            %%
+  %% ------------------------------------------------------------------------%%
+  methods(Access=public)
+    function obj = ucf(varargin)
+      % obj = ucf(varargin)
+      % Default contructor
+      % Input 
+      % ? verbosity         verbose output? (default: false)
+      % ? debug             debug output? (default: false)
+      par = inputParser;
+      addParamValue(par,'verbosity',0,@isnumeric);
+      addParamValue(par,'debug',0,@isnumeric);
+      parse(par,varargin{:});
+      obj.resetPublicProperties();
+      obj.resetPrivateProperties();
+      obj.resetCurrentStep();
+      obj.resetCurrentSet();
+      obj.fileID  = -1;
+      obj.tarflag = false;
+      obj.setVerbosity(par.Results.verbosity);
+      obj.setDebug(par.Results.debug);
+    end
+    function delete(obj)
+      % obj.delete()
+      % Default destructor
+      obj.close();
+    end
+  end
+  %% ------------------------------------------------------------------------%%
+  %%                     INITIALIZATION METHODS                              %%
+  %% ------------------------------------------------------------------------%%
+  methods(Access=public)
+    function open(obj,file)
+      % obj.open(file)
+      % Opens a file in read-only mode
+      obj.File   = file;                            % add filename as object property
+      obj.IOMode = 'read';                          % abstract IO mode
+      obj.ioflag = 'r';                             % internal IO mode
+      obj.tarflag = false;                          % data is within a tar file?
+      % Try to open the file
+      obj.fileID = fopen(obj.File,obj.ioflag);      
+      if obj.fileID<0
+        error('Unable to open file: %s',obj.File);
+      end
+      obj.fileBeg = ftell(obj.fileID);
+      fseek(obj.fileID,0,'eof');
+      obj.fileEnd = ftell(obj.fileID);
+      fseek(obj.fileID,0,'bof');
+      obj.FileSize = obj.fileEnd-obj.fileBeg;
+      % Read the file header
+      obj.readHeaderFile()
+      % Scan through file to get the basic structure (steps/sets)
+      obj.timeStep      = zeros(1,obj.scanBuffSize,'double');
+      obj.posStep       = zeros(1,obj.scanBuffSize,'double');
+      obj.numSetPerStep = zeros(1,obj.scanBuffSize,'double');
+      istep = 0;
+      while ftell(obj.fileID)<obj.FileSize
+        obj.readHeaderStep();
+        istep = istep+1;
+        obj.timeStep(istep)      = obj.currentStepTime;
+        obj.posStep(istep)       = ftell(obj.fileID);
+        obj.numSetPerStep(istep) = obj.currentStepNumSet;
+        if obj.currentStepSize==-1
+          break;
+        else
+          fseek(obj.fileID,obj.currentStepSize,'cof');
+        end
+      end
+      nstep = istep;
+      % Truncate buffered arrays
+      if nstep>obj.scanBuffSize
+        warning('Buffer overflow detected: increase scanBuffSize.');
+      end
+      obj.timeStep      = obj.timeStep(1:nstep);
+      obj.posStep       = obj.posStep(1:nstep);
+      obj.numSetPerStep = obj.numSetPerStep(1:nstep);
+      % Set some internal variables
+      obj.NumDataset    = max(obj.numSetPerStep);
+      obj.NumTimestep   = nstep;
+      obj.isFileHeaderWritten = true;
+      obj.isStepHeaderWritten = true;
+    end
+    function edit(obj,file)
+      % obj.edit(file)
+      % Opens a file in read-write mode
+      obj.File   = file;                            % add filename as object property
+      obj.IOMode = 'edit';                          % abstract IO mode
+      obj.ioflag = 'r+';                            % internal IO mode
+      obj.tarflag = false;                          % data is within a tar file?
+      % Try to open the file
+      obj.fileID = fopen(obj.File,obj.ioflag);      
+      if obj.fileID<0
+        error('Unable to open file: %s',obj.File);
+      end
+      obj.fileBeg = ftell(obj.fileID);
+      fseek(obj.fileID,0,'eof');
+      obj.fileEnd = ftell(obj.fileID);
+      fseek(obj.fileID,0,'bof');
+      obj.FileSize = obj.fileEnd-obj.fileBeg;
+      % Read the file header
+      obj.readHeaderFile()
+      % Scan through file to get the basic structure (steps/sets)
+      obj.timeStep      = zeros(1,obj.scanBuffSize,'double');
+      obj.posStep       = zeros(1,obj.scanBuffSize,'double');
+      obj.numSetPerStep = zeros(1,obj.scanBuffSize,'double');
+      istep = 0;
+      while ftell(obj.fileID)<obj.FileSize
+        obj.readHeaderStep();
+        istep = istep+1;
+        obj.timeStep(istep)      = obj.currentStepTime;
+        obj.posStep(istep)       = ftell(obj.fileID);
+        obj.numSetPerStep(istep) = obj.currentStepNumSet;
+        if obj.currentStepSize==-1
+          break;
+        else
+          fseek(obj.fileID,obj.currentStepSize,'cof');
+        end
+      end
+      nstep = istep;
+      % Truncate buffered arrays
+      if nstep>obj.scanBuffSize
+        warning('Buffer overflow detected: increase scanBuffSize.');
+      end
+      obj.timeStep      = obj.timeStep(1:nstep);
+      obj.posStep       = obj.posStep(1:nstep);
+      obj.numSetPerStep = obj.numSetPerStep(1:nstep);
+      % Set some internal variables
+      obj.NumDataset    = max(obj.numSetPerStep);
+      obj.NumTimestep   = nstep;
+      obj.isFileHeaderWritten = true;
+      obj.isStepHeaderWritten = true;
+    end
+    function create(obj,file,varargin)
+      % obj.create(file,varargin)
+      % Creates a new file and writes file header
+      % Input
+      %   file            file name/path
+      % ? type            file type {'field' (default),'particle'}
+      % ? rank            proc rank (default: [0,0,0,0])
+      % ? endian          endianess {'n' (dafault),'a','l','s','b'}
+      par = inputParser;
+      addParamValue(par,'type','field',@ischar);
+      addParamValue(par,'rank',[0,0,0,0],@(x)(isnumeric(x)&&numel(x)==4));
+      addParamValue(par,'endian','n',@ischar);
+      parse(par,varargin{:});
+      %
+      obj.File   = file;                            % add filename as object property
+      obj.IOMode = 'create';                        % abstract IO mode
+      obj.ioflag = 'w+';                            % internal IO mode
+      obj.tarflag = false;                          % data is within a tar file?
+      obj.isFileHeaderWritten = false;
+      obj.isStepHeaderWritten = false;
+      % Try to create the file
+      obj.fileID = fopen(obj.File,obj.ioflag);      
+      if obj.fileID<0
+        error('Unable to create file: %s',obj.File);
+      end
+      obj.fileBeg = ftell(obj.fileID);
+      obj.fileEnd = ftell(obj.fileID);
+      obj.FileSize = obj.fileEnd-obj.fileBeg;
+      switch par.Results.type
+      case 'field'
+        obj.typeID = obj.typeIDmatlabField;
+        obj.Class = 'field';
+      case 'particle'
+        obj.typeID = obj.typeIDmatlabParticle;
+        obj.Class = 'particle';
+      otherwise
+        error('Unknown file type: %s',par.Results.type);
+      end
+      obj.IORank = par.Results.rank;
+      obj.Endian = par.Results.endian;
+      obj.versionMajor = 0;
+      obj.versionMinor = 0;
+      obj.versionPatch = 0;
+      obj.versionFile  = 1;
+      obj.writeHeaderFile();
+    end
+    function opentar(obj,ptr)
+      % obj.opentar(ptr)
+      % Opens a subfile from tar in read-only mode
+      obj.File    = 'tar-mode';                      % Set generic file name
+      obj.IOMode  = 'read';                          % abstract IO mode
+      obj.ioflag  = 'r';                             % internal IO mode
+      obj.tarflag = true;                            % data is within a tar file?
+      % Set file ID to tar file ID
+      obj.fileID = ptr(1);      
+      if obj.fileID<0
+        error('Unable to access file: %s',obj.File);
+      end
+      obj.fileBeg = ptr(2);
+      obj.fileEnd = ptr(2)+ptr(3);
+      obj.FileSize = ptr(3);
+      % Read the file header
+      obj.readHeaderFile()
+      % Scan through file to get the basic structure (steps/sets)
+      obj.timeStep      = zeros(1,obj.scanBuffSize,'double');
+      obj.posStep       = zeros(1,obj.scanBuffSize,'double');
+      obj.numSetPerStep = zeros(1,obj.scanBuffSize,'double');
+      istep = 0;
+      while ftell(obj.fileID)<obj.fileEnd
+        obj.readHeaderStep();
+        istep = istep+1;
+        obj.timeStep(istep)      = obj.currentStepTime;
+        obj.posStep(istep)       = ftell(obj.fileID);
+        obj.numSetPerStep(istep) = obj.currentStepNumSet;
+        if obj.currentStepSize==-1
+          break;
+        else
+          fseek(obj.fileID,obj.currentStepSize,'cof');
+        end
+      end
+      nstep = istep;
+      % Truncate buffered arrays
+      if nstep>obj.scanBuffSize
+        warning('Buffer overflow detected: increase scanBuffSize.');
+      end
+      obj.timeStep      = obj.timeStep(1:nstep);
+      obj.posStep       = obj.posStep(1:nstep);
+      obj.numSetPerStep = obj.numSetPerStep(1:nstep);
+      % Set some internal variables
+      obj.NumDataset    = max(obj.numSetPerStep);
+      obj.NumTimestep   = nstep;
+      obj.isFileHeaderWritten = true;
+      obj.isStepHeaderWritten = true;
+    end
+  end
+  %% ------------------------------------------------------------------------%%
+  %%                          PUBLIC METHODS                                 %%
+  %% ------------------------------------------------------------------------%%
+  methods(Access=public)
+    function close(obj)
+      % obj.close()
+      % Closes a file
+      obj.resetPublicProperties();
+      obj.resetPrivateProperties();
+      obj.resetCurrentStep();
+      obj.resetCurrentSet();
+      if obj.tarflag
+        obj.tarflag = false;
+        obj.fileID = -1;
+        return;
+      end
+      if obj.fileID<0
+        return;
+      end
+      status = fclose(obj.fileID);
+      if status<0
+        warning('Unable to close file (exit code: %d)',status);
+        return;
+      end
+      obj.fileID = -1;
+    end
+    function [isType] = validateType(obj,type)
+      switch type
+      case 'grid'
+        isType = (obj.typeID==0);
+      case 'procgrid'
+        isType = (obj.typeID==10);
+      case 'field'
+        isType = (floor(obj.typeID/obj.factorTypeIDClass)==1);
+      case 'particle'
+        isType = (floor(obj.typeID/obj.factorTypeIDClass)==2);
+      case 'snapshot'
+        isType = (mod(obj.typeID,obj.factorTypeIDKind)==0);
+      case 'append'
+        isType = (mod(obj.typeID,obj.factorTypeIDKind)==1);
+      case {'uvwp','fluid'}
+        isType = (obj.typeID==1000);
+      case 'scalar'
+        isType = (obj.typeID==1010);
+      case 'matlab'
+        isType = (mod(obj.typeID,obj.factorTypeIDKind)==9);
+      otherwise
+        error('Unknown type: %s',type);
+      end 
+    end
+    % ------------------------------------------------------------------------%
+    %                            Read                                         %
+    % ------------------------------------------------------------------------%
+    function [data,params] = readSet(obj,tstep,dset)
+      % [data,params] = obj.readSet(tstep,dset)
+      % Reads a raw dataset from file (no reshaping or parsing of parameters)
+      % Input
+      %   tstep             index of timestep to be read
+      %   dset              index of dataset to be read
+      % Output
+      %   data              raw data with dim(1,nelements)
+      %   params            raw parameters with dim(1,nparams)
+      fseek(obj.fileID,obj.findSet(tstep,dset),'bof');
+      obj.readHeaderSet();
+      params = obj.currentSetParams;
+      data = fread(obj.fileID,[1,obj.currentSetNumElements],...
+                [obj.currentSetDatatype,'=>',obj.currentSetDatatype]);
+    end
+    function [params] = readParameters(obj,tstep,dset)
+      % [data,params] = obj.readParameters(tstep,dset)
+      % Reads a raw set of parameters without parsing.
+      % Input
+      %   tstep             index of timestep to be read
+      %   dset              index of dataset to be read
+      % Output
+      %   params            raw parameters with dim(1,nparams)
+      fseek(obj.fileID,obj.findSet(tstep,dset),'bof');
+      if obj.Debug
+        fprintf('Skipped to position %d\n',ftell(obj.fileID));
+      end
+      obj.readHeaderSet();
+      params = obj.currentSetParams;
+    end
+    % ------------------------------------------------------------------------%
+    %                            Append                                       %
+    % ------------------------------------------------------------------------%
+    function appendStep(obj,time)
+      % [] = obj.appendStep(time)
+      % Creates a new step at the end of opened file.
+      % Input
+      %   time        simulation time
+      if ~obj.verifyWriteMode()
+        error('File must be opened in create/edit mode.');
+      end
+      %
+      obj.resetCurrentStep();
+      obj.resetCurrentSet();
+      %
+      fseek(obj.fileID,0,'eof');
+      obj.currentStep          = obj.NumTimestep+1;
+      obj.currentStepPosHeader = ftell(obj.fileID);
+      obj.currentStepSize      = 0;
+      obj.currentStepTime      = time;
+      obj.currentStepNumSet    = 0;
+      obj.writeHeaderStep();
+      obj.currentStepPosData   = ftell(obj.fileID);
+      obj.currentSet           = 0;
+      %
+      obj.NumTimestep = obj.currentStep;
+      obj.posStep(obj.currentStep)       = obj.currentStepPosData;
+      obj.numSetPerStep(obj.currentStep) = obj.currentStepNumSet;
+      obj.timeStep(obj.currentStep)      = obj.currentStepTime;
+      %
+      if obj.Verbosity
+        fprintf('Created timestep #%d with time = %f\n',obj.NumTimestep,obj.currentStepTime);
+      end
+    end
+    function appendSet(obj,data,params)
+      % [] = obj.appendSet(data,params) 
+      % Creates a new dataset at the end of opened file and writes data.
+      % Input
+      %   data          data to append
+      %   parameters    dataset header parameters
+      if ~obj.verifyWriteMode()
+        error('File must be opened in create/edit mode.');
+      end
+      % Evaluate datatype
+      obj.currentSetDatatype = class(data);
+      switch obj.currentSetDatatype
+      case 'int32';  obj.currentSetDatatypeNumeric = 11; obj.currentSetSizeof = 4;
+      case 'int64';  obj.currentSetDatatypeNumeric = 12; obj.currentSetSizeof = 8;
+      case 'single'; obj.currentSetDatatypeNumeric = 21; obj.currentSetSizeof = 4;
+      case 'double'; obj.currentSetDatatypeNumeric = 22; obj.currentSetSizeof = 8;
+      otherwise
+        error('Datatype not implemented: %s',obj.currentSetDatatype);
+      end
+      % Append a new set header
+      fseek(obj.fileID,0,'eof');
+      obj.currentSet          = obj.numSetPerStep(obj.currentStep)+1;
+      obj.currentSetPosHeader = ftell(obj.fileID);
+      obj.currentSetSize      = numel(data)*obj.currentSetSizeof;
+      switch obj.Class
+        case 'field'; obj.currentSetNumParams = obj.nSetParamsField;
+        case 'particle'; obj.currentSetNumParams = obj.nSetParamsParticle;
+        otherwise; error('Invalid file class: %s',obj.Class);
+      end
+      obj.currentSetParams = zeros(1,obj.currentSetNumParams,'int64');
+      obj.currentSetParams(1:numel(params)) = params;
+      obj.writeHeaderSet();
+      % Append actual data
+      obj.currentSetPosData   = ftell(obj.fileID);
+      fwrite(obj.fileID,data(:),obj.currentSetDatatype,0,obj.Endian);
+      % Rewrite step header
+      obj.currentStepNumSet   = obj.currentStepNumSet+1;
+      obj.currentStepSize     = obj.currentStepSize+...
+                              (4+obj.currentSetNumParams)*obj.nByteHeaderStep+...
+                              obj.currentSetSize;
+      obj.writeHeaderStep();
+      % Update internal variables
+      obj.numSetPerStep(obj.currentStep) = obj.currentSet;
+      obj.NumDataset = max(obj.currentSet,obj.NumDataset);
+      % Verbose output
+      if obj.Verbosity
+        fprintf('Created dataset %d at timestep #%d\n',obj.currentSet,obj.currentStep);
+      end
+    end
+    function exchangeSet(obj,istep,iset,data)
+      % [] = obj.exchangeSet(istep,iset,data)
+      % Exchanges the data stored in dataset (istep,iset) with the data provided.
+      % Input
+      %   data          data, which replaces dataset (must be same size and type)
+      if ~obj.verifyWriteMode()
+        error('File must be opened in create/edit mode.');
+      end
+      % Evaluate datatype and datasize
+      dtype = class(data);
+      switch dtype
+      case 'int32';  dtypeNumeric = 11; dtypeSizeof = 4;
+      case 'int64';  dtypeNumeric = 12; dtypeSizeof = 8;
+      case 'single'; dtypeNumeric = 21; dtypeSizeof = 4;
+      case 'double'; dtypeNumeric = 22; dtypeSizeof = 8;
+      otherwise
+        error('Datatype not implemented: %s',dtype);
+      end
+      dsize = numel(data)*dtypeSizeof;
+      % Navigate to correct dataset
+      fseek(obj.fileID,obj.findSet(istep,iset),'bof');
+      obj.readHeaderSet();
+      % Check if datatype/datasize match
+      if dtypeNumeric~=obj.currentSetDatatypeNumeric
+        error('Datatype mismatch: %s,%s',dtype,obj.currentSetDatatype)
+      end
+      if dsize~=obj.currentSetSize
+        error('Datasize mismatch: %d,%d',dsize,obj.currentSetSize)
+      end
+      % Write data
+      fwrite(obj.fileID,data(:),obj.currentSetDatatype,0,obj.Endian);
+      % Verbose output
+      if obj.Verbosity
+        fprintf('Edited dataset %d at timestep #%d\n',obj.currentSet,obj.currentStep);
+      end
+    end
+    % ------------------------------------------------------------------------%
+    %                           Setter                                        %
+    % ------------------------------------------------------------------------%
+    function setVerbosity(obj,flag)
+      % obj.setVerbosity(flag)
+      % Sets verbosity
+      % Input
+      %   flag              '1' -> verbose output
+      obj.Verbosity = flag;
+    end
+    function setDebug(obj,flag)
+      % obj.setDebug(flag)
+      % Sets debug flag
+      % Input
+      %   flag              '1' -> debug output
+      obj.Debug = flag;
+    end
+    % ------------------------------------------------------------------------%
+    %                           Getter                                        %
+    % ------------------------------------------------------------------------%
+    function [nstep] = getNumTimesteps(obj)
+      % [nstep] = obj.getNumTimesteps()
+      % Gets total number of time steps.
+      nstep = obj.NumTimestep;
+    end
+    function [stime] = getSimulationTime(obj,varargin)
+      % [stime] = obj.getSimulationTime(varargin)
+      % Gets simulation time for one/all time steps
+      % Input
+      % ? istep           step index (default: -1, all steps)
+      par = inputParser;
+      addParamValue(par,'istep',-1,@isnumeric);
+      parse(par,varargin{:});
+      if par.Results.istep==-1
+        stime = obj.timeStep;
+      else
+        stime = obj.timeStep(par.Results.istep);
+      end
+    end
+    function [ndset] = getNumDatasets(obj,istep)
+      % [ndset] = obj.getNumDatasets(istep)
+      % Gets number datasets for a given time step.
+      % Input
+      %   istep           step index
+      ndset = obj.numSetPerStep(istep);
+    end
+  end
+  %% ------------------------------------------------------------------------%%
+  %%                          PRIVATE METHODS                                %%
+  %% ------------------------------------------------------------------------%%
+  methods (Access = private)
+    function readHeaderFile(obj)
+      % Skip to beginning of file
+      fseek(obj.fileID,obj.fileBeg,'bof');
+      % Determine endianess
+      mfmt = 'aslb';
+      for fmt=mfmt
+        currentMagic = fread(obj.fileID,1,'int64=>int64',0,fmt);
+        fseek(obj.fileID,obj.fileBeg,'bof');
+        if currentMagic==obj.magicFile
+          break;
+        end
+      end
+      if currentMagic~=obj.magicFile
+         error('Magic mismatch: invalid file header. %d',currentMagic);
+      end
+      obj.Endian = fmt;
+      % Read header
+      header = fread(obj.fileID,[1,obj.nHeaderFile],'int64=>int64',0,obj.Endian);
+      if obj.Debug
+        fprintf('Read the following file header at %d bytes\n',0);
+        fprintf('%d,%d,%d,%d,%d,%d,%d,%d\n',header);
+      end
+      % Parse version
+      obj.versionMajor = floor(double(header(2))/obj.factorMajor);
+      obj.versionMinor = floor(mod(double(header(2)),obj.factorMajor)/obj.factorMinor);
+      obj.versionPatch = floor(mod(double(header(2)),obj.factorMinor)/obj.factorPatch);
+      obj.CodeVersion  = sprintf('%d.%d.%d',obj.versionMajor,obj.versionMinor,obj.versionPatch);
+      obj.UCFVersion   = mod(double(header(2)),obj.factorPatch);
+      % Parse time stamp (UTC)
+      obj.creationTimeUnix = double(header(3));
+      obj.CreationTime = datestr(obj.creationTimeUnix/86400 + datenum(1970,1,1));
+      % Parse file type
+      obj.typeID = double(header(4));
+      switch obj.typeID
+        case 0000; obj.Type = 'grid';
+        case 0010; obj.Type = 'processor grid';
+        case 1000; obj.Type = 'fluid snapshot';
+        case 1010; obj.Type = 'scalar snapshot';
+        case 1999; obj.Type = 'matlab field data';
+        case 2000; obj.Type = 'particle snapshot';
+        case 2001; obj.Type = 'particle append';
+        case 2011; obj.Type = 'particle lagrange';
+        case 2021; obj.Type = 'particle balancing';
+        case 2999; obj.Type = 'matlab particle data';
+        case 3000; obj.Type = 'statistics fluid';
+        case 3010; obj.Type = 'statistics fluid pure';
+        case 3020; obj.Type = 'statistics scalar';
+        otherwise; obj.Type = 'unknown';
+      end
+      % Parse file class
+      switch floor(obj.typeID/obj.factorTypeIDClass)
+        case    1; obj.Class = 'field';
+        case    2; obj.Class = 'particle';
+        case    3; obj.Class = 'statistics';
+        otherwise; obj.Class = 'unknown';
+      end
+      % Parse IO rank
+      obj.IORank = double(header(5:8));
+    end
+    function readHeaderStep(obj)
+      % Read and parse
+      obj.currentStepPosHeader = ftell(obj.fileID);
+      header = fread(obj.fileID,[1,obj.nHeaderStep],'int64=>int64',0,obj.Endian);
+      obj.currentStepPosData = ftell(obj.fileID);
+      currentMagic = header(1);
+      obj.currentStepSize   = double(header(2));
+      obj.currentStepTime   = typecast(header(3),'double');
+      obj.currentStepNumSet = double(header(4));
+      if obj.Debug
+        fprintf('Read the following step header at %d bytes\n',obj.currentStepPosHeader);
+        fprintf('%d,%d,%f,%d\n',header(1),header(2),typecast(header(3),'double'),header(4)); 
+      end
+      % Check if magic is correct
+      if currentMagic~=obj.magicStep
+        error('Magic mismatch: invalid step header. %d',currentMagic);
+      end
+    end
+    function readHeaderSet(obj)
+      % Read and parse
+      obj.currentSetPosHeader = ftell(obj.fileID);
+      header = fread(obj.fileID,[1,obj.nHeaderSet],'int64=>int64',0,obj.Endian);
+      obj.currentSetPosData = ftell(obj.fileID);
+      currentMagic = header(1);
+      obj.currentSetSize = double(header(2));
+      obj.currentSetDatatypeNumeric = double(header(3));
+      switch obj.currentSetDatatypeNumeric
+      case 11; obj.currentSetSizeof = 4; obj.currentSetDatatype = 'int32';
+      case 12; obj.currentSetSizeof = 8; obj.currentSetDatatype = 'int64';
+      case 21; obj.currentSetSizeof = 4; obj.currentSetDatatype = 'single';
+      case 22; obj.currentSetSizeof = 8; obj.currentSetDatatype = 'double';
+      otherwise 
+        error('Unknown datatype: %d',obj.currentSetDatatypeNumeric);
+      end
+      obj.currentSetNumParams = double(header(4));
+      obj.currentSetNumElements = round(obj.currentSetSize/obj.currentSetSizeof);
+      if obj.Debug
+        fprintf('Read the following set header at %d bytes\n',obj.currentSetPosHeader);
+        fprintf('%d,%d,%d,%d\n',header(1),header(2),header(3),header(4));
+      end
+      % Check if magic is correct
+      if currentMagic~=obj.magicSet
+        error('Magic mismatch: invalid dataset header. %d',currentMagic);
+      end
+      % Read variable number of parameters
+      obj.currentSetParams = fread(obj.fileID,[1,obj.currentSetNumParams],'int64=>int64',0,obj.Endian);
+      if obj.Debug
+        fprintf('with parameters:')
+        for ii=1:obj.currentSetNumParams
+          fprintf(' %d',obj.currentSetParams(ii));
+        end
+        fprintf('\n');
+      end
+    end
+    function [flag] = verifyWriteMode(obj)
+      flag = false;
+      switch obj.IOMode
+      case {'edit','create'}
+        flag = true;
+      end
+    end
+    function writeHeaderFile(obj)
+      obj.creationTimeUnix = int64(java.lang.System.currentTimeMillis/1000); % using java for compatability before Matlab2014a
+      header = zeros(1,obj.nHeaderFile,'int64');
+      header(1) = obj.magicFile;
+      header(2) = obj.versionMajor*obj.factorMajor + ...
+                  obj.versionMinor*obj.factorMinor + ...
+                  obj.versionPatch*obj.factorPatch + ...
+                  obj.versionFile;
+      header(3) = obj.creationTimeUnix;
+      header(4) = int64(obj.typeID);
+      header(5:8) = int64(obj.IORank);
+      fseek(obj.fileID,obj.fileBeg,'bof');
+      if obj.Debug
+        fprintf('Writing the following file header at %d bytes\n',ftell(obj.fileID));
+        fprintf('%d,%d,%d,%d,%d,%d,%d,%d\n',header); 
+      end
+      fwrite(obj.fileID,header,'int64',0,obj.Endian);
+      obj.isFileHeaderWritten = true;
+    end
+    function writeHeaderStep(obj)
+      if ~obj.isFileHeaderWritten
+        error('No file header has been written yet.');
+      end
+      fseek(obj.fileID,obj.currentStepPosHeader,'bof');
+      if obj.Debug
+        fprintf('Writing the following step header at %d bytes\n',ftell(obj.fileID));
+        fprintf('%d,%d,%f,%d\n',obj.magicStep,obj.currentStepSize,obj.currentStepTime,obj.currentStepNumSet); 
+      end
+      fwrite(obj.fileID,obj.magicStep,'int64',0,obj.Endian);
+      fwrite(obj.fileID,obj.currentStepSize,'int64',0,obj.Endian);
+      fwrite(obj.fileID,obj.currentStepTime,'double',0,obj.Endian);
+      fwrite(obj.fileID,obj.currentStepNumSet,'int64',0,obj.Endian);
+      obj.isStepHeaderWritten = true;
+    end
+    function writeHeaderSet(obj)
+      if ~obj.isStepHeaderWritten
+        error('No step header has been written yet.');
+      end
+      fseek(obj.fileID,obj.currentSetPosHeader,'bof');
+      header = zeros(1,obj.nHeaderSet,'int64');
+      header(1) = obj.magicSet;
+      header(2) = obj.currentSetSize;
+      header(3) = obj.currentSetDatatypeNumeric;
+      header(4) = obj.currentSetNumParams;
+      obj.currentSetParams = obj.currentSetParams(1:obj.currentSetNumParams);
+      if obj.Debug
+        fprintf('Writing the following set header at %d bytes\n',ftell(obj.fileID));
+        fprintf('%d,%d,%d,%d\n',header);
+        fprintf('with parameters:')
+        for ii=1:obj.currentSetNumParams
+          fprintf(' %d',obj.currentSetParams(ii));
+        end
+        fprintf('\n');
+      end
+      fwrite(obj.fileID,header,'int64',0,obj.Endian);
+      fwrite(obj.fileID,obj.currentSetParams,'int64',0,obj.Endian);
+    end
+    function [posHeader] = findSet(obj,tstep,dset)
+      % Check input
+      if tstep>obj.NumTimestep
+        error('Out of bounds: timestep. %d, %d',tstep,obj.NumTimestep);
+      end
+      if dset>obj.numSetPerStep(tstep)
+        error('Out of bounds: dataset. %d, %d',dset,obj.NumDataset);
+      end
+      % Navigate to correct set
+      fseek(obj.fileID,obj.posStep(tstep),'bof');
+      for iset=1:dset-1
+        obj.readHeaderSet();
+        fseek(obj.fileID,obj.currentSetSize,'cof');
+      end
+      posHeader = ftell(obj.fileID);
+      if obj.Debug
+        fprintf('Found step #%d, set #%d at position %d\n',tstep,dset,posHeader)
+      end
+    end
+    function resetPublicProperties(obj)
+      obj.File            = [];
+      obj.Type            = [];
+      obj.Class           = [];
+      obj.Endian          = [];
+      obj.CodeVersion     = [];
+      obj.UCFVersion      = [];
+      obj.NumDataset      = [];
+      obj.NumTimestep     = [];
+      obj.FileSize        = [];
+      obj.CreationTime    = [];
+      obj.IOMode          = [];
+      obj.IORank          = [];
+      obj.Verbosity       = [];
+      obj.Debug           = [];
+    end
+    function resetPrivateProperties(obj)
+      obj.fileBeg          = [];
+      obj.fileEnd          = [];
+      obj.typeID           = [];
+      obj.ioflag           = [];
+      obj.creationTimeUnix = [];
+      obj.versionMajor     = [];
+      obj.versionMinor     = [];
+      obj.versionPatch     = [];
+      obj.versionFile      = [];
+      obj.posStep          = [];
+      obj.numSetPerStep    = [];
+      obj.timeStep         = [];
+      obj.isFileHeaderWritten = [];
+      obj.isStepHeaderWritten = [];
+    end
+    function resetCurrentStep(obj)
+      obj.currentStep           = [];
+      obj.currentStepPosHeader  = [];
+      obj.currentStepPosData    = [];
+      obj.currentStepSize       = [];
+      obj.currentStepTime       = [];
+      obj.currentStepNumSet     = [];
+    end
+    function resetCurrentSet(obj)
+      obj.currentSet                = [];
+      obj.currentSetPosHeader       = [];
+      obj.currentSetPosData         = [];
+      obj.currentSetSize            = [];
+      obj.currentSetDatatype        = [];
+      obj.currentSetDatatypeNumeric = [];
+      obj.currentSetSizeof          = [];
+      obj.currentSetNumParams       = [];
+      obj.currentSetParams          = [];
+      obj.currentSetNumElements     = [];
+    end
+  end
+end

matlab/@ucfmulti/ucfmulti.m

@@ -0,0 +1,163 @@
+classdef ucfmulti < handle
+  % High-level class to treat a directory with UCF files similarly to UCFtar files
+  properties (Access = public)
+    Directory           % directory name
+    Seq                 % sequence number
+    IOMode              % file opened in read-only or read-write mode?
+    NumberOfSubfiles    % number of subfiles
+  end
+  properties (Access = private)
+    % File info
+    fileID
+    ioflag
+    subFile
+    subFileAlias
+    subFileSize
+    scanBuffSize = 2^17;    % buffer size of scanner (max. number of files in tar) 
+  end
+  %% ------------------------------------------------------------------------%%
+  %%                      CONSTRUCORS/DESTRUCTORS                            %%
+  %% ------------------------------------------------------------------------%%
+  methods(Access=public)
+    function obj = ucfmulti()
+      % obj = ucf()
+      % Default contructor
+      obj.resetPublicProperties();
+      obj.resetPrivateProperties();
+    end
+    function delete(obj)
+      % obj.delete()
+      % Default destructor
+      obj.close();
+    end
+  end
+  %% ------------------------------------------------------------------------%%
+  %%                     INITIALIZATION METHODS                              %%
+  %% ------------------------------------------------------------------------%%
+  methods(Access=public)
+    function open(obj,directory,iseq)
+      % obj.open(file)
+      % Opens a file in read-only mode
+      tmp = what(directory);  % get absoulte path
+      obj.Directory = tmp.path;
+      obj.Seq = iseq;
+      obj.IOMode = 'read';
+      obj.ioflag = 'r';     
+      if ~exist(directory,'dir')
+        error('Unable to open directory: %s',obj.Directory);
+      end
+      obj.scanDirectory();
+    end
+  end
+  %% ------------------------------------------------------------------------%%
+  %%                         PUBLIC METHODS                                  %%
+  %% ------------------------------------------------------------------------%%
+  methods(Access=public)
+    function close(obj)
+      % obj.close()
+      % Closes a file
+      if obj.fileID<0
+        return;
+      end
+      status = fclose(obj.fileID);
+      if status<0
+        warning('Unable to close file (exit code: %d)',status);
+        return;
+      end
+      obj.resetPublicProperties();
+      obj.resetPrivateProperties();
+      obj.fileID = -1;
+    end
+    function [ptr] = pointer(obj,fname)
+      % [ptr] = obj.pointer(fname)
+      % Returns a 'pointer' to the requested file within the tar-ball
+      % which can be used to read the data without extracting.
+      % Input
+      %   fname           file name of subfile within tar-ball
+      % Output
+      %   ptr             pointer: [fid,first byte,number of bytes]
+      if obj.fileID>=0
+        fclose(obj.fileID);
+      end
+      idx = obj.findSubfile(fname);
+      filepath = sprintf('%s/%s',obj.Directory,obj.subFile{idx});
+      obj.fileID = fopen(filepath,obj.ioflag);
+      ptr = [obj.fileID,0,obj.subFileSize(idx)];
+    end
+    function [fname,fsize] = list(obj)
+      % [fname,fsize] = obj.list()
+      % Returns a list of name/size of all subfiles within the tar-ball
+      % Output
+      %   fname           cell array with filenames
+      %   fsize           array with file sizes in bytes
+      fname = obj.subFileAlias;
+      fsize = obj.subFileSize;
+    end
+    function [flag] = isSubfile(obj,fname)
+      % [flag] = obj.isSubfile(fname)
+      % Checks if a subfile exists within tar-ball.
+      % Input
+      %   fname           name of subfile
+      flag = any(ismember(obj.subFileAlias,fname));
+    end
+  end
+  %% ------------------------------------------------------------------------%%
+  %%                          PRIVATE METHODS                                %%
+  %% ------------------------------------------------------------------------%%
+  methods(Access=private)
+    function scanDirectory(obj)
+      % obj.scanArchive()
+      % Scans the directory for subfiles and stores meta-data in class variables.
+      obj.subFile     = cell(obj.scanBuffSize,1);
+      obj.subFileSize = zeros(obj.scanBuffSize,1);
+      
+      tmp = dir(obj.Directory);
+      strseq = sprintf('_%04d.',obj.Seq);
+      ii = 0;
+      for itmp=1:numel(tmp)
+        if tmp(itmp).isdir || isempty(strfind(tmp(itmp).name,strseq))
+          continue;
+        end
+        ii = ii+1;
+        obj.subFile{ii} = tmp(itmp).name;
+        obj.subFileSize(ii) = tmp(itmp).bytes;
+        obj.subFileAlias{ii} = strrep(tmp(itmp).name,strseq,'.');
+      end
+      % Truncate preallocated arrays
+      obj.NumberOfSubfiles = ii;
+      obj.subFile     = obj.subFile(1:ii);
+      obj.subFileAlias= obj.subFileAlias(1:ii);
+      obj.subFileSize = obj.subFileSize(1:ii);
+      if obj.NumberOfSubfiles>obj.scanBuffSize
+        warning('Number of subfiles exceeds scanBuffSize.');
+      end
+    end
+    function [idx] = findSubfile(obj,fname)
+      % [idx] = obj.findSubfile(fname)
+      % Get index of requested subfile
+      % Input
+      %   fname             name of subfile
+      % Output
+      %   idx               index of subfile
+      isReqFile = ismember(obj.subFileAlias,fname);
+      switch sum(isReqFile)
+      case 0;    error('File not found: %s',fname);
+      case 1;
+      otherwise; warning('More than one matching file found.');
+      end
+      idx = find(isReqFile);
+    end
+    function resetPublicProperties(obj)
+      obj.Directory        = [];
+      obj.Seq              = [];
+      obj.IOMode           = [];
+      obj.NumberOfSubfiles = [];
+    end
+    function resetPrivateProperties(obj)
+      obj.ioflag      = [];
+      obj.subFile     = [];
+      obj.subFileAlias= [];
+      obj.subFileSize = [];
+    end
+  end
+end

matlab/@ustar/ustar.m

@@ -0,0 +1,477 @@
+classdef ustar < handle
+  % Low-level utilities for UNIX standard tar files.
+  properties (Access = public)
+    File                % tar file name
+    IndexFile           % index file name
+    IOMode              % file opened in read-only or read-write mode?
+    NumberOfSubfiles    % number of subfiles
+  end
+  properties (Access = private)
+    % File info
+    fileID
+    ioflag
+    subFile
+    subFileBeg
+    subFileSize
+    % Current subfile information
+    currentFile
+    currentMode
+    currentUID
+    currentGID
+    currentFileSize
+    currentModtime
+    currentLink
+    currentLinkname
+    currentUsername
+    currentGroupname
+    currentDevMajor
+    currentDevMinor
+    currentFileBeg
+    % Constants
+    scanBuffSize = 2^17;    % buffer size of scanner (max. number of files in tar) 
+    extrBuffSize = 4194304; % buffer size of extracter
+    blockSize    = 512;     % ustar block size (do not change)
+  end
+  %% ------------------------------------------------------------------------%%
+  %%                      CONSTRUCORS/DESTRUCTORS                            %%
+  %% ------------------------------------------------------------------------%%
+  methods(Access=public)
+    function obj = ustar()
+      % obj = ucf()
+      % Default contructor
+      obj.resetPublicProperties();
+      obj.resetPrivateProperties();
+      obj.resetCurrent();
+    end
+    function delete(obj)
+      % obj.delete()
+      % Default destructor
+      obj.close();
+    end
+  end
+  %% ------------------------------------------------------------------------%%
+  %%                     INITIALIZATION METHODS                              %%
+  %% ------------------------------------------------------------------------%%
+  methods(Access=public)
+    function open(obj,file)
+      % obj.open(file)
+      % Opens a file in read-only mode
+      obj.File = file;
+      obj.IOMode = 'read';
+      obj.ioflag = 'r';
+      obj.fileID = fopen(obj.File,obj.ioflag);      
+      if obj.fileID<0
+        error('Unable to open file: %s',obj.File);
+      end
+      obj.scanArchive();
+      obj.resetCurrent();
+    end
+    function openIndexed(obj,tarfile,indexfile)
+      % obj.open(tarfile,indexfile)
+      % Opens a file in read-only mode while using available
+      % indexing data.
+      % Input
+      %   tarfile         path to TAR file
+      %   indexfile       path to index file (in json format)
+      obj.File      = tarfile;
+      obj.IndexFile = indexfile;
+      obj.IOMode    = 'read';
+      obj.ioflag    = 'r';
+      obj.fileID    = fopen(obj.File,obj.ioflag);      
+      if obj.fileID<0
+        error('Unable to open file: %s',obj.File);
+      end
+      obj.scanIndexFile();
+      obj.resetCurrent();
+    end
+  end
+  %% ------------------------------------------------------------------------%%
+  %%                         PUBLIC METHODS                                  %%
+  %% ------------------------------------------------------------------------%%
+  methods(Access=public)
+    function close(obj)
+      % obj.close()
+      % Closes a file
+      if obj.fileID<0
+        return;
+      end
+      status = fclose(obj.fileID);
+      if status<0
+        warning('Unable to close file (exit code: %d)',status);
+        return;
+      end
+      obj.resetPublicProperties();
+      obj.resetPrivateProperties();
+      obj.resetCurrent();
+      obj.fileID = -1;
+    end
+    function [ptr] = pointer(obj,fname)
+      % [ptr] = obj.pointer(fname)
+      % Returns a 'pointer' to the requested file within the tar-ball
+      % which can be used to read the data without extracting.
+      % Input
+      %   fname           file name of subfile within tar-ball
+      % Output
+      %   ptr             pointer: [fid,first byte,number of bytes]
+      idx = obj.findSubfile(fname);
+      ptr = [obj.fileID,obj.subFileBeg(idx),obj.subFileSize(idx)];
+    end
+    function [fname,fsize] = list(obj)
+      % [fname,fsize] = obj.list()
+      % Returns a list of name/size of all subfiles within the tar-ball
+      % Output
+      %   fname           cell array with filenames
+      %   fsize           array with file sizes in bytes
+      fname = obj.subFile;
+      fsize = obj.subFileSize;
+    end
+    function extract(obj,fname,varargin)
+      % obj.extract(fname)
+      % Extracts the requested subfile to a standalone file.
+      % Input
+      %   fname           name of subfile
+      % ? outfile         path of output file (default: fname)
+      par = inputParser;
+      addParamValue(par,'outfile',fname,@ischar);
+      parse(par,varargin{:});
+      outfile = par.Results.outfile;
+      
+      idx = obj.findSubfile(fname);
+      fbeg  = obj.subFileBeg(idx);
+      fsize = obj.subFileSize(idx);
+      fidw = fopen(outfile,'w');
+      fseek(obj.fileID,fbeg,'bof');
+      % Chunk the file
+      nchunk     = ceil(fsize/obj.extrBuffSize);
+      nchunkFull = floor(fsize/obj.extrBuffSize);
+      nchunkPart = nchunk-nchunkFull;
+      for ichunk=1:nchunkFull
+        buff = fread(obj.fileID,[1,obj.extrBuffSize],'*uint8');
+        fwrite(fidw,buff);
+      end
+      if nchunkPart>0
+        sizeChunkPart = mod(fsize,obj.extrBuffSize);
+        buff = fread(obj.fileID,[1,sizeChunkPart],'*uint8');
+        fwrite(fidw,buff);
+      end
+      fclose(fidw);
+    end
+    function [flag] = isSubfile(obj,fname)
+      % [flag] = obj.isSubfile(fname)
+      % Checks if a subfile exists within tar-ball.
+      % Input
+      %   fname           name of subfile
+      flag = any(ismember(obj.subFile,fname));
+    end
+    function writeIndex(obj,outfile)
+      % obj.writeIndex(outfile)
+      % Write a index file for tar archive in custom '.taridx' format
+      % The format is:
+      % nsubfile      int64
+      % [nsubfile times]
+      % subFileName      256*char
+      % subFileBeg       int64
+      % subFileSize      int64
+      % Input
+      %   outfile     name of index file to be written (with extension '.taridx')
+      fid = fopen(outfile,'wb');
+      fwrite(fid,obj.NumberOfSubfiles,'int64');
+      for ii=1:obj.NumberOfSubfiles
+        nchar = length(obj.subFile{ii});
+        subfile = blanks(256);
+        subfile(1:nchar) = obj.subFile{ii};
+        subfile(nchar+1) = 0;
+        fwrite(fid,subfile,'256*char');
+        fwrite(fid,obj.subFileBeg(ii),'int64');
+        fwrite(fid,obj.subFileSize(ii),'int64');
+      end
+      fclose(fid);
+    end
+    function [fname,foffset,fsize,nfile] = dumpIndex(obj)
+      % obj.dumpIndex()
+      % Get indexing data of tarfile
+      % Output
+      %   fname     cell array of file names
+      %   foffset   data offset within tar file
+      %   fsize     data size
+      %   nfile     number of files in archive
+      nfile   = obj.NumberOfSubfiles;
+      fname   = obj.subFile;
+      foffset = obj.subFileBeg;
+      fsize   = obj.subFileSize;
+    end
+  end
+  %% ------------------------------------------------------------------------%%
+  %%                          PRIVATE METHODS                                %%
+  %% ------------------------------------------------------------------------%%
+  methods(Access=private)
+    function scanArchive(obj)
+      % obj.scanArchive()
+      % Scans the tar-ball for subfiles and stores meta-data in class variables.
+      obj.subFile     = cell(obj.scanBuffSize,1);
+      obj.subFileBeg  = zeros(obj.scanBuffSize,1);
+      obj.subFileSize = zeros(obj.scanBuffSize,1);
+      % Jump to start of file
+      fseek(obj.fileID,0,'bof');
+      % Loop over (unknown) number of subfiles and evaluate header
+      ii = 0;
+      while ~obj.checkEOF()
+        ii = ii+1;
+        obj.readHeader(true);
+        obj.subFile{ii}     = obj.currentFile;
+        obj.subFileSize(ii) = obj.currentFileSize;
+        obj.subFileBeg(ii)  = obj.currentFileBeg;
+        nblock = ceil(obj.currentFileSize/obj.blockSize);
+        fseek(obj.fileID,nblock*obj.blockSize,'cof');
+      end
+      % Truncate preallocated arrays
+      obj.NumberOfSubfiles = ii;
+      obj.subFile     = obj.subFile(1:ii);
+      obj.subFileSize = obj.subFileSize(1:ii);
+      obj.subFileBeg  = obj.subFileBeg(1:ii);
+      if obj.NumberOfSubfiles>obj.scanBuffSize
+        warning('Number of subfiles exceeds scanBuffSize.');
+      end
+      obj.resetCurrent();
+    end
+    function scanIndexFile(obj)
+      % obj.scanIndexFile()
+      % Reads tar meta-data from index file into class variables.
+
+      % Check encoding of indexing file
+      [~,~,fileExtension] = fileparts(obj.IndexFile);
+      switch fileExtension
+      case '.simplejson'
+        % Open and read file contents (ASCII)
+        indexfileID = fopen(obj.IndexFile,'r');
+        if indexfileID<0
+          error('Unable to open file: %s',obj.IndexFile);
+        end
+        fseek(indexfileID,0,'bof');
+        jsonstr = fread(indexfileID,'char=>char')';
+        fclose(indexfileID);
+        % Parse JSON and reconstruct filenames
+        if ~isempty(which('jsonlab.loadjson')) 
+          % User function from matlab central
+          % This function is preferred, since filenames can be
+          % reconstructed safely from parsed JSON (. <=> _0x2E_)
+          json = jsonlab.loadjson(jsonstr);
+          jsonFields = fieldnames(json);
+          tarFileName = strrep(jsonFields,'_0x2E_','.');
+        elseif ~isempty(which('jsondecode')) 
+          % Built-in function
+          % Second choice only, since filename might be ambiguous
+          % if it has no extension, but contains underscore. (. => _)
+          json = jsondecode(jsonstr);
+          jsonFields = fieldnames(json);
+          idxtmp = strfind(jsonFields,'_');
+          tarFileName = jsonFields;
+          for ifield=1:length(jsonFields)
+            if ~isempty(idxtmp{ifield})
+              tarFileName{ifield}(idxtmp{ifield}(end)) = '.';
+            end
+          end
+        else % no JSON decoder available
+          error('No JSON decoder available.');
+        end
+        % Extract important fields
+        nsubfile = length(jsonFields);
+        for isub=1:nsubfile
+          tarFileOffset(isub) = json.(jsonFields{isub}).offset;
+          tarFileSize(isub)   = json.(jsonFields{isub}).size;
+        end
+      case '.msgpack'
+        % Open and read file contents (binary)
+        indexfileID = fopen(obj.IndexFile,'rb');
+        if indexfileID<0
+          error('Unable to open file: %s',obj.IndexFile);
+        end
+        fseek(indexfileID,0,'bof');
+        msgbytes = fread(indexfileID,'uint8=>uint8');
+        fclose(indexfileID);
+        % Parse msgpack
+        if ~isempty(which('msgpack.parsemsgpack')) 
+          msg = msgpack.parsemsgpack(msgbytes);
+          tarFileName   = msg.keys;
+          nsubfile      = length(tarFileName);
+          tarFileSize   = zeros(1,nsubfile);
+          tarFileOffset = zeros(1,nsubfile);
+          for isub=1:nsubfile
+            tmp = msg(tarFileName{isub});
+            tarFileOffset(isub) = double(tmp{1});
+            tarFileSize(isub)   = double(tmp{2});
+          end
+        else % no msgpack decoder available
+          error('No msgpack decoder available.');
+        end
+      case '.taridx'
+        % Open and read file contents (binary)
+        indexfileID = fopen(obj.IndexFile,'rb');
+        if indexfileID<0
+          error('Unable to open file: %s',obj.IndexFile);
+        end
+        fseek(indexfileID,0,'bof');
+        nsubfile = fread(indexfileID,1,'int64=>double');
+        tarFileName   = cell(1,nsubfile);
+        tarFileSize   = zeros(1,nsubfile);
+        tarFileOffset = zeros(1,nsubfile);
+        for isub=1:nsubfile
+          tarFileName{isub}   = deblank(fread(indexfileID,[1,256],'char=>char'));
+          tarFileOffset(isub) = fread(indexfileID,1,'int64=>double');
+          tarFileSize(isub)   = fread(indexfileID,1,'int64=>double');
+        end
+        fclose(indexfileID);
+      otherwise
+        error('Unknown file extension of index file: %s',fileExtension);
+      end
+      
+      % Order by offset, i.e. file order within tarball and assign 
+      % to class variables
+      [~,idxsort] = sort(tarFileOffset);
+      obj.subFile     = {tarFileName{idxsort}}';
+      obj.subFileBeg  = tarFileOffset(idxsort)';
+      obj.subFileSize = tarFileSize(idxsort)';
+      obj.NumberOfSubfiles = nsubfile;
+    end
+    function readHeader(obj,scanMode)
+      % obj.readHeader(scanMode)
+      % Reads header data of a subfile in tar-ball and stores information
+      % in 'current*' class-variables.
+      % Input
+      %   scanMode            when set to true, omit parts which are not needed during scan
+      header = fread(obj.fileID,[1,obj.blockSize],'char=>char');
+      % Extract header information
+      name     = header(1:100);
+      mode     = header(101:108);
+      uid      = header(109:116);
+      gid      = header(117:124);
+      fsize    = header(125:136);
+      mtime    = header(137:148);
+      chksum   = header(149:156);
+      link     = header(157);
+      linkname = header(158:257);
+      magic    = header(258:263);
+      version  = header(264:265);
+      uname    = header(266:297);
+      gname    = header(298:329);
+      devmajor = header(330:337);
+      devminor = header(338:345);
+      prefix   = header(346:500);
+      % Evaluate checksum
+      chksum1 = ustar.computeChecksum(header);
+      chksum2 = ustar.parseOctalStr(chksum);
+      if chksum1~=chksum2
+        error('Checksum mismatch! %d,%d',chksum1,chksum2);
+      end
+      % Evaluate magic
+      %if ~strcmp(ustar.parseStr(magic),'ustar')
+      if isempty(strfind(ustar.parseStr(magic),'ustar'))
+        error(' Not a UNIX standard tar file.')
+      end
+      % Parse header information
+      obj.currentFile      = ustar.parseStr([prefix,name]);
+      obj.currentFileBeg   = ftell(obj.fileID);
+      obj.currentFileSize  = ustar.parseOctalStr(fsize);
+      if ~scanMode
+        obj.currentMode      = ustar.parseStr(mode);
+        obj.currentUID       = ustar.parseOctalStr(uid);
+        obj.currentGID       = ustar.parseOctalStr(gid);
+        obj.currentModtime   = datestr(ustar.parseOctalStr(mtime)/86400+datenum(1970,1,1));
+        obj.currentLink      = ustar.parseOctalStr(link);
+        obj.currentLinkname  = ustar.parseStr(linkname);
+        obj.currentUsername  = ustar.parseStr(uname);
+        obj.currentGroupname = ustar.parseStr(gname);
+        obj.currentDevMajor  = ustar.parseOctalStr(devmajor);
+        obj.currentDevMinor  = ustar.parseOctalStr(devminor);
+      end
+    end
+    function [isEOF] = checkEOF(obj)
+      % [isEOF] = obj.checkEOF()
+      % Checks if end-of-file is reached (two blocks of binary zeros).
+      % Output
+      %   isEOF           flag which indicates end-of-file
+      isEOF = false;
+      curPosition = ftell(obj.fileID);
+      blockref = zeros(1,obj.blockSize,'int8');
+      blockcur = fread(obj.fileID,[1,obj.blockSize],'int8=>int8');
+      if feof(obj.fileID)
+        isEOF = true;
+      end
+      if isequal(blockcur,blockref)
+        blockcur = fread(obj.fileID,[1,obj.blockSize],'int8=>int8');
+        if isequal(blockcur,blockref)
+          isEOF = true;
+          return;
+        end
+      end
+      fseek(obj.fileID,curPosition,'bof');
+    end
+    function [idx] = findSubfile(obj,fname)
+      % [idx] = obj.findSubfile(fname)
+      % Get index of requested subfile
+      % Input
+      %   fname             name of subfile
+      % Output
+      %   idx               index of subfile
+      isReqFile = ismember(obj.subFile,fname);
+      switch sum(isReqFile)
+      case 0;    error('File not found: %s',fname);
+      case 1;
+      otherwise; warning('More than one matching file found.');
+      end
+      idx = find(isReqFile);
+    end
+    function resetPublicProperties(obj)
+      obj.File             = [];
+      obj.IOMode           = [];
+      obj.NumberOfSubfiles = [];
+    end
+    function resetPrivateProperties(obj)
+      obj.ioflag      = [];
+      obj.subFile     = [];
+      obj.subFileBeg  = [];
+      obj.subFileSize = [];
+    end
+    function resetCurrent(obj)
+      obj.currentFile      = [];
+      obj.currentMode      = [];
+      obj.currentUID       = [];
+      obj.currentGID       = [];
+      obj.currentFileSize  = [];
+      obj.currentModtime   = [];
+      obj.currentLink      = [];
+      obj.currentLinkname  = [];
+      obj.currentUsername  = [];
+      obj.currentGroupname = [];
+      obj.currentDevMajor  = [];
+      obj.currentDevMinor  = [];
+      obj.currentFileBeg   = [];
+    end
+  end
+  %% ------------------------------------------------------------------------%%
+  %%                      PRIVATE STATIC METHODS                             %%
+  %% ------------------------------------------------------------------------%%
+  methods(Access=private,Static)
+    function [chksum] = computeChecksum(block)
+      block(149:156) = '        '; % checksum is computed with spaces in check sum field
+      chksum = sum(block);
+    end
+    function [str] = parseStr(str)
+      charZero = cast(0,'char');
+      str = strrep(str,charZero,'');
+    end
+    function [num] = parseOctalStr(str)
+      num = ustar.oct2dec_long(str2double(ustar.parseStr(str)));
+    end
+    function [dec] = oct2dec_long(oct)
+      dec = 0;
+      ii  = 1;
+      while floor(oct/10^(ii-1))~=0
+        cbase = 8^(ii-1);
+        cfact = floor(mod(oct,10^ii)/10^(ii-1));
+        dec   = dec + cfact*cbase;
+        ii    = ii+1;
+      end
+    end
+  end
+end

matlab/colmap_from_flags.m

@@ -0,0 +1,58 @@
+function [col] = colmap_from_flags(irank,ihybrid,idem,iscal)
+  % [col] = colmap_from_flags(irank,ihybrid,idem,iscal)
+  % Create a containers.Map object with particle column order.
+  % Input
+  %   irank       rank written?
+  %   ihybrid     hybrid written?
+  %   idem        DEM written?
+  %   iscal       scalar written? (number of scalars)
+  % Output
+  %   col         column map which can be indexed by e.g. col('x')
+  
+  col = containers.Map('KeyType','char','ValueType','double');
+  ioffset = 0;
+  if irank
+    col('rank') = ioffset+1;
+    ioffset = ioffset+1;
+  end
+  if ihybrid
+    col('id') = ioffset+1;
+    col('x')  = ioffset+2;
+    col('y')  = ioffset+3;
+    col('z')  = ioffset+4;
+    col('r')  = ioffset+5;
+    col('rho')= ioffset+6;
+    col('ax') = ioffset+7;
+    col('ay') = ioffset+8;
+    col('az') = ioffset+9;
+    col('u')  = ioffset+10;
+    col('v')  = ioffset+11;
+    col('w')  = ioffset+12;
+    col('ox') = ioffset+13;
+    col('oy') = ioffset+14;
+    col('oz') = ioffset+15;
+    col('fx') = ioffset+16;
+    col('fy') = ioffset+17;
+    col('fz') = ioffset+18;
+    col('tx') = ioffset+19;
+    col('ty') = ioffset+20;
+    col('tz') = ioffset+21;
+    ioffset = ioffset+21;
+  end
+  if idem
+    col('fxc') = ioffset+1;
+    col('fyc') = ioffset+2;
+    col('fzc') = ioffset+3;
+    col('txc') = ioffset+4;
+    col('tyc') = ioffset+5;
+    col('tzc') = ioffset+6;
+    ioffset = ioffset+6;
+  end
+  if iscal
+    for ii=1:iscal
+      col(['s',sprintf('%d',ii)]) = ioffset+1;
+      col(['q',sprintf('%d',ii)]) = ioffset+2;
+      ioffset = ioffset+2;
+    end
+  end
+end

matlab/colmap_oldformat.m

@@ -0,0 +1,40 @@
+function [col] = colmap_oldformat(iscal)
+  % [col] = colmap_oldformat(iscal)
+  % Create a containers.Map object with particle column order.
+  % Input
+  %   iscal       scalar written? (number of scalars)
+  % Output
+  %   col         column map which can be indexed by e.g. col('x')
+  
+  col = containers.Map('KeyType','char','ValueType','double');
+  ioffset = 0;
+  col('x')  = ioffset+1;
+  col('y')  = ioffset+2;
+  col('z')  = ioffset+3;
+  col('r')  = ioffset+4;
+  col('rho')= ioffset+5;
+  col('ax') = ioffset+6;
+  col('ay') = ioffset+7;
+  col('az') = ioffset+8;
+  col('u')  = ioffset+9;
+  col('v')  = ioffset+10;
+  col('w')  = ioffset+11;
+  col('ox') = ioffset+12;
+  col('oy') = ioffset+13;
+  col('oz') = ioffset+14;
+  col('fx') = ioffset+15;
+  col('fy') = ioffset+16;
+  col('fz') = ioffset+17;
+  col('tx') = ioffset+18;
+  col('ty') = ioffset+19;
+  col('tz') = ioffset+20;
+  ioffset = ioffset+20;
+  if iscal
+    for ii=1:iscal
+      col(['s',sprintf('%d',ii)]) = ioffset+1;
+      col(['q',sprintf('%d',ii)]) = ioffset+2;
+      ioffset = ioffset+2;
+    end
+  end
+  col('id') = ioffset+1;
+end

matlab/convert_colmap_matlab2paraview.m

@@ -0,0 +1,41 @@
+function [col] = convert_colmap_matlab2paraview(col)
+    % [col] = convert_colmap_matlab2paraview(col)
+    % Renames fieldnames of vector quantities, so that paraview
+    % recognizes them as vectors
+    
+    colval  = cell2mat(col.values);
+    keysold = col.keys;
+    
+    keysnew = keysold;
+    keysnew = regexprep(keysnew,'^id$'     ,'ID'               );
+    keysnew = regexprep(keysnew,'^x$'      ,'Coords_0'         );
+    keysnew = regexprep(keysnew,'^y$'      ,'Coords_1'         );
+    keysnew = regexprep(keysnew,'^z$'      ,'Coords_2'         );
+    keysnew = regexprep(keysnew,'^r$'      ,'Radius'           );
+    keysnew = regexprep(keysnew,'^rho$'    ,'Density'          );
+    keysnew = regexprep(keysnew,'^ax$'     ,'AxCoords_0'       );
+    keysnew = regexprep(keysnew,'^ay$'     ,'AxCoords_1'       );
+    keysnew = regexprep(keysnew,'^az$'     ,'AxCoords_2'       );
+    keysnew = regexprep(keysnew,'^u$'      ,'Velocity_0'       );
+    keysnew = regexprep(keysnew,'^v$'      ,'Velocity_1'       );
+    keysnew = regexprep(keysnew,'^w$'      ,'Velocity_2'       ); 
+    keysnew = regexprep(keysnew,'^ox$'     ,'AxVelocity_0'     );
+    keysnew = regexprep(keysnew,'^oy$'     ,'AxVelocity_1'     );
+    keysnew = regexprep(keysnew,'^oz$'     ,'AxVelocity_2'     );
+    keysnew = regexprep(keysnew,'^fx$'     ,'Force_0'          );
+    keysnew = regexprep(keysnew,'^fy$'     ,'Force_1'          );
+    keysnew = regexprep(keysnew,'^fz$'     ,'Force_2'          );
+    keysnew = regexprep(keysnew,'^tx$'     ,'Torque_0'         );
+    keysnew = regexprep(keysnew,'^ty$'     ,'Torque_1'         );
+    keysnew = regexprep(keysnew,'^tz$'     ,'Torque_2'         );
+    keysnew = regexprep(keysnew,'^fxc$'    ,'CollisionForce_0' );
+    keysnew = regexprep(keysnew,'^fyc$'    ,'CollisionForce_1' );
+    keysnew = regexprep(keysnew,'^fzc$'    ,'CollisionForce_2' );
+    keysnew = regexprep(keysnew,'^txc$'    ,'CollisionTorque_0');
+    keysnew = regexprep(keysnew,'^tyc$'    ,'CollisionTorque_1');
+    keysnew = regexprep(keysnew,'^tzc$'    ,'CollisionTorque_2');
+    keysnew = regexprep(keysnew,'^s(?=\d*)','ScalarVal'        );
+    keysnew = regexprep(keysnew,'^q(?=\d*)','ScalarFlux'       );
+    
+    col = containers.Map(keysnew,colval);
+end

matlab/convert_fluid_hdf2ucf.m

@@ -0,0 +1,211 @@
+function [] = convert_fluid_hdf2ucf(fhdf,fucf,fproc)
+    % [] = convert_particles_hdf2ucf(fhdf,fucf)
+    % Converts a H5Part file containing particle data to a file in UCF format.
+    % Input
+    %   fhdf          file path to HDF file (input file)
+    %   fucf          file path to UCF file (output file)
+
+    % Get basename for UCF chunks
+    [fdir,fbase] = fileparts(fucf);
+    fbasepath = fullfile(fdir,fbase);
+        
+    % Loop through timesteps
+    fprintf('[HDF read]  %s\n',fhdf);
+    
+    % Read target proc grid from file
+    [ibegu,iendu,jbegu,jendu,kbegu,kendu,...
+     ibegv,iendv,jbegv,jendv,kbegv,kendv,...
+     ibegw,iendw,jbegw,jendw,kbegw,kendw,...
+     ibegp,iendp,jbegp,jendp,kbegp,kendp] = read_procgrid_ucf(fproc);
+    nxprocs = numel(ibegu);
+    nyprocs = numel(jbegu);
+    nzprocs = numel(kbegu);
+    nprocs = nxprocs*nyprocs*nzprocs;
+
+    % Open HDF
+    id_hdf = H5F.open(fhdf,'H5F_ACC_RDONLY','H5P_DEFAULT');
+    
+    % Read simulation time from HDF and create UCF timestep
+    id_dset = H5D.open(id_hdf,'Time');
+    time = H5D.read(id_dset);
+    H5D.close(id_dset);
+    
+    % Read periodicity for reconstruction of ghost cells
+    id_group = H5G.open(id_hdf,'/Domain');
+    id_dset = H5D.open(id_group,'PeriodicityX'); x_periodic = H5D.read(id_dset); H5D.close(id_dset);
+    id_dset = H5D.open(id_group,'PeriodicityY'); y_periodic = H5D.read(id_dset); H5D.close(id_dset);
+    id_dset = H5D.open(id_group,'PeriodicityZ'); z_periodic = H5D.read(id_dset); H5D.close(id_dset);
+    H5G.close(id_group);
+
+    % Read full fields from HDF
+    id_group = H5G.open(id_hdf,'/Fluid');
+    id_dset = H5D.open(id_group,'VelocityX'); u = H5D.read(id_dset); H5D.close(id_dset);
+    id_dset = H5D.open(id_group,'VelocityY'); v = H5D.read(id_dset); H5D.close(id_dset);
+    id_dset = H5D.open(id_group,'VelocityZ'); w = H5D.read(id_dset); H5D.close(id_dset);
+    id_dset = H5D.open(id_group,'Pressure');  p = H5D.read(id_dset); H5D.close(id_dset);
+    H5G.close(id_group);
+    
+    % Close HDF
+    H5F.close(id_hdf);
+    clear id_hdf    
+
+    % Determine global dimensions
+    nxu = size(u,1); nxv = size(v,1); nxw = size(w,1); nxp = size(p,1);
+    nyu = size(u,2); nyv = size(v,2); nyw = size(w,2); nyp = size(p,2);
+    nzu = size(u,3); nzv = size(v,3); nzw = size(w,3); nzp = size(p,3);
+
+    if nxp~=iendp(end) || nyp~=jendp(end) || nzp~=kendp(end)
+      error('Processor grid does not match data. %d,%d,%d <-> %d,%d,%d',nxp,nyp,nzp,iendp(end),jendp(end),kendp(end));
+    end
+
+    for ixproc=0:nxprocs-1
+      for iyproc=0:nyprocs-1
+        for izproc=0:nzprocs-1
+          [iproc] = locfun_proc_id(ixproc,iyproc,izproc,nxprocs,nyprocs,nzprocs);
+          fname = sprintf('%s.%05d',fbasepath,iproc);
+          fprintf('[UCF write] %s (out of %d)\n',fname,nprocs);
+          
+          obj = ucf(fname,'create');
+          obj.setFileHeader('field','rank',[iproc,ixproc,iyproc,izproc],'endian','a');
+          obj.appendStep(time);
+          
+          % Get processor bounadries for this file
+          ibu = ibegu(ixproc+1); ieu = iendu(ixproc+1); 
+          jbu = jbegu(iyproc+1); jeu = jendu(iyproc+1); 
+          kbu = kbegu(izproc+1); keu = kendu(izproc+1);
+          
+          ibv = ibegv(ixproc+1); iev = iendv(ixproc+1); 
+          jbv = jbegv(iyproc+1); jev = jendv(iyproc+1); 
+          kbv = kbegv(izproc+1); kev = kendv(izproc+1); 
+          
+          ibw = ibegw(ixproc+1); iew = iendw(ixproc+1); 
+          jbw = jbegw(iyproc+1); jew = jendw(iyproc+1); 
+          kbw = kbegw(izproc+1); kew = kendw(izproc+1); 
+          
+          ibp = ibegp(ixproc+1); iep = iendp(ixproc+1);
+          jbp = jbegp(iyproc+1); jep = jendp(iyproc+1);
+          kbp = kbegp(izproc+1); kep = kendp(izproc+1);
+          
+          % Determine local grid size
+          nxul = ieu-ibu+1; nxvl = iev-ibv+1; nxwl = iew-ibw+1; nxpl = iep-ibp+1;
+          nyul = jeu-jbu+1; nyvl = jev-jbv+1; nywl = jew-jbw+1; nypl = jep-jbp+1;
+          nzul = keu-kbu+1; nzvl = kev-kbv+1; nzwl = kew-kbw+1; nzpl = kep-kbp+1;
+          
+          % Reconstruct ghost cells
+          if x_periodic
+            if ibu==1;   ium1=nxu; else; ium1=ibu-1; end
+            if ibv==1;   ivm1=nxv; else; ivm1=ibv-1; end
+            if ibw==1;   iwm1=nxw; else; iwm1=ibw-1; end
+            if ibp==1;   ipm1=nxp; else; ipm1=ibp-1; end
+            if ieu==nxu; iup1=1;   else; iup1=ieu+1; end
+            if iev==nxv; ivp1=1;   else; ivp1=iev+1; end
+            if iew==nxw; iwp1=1;   else; iwp1=iew+1; end
+            if iep==nxp; ipp1=1;   else; ipp1=iep+1; end
+          else
+            if ibu==1;   ium1=1;   else; ium1=ibu-1; end
+            if ibv==1;   ivm1=1;   else; ivm1=ibv-1; end
+            if ibw==1;   iwm1=1;   else; iwm1=ibw-1; end
+            if ibp==1;   ipm1=1;   else; ipm1=ibp-1; end
+            if ieu==nxu; iup1=nxu; else; iup1=ieu+1; end
+            if iev==nxv; ivp1=nxv; else; ivp1=iev+1; end
+            if iew==nxw; iwp1=nxw; else; iwp1=iew+1; end
+            if iep==nxp; ipp1=nxp; else; ipp1=iep+1; end
+          end
+          if y_periodic
+            if jbu==1;   jum1=nyu; else; jum1=jbu-1; end
+            if jbv==1;   jvm1=nyv; else; jvm1=jbv-1; end
+            if jbw==1;   jwm1=nyw; else; jwm1=jbw-1; end
+            if jbp==1;   jpm1=nyp; else; jpm1=jbp-1; end
+            if jeu==nyu; jup1=1;   else; jup1=jeu+1; end
+            if jev==nyv; jvp1=1;   else; jvp1=jev+1; end
+            if jew==nyw; jwp1=1;   else; jwp1=jew+1; end
+            if jep==nyp; jpp1=1;   else; jpp1=jep+1; end
+          else
+            if jbu==1;   jum1=1;   else; jum1=jbu-1; end
+            if jbv==1;   jvm1=1;   else; jvm1=jbv-1; end
+            if jbw==1;   jwm1=1;   else; jwm1=jbw-1; end
+            if jbp==1;   jpm1=1;   else; jpm1=jbp-1; end
+            if jeu==nyu; jup1=nyu; else; jup1=jeu+1; end
+            if jev==nyv; jvp1=nyv; else; jvp1=jev+1; end
+            if jew==nyw; jwp1=nyw; else; jwp1=jew+1; end
+            if jep==nyp; jpp1=nyp; else; jpp1=jep+1; end
+          end
+          if z_periodic
+            if kbu==1;   kum1=nzu; else; kum1=kbu-1; end
+            if kbv==1;   kvm1=nzv; else; kvm1=kbv-1; end
+            if kbw==1;   kwm1=nzw; else; kwm1=kbw-1; end
+            if kbp==1;   kpm1=nzp; else; kpm1=kbp-1; end
+            if keu==nzu; kup1=1;   else; kup1=keu+1; end
+            if kev==nzv; kvp1=1;   else; kvp1=kev+1; end
+            if kew==nzw; kwp1=1;   else; kwp1=kew+1; end
+            if kep==nzp; kpp1=1;   else; kpp1=kep+1; end
+          else
+            if kbu==1;   kum1=1;   else; kum1=kbu-1; end
+            if kbv==1;   kvm1=1;   else; kvm1=kbv-1; end
+            if kbw==1;   kwm1=1;   else; kwm1=kbw-1; end
+            if kbp==1;   kpm1=1;   else; kpm1=kbp-1; end
+            if keu==nzu; kup1=nzu; else; kup1=keu+1; end
+            if kev==nzv; kvp1=nzv; else; kvp1=kev+1; end
+            if kew==nzw; kwp1=nzw; else; kwp1=kew+1; end
+            if kep==nzp; kpp1=nzp; else; kpp1=kep+1; end
+          end
+         
+          % Write u
+          %fprintf('%d,%d,%d,%d - %d,%d,%d,%d - %d,%d,%d,%d\n',ium1,ibu,ieu,iup1,jum1,jbu,jeu,jup1,kum1,kbu,keu,kup1);
+          %fprintf('%d,%d,%d,%d - %d,%d,%d,%d - %d,%d,%d,%d\n',ivm1,ibv,iev,ivp1,jvm1,jbv,jev,jvp1,kvm1,kbv,kev,kvp1);
+          %fprintf('%d,%d,%d,%d - %d,%d,%d,%d - %d,%d,%d,%d\n',iwm1,ibw,iew,iwp1,jwm1,jbw,jew,jwp1,kwm1,kbw,kew,kwp1);
+          %fprintf('%d,%d,%d,%d - %d,%d,%d,%d - %d,%d,%d,%d\n',ipm1,ibp,iep,ipp1,jpm1,jbp,jep,jpp1,kpm1,kbp,kep,kpp1);
+          tmp = zeros(nxul+2,nyul+2,nzul+2);
+          tmp(2:end-1,2:end-1,2:end-1) = u(ibu:ieu,jbu:jeu,kbu:keu);
+          tmp(1,:,:)   = u(ium1,[jum1,jbu:jeu,jup1],[kum1,kbu:keu,kup1]);
+          tmp(end,:,:) = u(iup1,[jum1,jbu:jeu,jup1],[kum1,kbu:keu,kup1]);
+          tmp(:,1,:)   = u([ium1,ibu:ieu,iup1],jum1,[kum1,kbu:keu,kup1]);
+          tmp(:,end,:) = u([ium1,ibu:ieu,iup1],jup1,[kum1,kbu:keu,kup1]);
+          tmp(:,:,1)   = u([ium1,ibu:ieu,iup1],[jum1,jbu:jeu,jup1],kum1);
+          tmp(:,:,end) = u([ium1,ibu:ieu,iup1],[jum1,jbu:jeu,jup1],kup1);
+          obj.appendField(tmp,1,ibu,jbu,kbu,nxu,nyu,nzu);
+          
+          % Write v
+          tmp = zeros(nxvl+2,nyvl+2,nzvl+2);
+          tmp(2:end-1,2:end-1,2:end-1) = v(ibv:iev,jbv:jev,kbv:kev);
+          tmp(1,:,:)   = v(ivm1,[jvm1,jbv:jev,jvp1],[kvm1,kbv:kev,kvp1]);
+          tmp(end,:,:) = v(ivp1,[jvm1,jbv:jev,jvp1],[kvm1,kbv:kev,kvp1]);
+          tmp(:,1,:)   = v([ivm1,ibv:iev,ivp1],jvm1,[kvm1,kbv:kev,kvp1]);
+          tmp(:,end,:) = v([ivm1,ibv:iev,ivp1],jvp1,[kvm1,kbv:kev,kvp1]);
+          tmp(:,:,1)   = v([ivm1,ibv:iev,ivp1],[jvm1,jbv:jev,jvp1],kvm1);
+          tmp(:,:,end) = v([ivm1,ibv:iev,ivp1],[jvm1,jbv:jev,jvp1],kvp1);
+          obj.appendField(tmp,1,ibv,jbv,kbv,nxv,nyv,nzv);
+          
+          % Write w
+          tmp = zeros(nxwl+2,nywl+2,nzwl+2);
+          tmp(2:end-1,2:end-1,2:end-1) = w(ibw:iew,jbw:jew,kbw:kew);
+          tmp(1,:,:)   = w(iwm1,[jwm1,jbw:jew,jwp1],[kwm1,kbw:kew,kwp1]);
+          tmp(end,:,:) = w(iwp1,[jwm1,jbw:jew,jwp1],[kwm1,kbw:kew,kwp1]);
+          tmp(:,1,:)   = w([iwm1,ibw:iew,iwp1],jwm1,[kwm1,kbw:kew,kwp1]);
+          tmp(:,end,:) = w([iwm1,ibw:iew,iwp1],jwp1,[kwm1,kbw:kew,kwp1]);
+          tmp(:,:,1)   = w([iwm1,ibw:iew,iwp1],[jwm1,jbw:jew,jwp1],kwm1);
+          tmp(:,:,end) = w([iwm1,ibw:iew,iwp1],[jwm1,jbw:jew,jwp1],kwp1);
+          obj.appendField(tmp,1,ibw,jbw,kbw,nxw,nyw,nzw);
+          
+          % Write p
+          tmp = zeros(nxpl+2,nypl+2,nzpl+2);
+          tmp(2:end-1,2:end-1,2:end-1) = p(ibp:iep,jbp:jep,kbp:kep);
+          tmp(1,:,:)   = p(ipm1,[jpm1,jbp:jep,jpp1],[kpm1,kbp:kep,kpp1]);
+          tmp(end,:,:) = p(ipp1,[jpm1,jbp:jep,jpp1],[kpm1,kbp:kep,kpp1]);
+          tmp(:,1,:)   = p([ipm1,ibp:iep,ipp1],jpm1,[kpm1,kbp:kep,kpp1]);
+          tmp(:,end,:) = p([ipm1,ibp:iep,ipp1],jpp1,[kpm1,kbp:kep,kpp1]);
+          tmp(:,:,1)   = p([ipm1,ibp:iep,ipp1],[jpm1,jbp:jep,jpp1],kpm1);
+          tmp(:,:,end) = p([ipm1,ibp:iep,ipp1],[jpm1,jbp:jep,jpp1],kpp1);
+          obj.appendField(tmp,1,ibp,jbp,kbp,nxp,nyp,nzp);
+
+          obj.close();
+          clear obj
+        end
+      end
+    end
+end
+
+function [ind] = locfun_proc_id(ii,jj,kk,nx,ny,nz)
+  % local version of 'sub2ind_zero_row'
+  ind=ii*ny*nz+jj*nz+kk;
+end

matlab/convert_particles_array2struct.m

@@ -0,0 +1,22 @@
+function [ppstruct] = convert_particles_array2struct(pp,col)
+    % [ppstruct] = convert_particles_array2struct(pp,col)
+    % Converts MATLAB particle data from 'array' format to 'struct'
+    % Input
+    %   pp            particle data in 'array' format
+    %   col           column map
+    % Output
+    %   ppstruct      particle data in 'struct' format
+    
+    % Convert map to (cell) arrays and sort by ascending array indices
+    [colvals,idx] = sort(cell2mat(col.values));
+    colkeys = col.keys;
+    colkeys = {colkeys{idx}};
+    nkeys = col.Count;
+          
+    nstep = size(pp,3);
+    for istep=nstep:-1:1
+      for ikey=1:nkeys
+        ppstruct(istep).(colkeys{ikey}) = pp(colvals(ikey),:,istep);
+      end
+    end
+end

matlab/convert_particles_array_colmap.m

@@ -0,0 +1,26 @@
+function [pp2] = convert_particles_array_colmap(pp1,col1,col2)
+    % [pp2] = convert_particles_array_colmap(pp1,col1,col2)
+    % Converts particle array data with colmap 1 to array with colmap 2
+    % Input
+    %   pp1           original particle data in 'array' format
+    %   col1          column map of pp1
+    %   col2          new column map
+    % Output
+    %   pp2           particle data with column map col2
+    ncol2 = col2.Count;
+    np = size(pp1,2);
+    nt = size(pp1,3);
+    pp2 = zeros(ncol2,np,nt);
+    
+    [colvals,idx] = sort(cell2mat(col2.values));
+    colkeys = col2.keys;
+    colkeys = {colkeys{idx}};
+    for it=1:nt
+      for icol=1:ncol2
+        key = colkeys{icol};
+        if col1.isKey(key)
+          pp2(icol,:,it) = pp1(col1(key),:,it);
+        end
+      end
+    end
+end

matlab/convert_particles_cell2struct.m

@@ -0,0 +1,22 @@
+function [ppstruct] = convert_particles_cell2struct(pp,col)
+    % [ppstruct] = convert_particles_cell2struct(pp,col)
+    % Converts MATLAB particle data from 'cell' format to 'struct'
+    % Input
+    %   pp            particle data in 'cell' format
+    %   col           column map
+    % Output
+    %   ppstruct      particle data in 'struct' format
+    
+    % Convert map to (cell) arrays and sort by ascending array indices
+    [colvals,idx] = sort(cell2mat(col.values));
+    colkeys = col.keys;
+    colkeys = {colkeys{idx}};
+    nkeys = col.Count;
+          
+    nstep = numel(pp);
+    for istep=nstep:-1:1
+      for ikey=1:nkeys
+        ppstruct(istep).(colkeys{ikey}) = pp{istep}(colvals(ikey),:);
+      end
+    end
+end

matlab/convert_particles_hdf2ucf.m

@@ -0,0 +1,87 @@
+function [] = convert_particles_hdf2ucf(fhdf,fucf)
+    % [] = convert_particles_hdf2ucf(fhdf,fucf)
+    % Converts a H5Part file containing particle data to a file in UCF format.
+    % Input
+    %   fhdf          file path to HDF file (input file)
+    %   fucf          file path to UCF file (output file)
+
+    % Check if file contains DEM data
+    try
+    	h5info(fhdf,'/Parameters/DEM');
+      isDEM = true;
+    catch
+      isDEM = false;
+    end
+    
+    % Create column maps
+    colml = ucf.partColmapFromFlags(false,true,isDEM,false);
+    colpv = convert_colmap_matlab2paraview(colml);
+
+    % Determine final length of set
+    ncol = colpv.length;
+
+    % Open HDF file ro and UCF file w+
+    id_file = H5F.open(fhdf,'H5F_ACC_RDONLY','H5P_DEFAULT');
+    obj = ucf(fucf,'create');
+    obj.setFileHeader('particle');
+
+    % Read number of steps from HDF5 file
+    id_dset = H5D.open(id_file,'/NumberOfSteps');
+    nstep = H5D.read(id_dset);
+    H5D.close(id_dset);
+
+    % Loop through timesteps
+    fprintf('[HDF read]  %s\n',fhdf);
+    fprintf('[UCF write] %s\n',fucf)
+    for istep=1:nstep
+      % Open group
+      groupname = sprintf('Step#%d',istep-1);
+      id_group = H5G.open(id_file,groupname);
+
+      % Read simulation time from HDF and create UCF timestep
+      id_att = H5A.open(id_group,'TimeValue');
+      time = H5A.read(id_att);
+      H5A.close(id_att);
+      obj.appendStep(time);
+
+      % Read number of particles from attribute
+      id_att = H5A.open(id_group,'NumberOfParticles');
+      np = H5A.read(id_att);
+      H5A.close(id_att);
+      
+      % Prepare array to store current timestep data
+      id_dset = H5D.open(id_group,'Radius');
+      id_type = H5D.get_type(id_dset);
+      type_size = H5T.get_size(id_type);
+      if type_size==4
+        pp = zeros(ncol,np,'single');
+      else
+        pp = zeros(ncol,np,'double');
+      end
+      H5T.close(id_type);
+      H5D.close(id_dset);
+
+      colval = cell2mat(colpv.values);
+      [colval,sortidx] = sort(colval);
+      colkey = colpv.keys;
+      colkey = colkey(sortidx);
+      
+      % Read data
+      for icol=1:ncol
+        id_dset = H5D.open(id_group,colkey{icol});
+        tmp = H5D.read(id_dset);
+        H5D.close(id_dset);
+        pp(colval(icol),:) = tmp;
+      end
+      
+      % Add data to UCF file
+      obj.appendParticle(pp,colml);
+      
+      % Close current timestep
+      H5G.close(id_group);
+    end
+
+    % Close files
+    H5F.close(id_file);
+    obj.close();
+end

matlab/flags_from_colmap.m

@@ -0,0 +1,22 @@
+function [irank,ihybrid,idem,iscal] = flags_from_colmap(col)
+  % [irank,ihybrid,idem,iscal] = partFlagsFromColmap(col)
+  % Extracts flags from containers.Map object
+  % Input
+  %   col         column map which can be indexed by e.g. col('x')
+  % Output
+  %   irank       rank written?
+  %   ihybrid     hybrid written?
+  %   idem        DEM written?
+  %   iscal       scalar written? (number of scalars)
+  
+  irank   = 0;
+  ihybrid = 0;
+  idem    = 0;
+  iscal   = 0;
+  if col.isKey('rank'); irank   = 1; end
+  if col.isKey('fx');   ihybrid = 1; end
+  if col.isKey('fxc');  idem    = 1; end
+  while col.isKey(sprintf('s%d',iscal+1))
+    iscal = iscal+1;
+  end
+end

matlab/generate_grid.m

@@ -0,0 +1,120 @@
+function [nxu,nyu,nzu,nxv,nyv,nzv,nxw,nyw,nzw,...
+          xu,yu,zu,xv,yv,zv,xw,yw,zw,xp,yp,zp,dx,dy,dz]=...
+            generate_grid(a,b,c,d,e,f,nxp,nyp,nzp,xperiodic,yperiodic,zperiodic)
+  % [nxu,nyu,nzu,nxv,nyv,nzv,nxw,nyw,nzw,...
+  %  xu,yu,zu,xv,yv,zv,xw,yw,zw,xp,yp,zp,dx,dy,dz]=...
+  %          generate_grid(a,b,c,d,e,f,nxp,nyp,nzp,xperiodic,yperiodic,zperiodic)
+  % Generates a staggered grid.
+  % Input
+  %   a,b,c,d,e,f       domain bounds
+  %   nxp,nyp,nzp       number of points for pressure grid
+  %   x/y/zperiodic     domain periodicity
+  
+  if xperiodic==1
+    nxu=nxp;
+    nxv=nxp;
+    nxw=nxp;
+  else
+    nxu=nxp+1;
+    nxv=nxp;
+    nxw=nxp;
+  end
+  if yperiodic==1
+    nyu=nyp;
+    nyv=nyp;
+    nyw=nyp;
+  else
+    nyu=nyp;
+    nyv=nyp+1;
+    nyw=nyp;
+  end
+  if zperiodic==1
+    nzu=nzp;
+    nzv=nzp;
+    nzw=nzp;
+  else
+    nzu=nzp;
+    nzv=nzp;
+    nzw=nzp+1;
+  end
+  %grid step:
+  if xperiodic==1
+    dx=(b-a)/(nxp);
+  else
+    dx=(b-a)/(nxp-1);
+  end
+  if yperiodic==1
+    dy=(d-c)/(nyp);
+  else
+    dy=(d-c)/(nyp-1);
+  end
+  if zperiodic==1
+    dz=(f-e)/(nzp);
+  else
+    dz=(f-e)/(nzp-1);
+  end
+  %
+  if xperiodic==1
+    xu=a+((1:nxu)-1)*dx;
+  else
+    xu=a+((1:nxu)-3/2)*dx;
+  end
+  if yperiodic==1
+    yu=c+((1:nyu)-1/2)*dy;
+  else
+    yu=c+((1:nyu)-1)*dy;
+  end
+  if zperiodic==1
+    zu=e+((1:nzu)-1/2)*dz;
+  else
+    zu=e+((1:nzu)-1)*dz;
+  end
+  %
+  if xperiodic==1
+    xv=a+((1:nxv)-1/2)*dx;
+  else
+    xv=a+((1:nxv)-1)*dx;
+  end
+  if yperiodic==1
+    yv=c+((1:nyv)-1)*dy;
+  else
+    yv=c+((1:nyv)-3/2)*dy;
+  end
+  if zperiodic==1
+    zv=e+((1:nzv)-1/2)*dz;
+  else
+    zv=e+((1:nzv)-1)*dz;
+  end
+  %
+  if xperiodic==1
+    xw=a+((1:nxw)-1/2)*dx;
+  else
+    xw=a+((1:nxw)-1)*dx;
+  end
+  if yperiodic==1
+    yw=c+((1:nyw)-1/2)*dy;
+  else
+    yw=c+((1:nyw)-1)*dy;
+  end
+  if zperiodic==1
+    zw=e+((1:nzw)-1)*dz;
+  else
+    zw=e+((1:nzw)-3/2)*dz;
+  end
+  %
+  if xperiodic==1
+    xp=a+((1:nxp)-1/2)*dx;
+  else
+    xp=a+((1:nxp)-1)*dx;
+  end
+  if yperiodic==1
+    yp=c+((1:nyp)-1/2)*dy;
+  else
+    yp=c+((1:nyp)-1)*dy;
+  end
+  if zperiodic==1
+    zp=e+((1:nzp)-1/2)*dz;
+  else
+    zp=e+((1:nzp)-1)*dz;
+  end
+end

matlab/generate_procgrid.m

@@ -0,0 +1,59 @@
+function [ibegu,iendu,jbegu,jendu,kbegu,kendu,...
+          ibegv,iendv,jbegv,jendv,kbegv,kendv,...
+          ibegw,iendw,jbegw,jendw,kbegw,kendw,...
+          ibegp,iendp,jbegp,jendp,kbegp,kendp]=generate_procgrid(...
+              nxu,nyu,nzu,nxv,nyv,nzv,nxw,nyw,nzw,...
+              nxp,nyp,nzp,nxprocs,nyprocs,nzprocs)
+  %[ibegu,iendu,jbegu,jendu,kbegu,kendu,...
+  % ibegv,iendv,jbegv,jendv,kbegv,kendv,...
+  % ibegw,iendw,jbegw,jendw,kbegw,kendw,...
+  % ibegp,iendp,jbegp,jendp,kbegp,kendp]=generate_procgrid(...
+  %     nxu,nyu,nzu,nxv,nyv,nzv,nxw,nyw,nzw,...
+  %     nxp,nyp,nzp,nxprocs,nyprocs,nzprocs)
+  % Generates a processore grid for a given grid.
+  % Input
+  %   nxu,nyu,nzu,...     staggered grids
+  %   nxprocs,...         number of processors
+  
+  chi='_';
+  % U:
+  [ibegu,iendu]=mpe_decomp1d(nxu,nxprocs,chi);
+  [jbegu,jendu]=mpe_decomp1d(nyu,nyprocs,chi);
+  [kbegu,kendu]=mpe_decomp1d(nzu,nzprocs,chi);
+  % V:
+  [ibegv,iendv]=mpe_decomp1d(nxv,nxprocs,chi);
+  [jbegv,jendv]=mpe_decomp1d(nyv,nyprocs,chi);
+  [kbegv,kendv]=mpe_decomp1d(nzv,nzprocs,chi);
+  % W:
+  [ibegw,iendw]=mpe_decomp1d(nxw,nxprocs,chi);
+  [jbegw,jendw]=mpe_decomp1d(nyw,nyprocs,chi);
+  [kbegw,kendw]=mpe_decomp1d(nzw,nzprocs,chi);
+  % P:
+  [ibegp,iendp]=mpe_decomp1d(nxp,nxprocs,chi);
+  [jbegp,jendp]=mpe_decomp1d(nyp,nyprocs,chi);
+  [kbegp,kendp]=mpe_decomp1d(nzp,nzprocs,chi);
+end
+
+function [s,e]=mpe_decomp1d(n,numprocs,chi)
+  %
+  % determines 1d decomposition (as in fortran dns code)
+  %
+  for myid=0:numprocs-1
+    nlocal  = floor(n/ numprocs);
+    s(myid+1) = myid * nlocal + 1;
+    deficit = mod(n,numprocs);
+    s(myid+1) = s(myid+1) + min(myid,deficit);
+    if myid<deficit
+      nlocal = nlocal + 1;
+    end
+    e(myid+1) = s(myid+1) + nlocal - 1;
+    if (e(myid+1)> n | myid==numprocs-1)
+      e(myid+1) = n;
+    end
+    if chi=='p'
+      %    /* finally: augment all counts by one */
+      s(myid+1) = s(myid+1) + 1;
+      e(myid+1) = e(myid+1) + 1;
+    end
+  end
+end

matlab/interp_uvwp_ucf.m

@@ -0,0 +1,292 @@
+function [] = interp_uvwp_ucf(hucf,dout,nxpn,nypn,nzpn,nxprocsn,nyprocsn,nzprocsn,varargin)
+  % [] = interp_uvwp_ucf(hucf,dout,nxpn,nypn,nzpn,nxprocsn,nyprocsn,nzprocsn,varargin)
+  % Interpolates a given flow field onto a new grid. 
+  % This function reads every chunk of the original field only once, while
+  % trying to minimize the memory footprint by writing the new chunks as
+  % soon as possible to disk. Memory for the new chunks is only allocated if
+  % they contain partial data. 
+  % Input
+  %   hucf              UCF handle (ustar,ucfmulti)
+  %   dout              output directory
+  %   nxpn,...          number of grid points of the new pressure grid
+  %   nxprocs,...       number of processors of the new processor grid
+  % ? verbosity         verbose output? (includes estimation of the current memory footprint) [default: 0]
+  % ? warnings          do some sanity checks and throw warnings? [default: 0]
+  % ? filebase          basename of the output files [default: uvwp]
+  % ? iseq              sequence number of the output files [default: 0]
+  % ? time              simulation time of the output files [default: 0.0]
+  
+  % Parse input
+  par = inputParser;
+  addParamValue(par,'verbosity',0,@isnumeric);
+  addParamValue(par,'warnings',0,@isnumeric);
+  addParamValue(par,'filebase','uvwp',@ischar);
+  addParamValue(par,'iseq',0,@isnumeric);
+  addParamValue(par,'time',0.0,@isnumeric);
+  parse(par,varargin{:});
+  flag_verb = par.Results.verbosity;
+  flag_warn = par.Results.warnings;
+  filebase  = par.Results.filebase;
+  iseq      = par.Results.iseq;
+  simtime   = par.Results.time;
+
+  % Read info from tar-archive
+  [xuo,yuo,zuo,xvo,yvo,zvo,xwo,ywo,zwo,xpo,ypo,zpo] = read_grid_ucf(hucf);
+  [params] = read_parameters_ucf(hucf);
+  nprocso = params.parallel.nprocs;
+  a = params.geometry.a;
+  b = params.geometry.b;
+  c = params.geometry.c;
+  d = params.geometry.d;
+  e = params.geometry.e;
+  f = params.geometry.f;
+  xperiodic = params.geometry.xperiodic;
+  yperiodic = params.geometry.yperiodic;
+  zperiodic = params.geometry.zperiodic;
+
+  [nxun,nyun,nzun,nxvn,nyvn,nzvn,nxwn,nywn,nzwn,...
+   xun,yun,zun,xvn,yvn,zvn,xwn,ywn,zwn,xpn,ypn,zpn,dxn,dyn,dzn]=...
+              generate_grid(a,b,c,d,e,f,nxpn,nypn,nzpn,xperiodic,yperiodic,zperiodic);
+  [ibegun,iendun,jbegun,jendun,kbegun,kendun,...
+   ibegvn,iendvn,jbegvn,jendvn,kbegvn,kendvn,...
+   ibegwn,iendwn,jbegwn,jendwn,kbegwn,kendwn,...
+   ibegpn,iendpn,jbegpn,jendpn,kbegpn,kendpn]=generate_procgrid(...
+                nxun,nyun,nzun,nxvn,nyvn,nzvn,nxwn,nywn,nzwn,...
+                nxpn,nypn,nzpn,nxprocsn,nyprocsn,nzprocsn);
+                
+  if flag_warn 
+    if (~xperiodic && mod(nxpn,2)==0) || (~yperiodic && mod(nypn,2)==0) || (~zperiodic && mod(nzpn,2)==0)
+      warning('Pressure grid should contain extra point in non-periodic directions.')
+    end
+    if abs((dxn-dyn)/dxn)>1e-8 || abs((dyn-dzn)/dyn)>1e-8
+      warning('New mesh is not equidistant: %f, %f, %f.',dxn,dyn,dzn);
+    end
+    if mod(nxpn-~xperiodic,64)~=0
+      warning('nxpn is not a multiple of 64: MGD performance might be bad.');
+    end
+    if mod(nypn-~yperiodic,64)~=0
+      warning('nypn is not a multiple of 64: MGD performance might be bad.');
+    end
+    if mod(nzpn-~xperiodic,64)~=0
+      warning('nzpn is not a multiple of 64: MGD performance might be bad.');
+    end
+  end
+
+  % Cell arrays which hold interpolation results
+  un = cell(nxprocsn,nyprocsn,nzprocsn);
+  vn = cell(nxprocsn,nyprocsn,nzprocsn);
+  wn = cell(nxprocsn,nyprocsn,nzprocsn);
+  pn = cell(nxprocsn,nyprocsn,nzprocsn);
+
+  % Cell arrays which indicate if points have been interpolated yet
+  un_ind = cell(nxprocsn,nyprocsn,nzprocsn);
+  vn_ind = cell(nxprocsn,nyprocsn,nzprocsn);
+  wn_ind = cell(nxprocsn,nyprocsn,nzprocsn);
+  pn_ind = cell(nxprocsn,nyprocsn,nzprocsn);
+
+  % Logical arrays which indicate whether memory has been allocated
+  un_init = false(nxprocsn,nyprocsn,nzprocsn);
+  vn_init = false(nxprocsn,nyprocsn,nzprocsn);
+  wn_init = false(nxprocsn,nyprocsn,nzprocsn);
+  pn_init = false(nxprocsn,nyprocsn,nzprocsn);
+
+  % Logical array which indicate whether a chunk has been finalized
+  chunk_final = false(nxprocsn,nyprocsn,nzprocsn);
+
+  imem = 0;
+  maxmem = 0;
+  for iproco=0:nprocso-1
+    if flag_verb
+      fprintf('Processing original chunk %5d ',iproco);
+    end
+    [uo,ibuo,jbuo,kbuo,nxuol,nyuol,nzuol,...
+     vo,ibvo,jbvo,kbvo,nxvol,nyvol,nzvol,...
+     wo,ibwo,jbwo,kbwo,nxwol,nywol,nzwol,...
+     po,ibpo,jbpo,kbpo,nxpol,nypol,nzpol,ighost] = ...
+              read_uvwp_chunk_ucf(hucf,'rank',iproco);
+    imem = imem+(numel(uo)+numel(vo)+numel(wo)+numel(po))*8;
+    if flag_verb
+      fprintf('(current memory footprint: %7.0f MiB)\n',imem/(1024*1024));
+    end
+    maxmem = max(imem,maxmem);
+    
+    [xuol,yuol,zuol] = grid_chunk(xuo,yuo,zuo,ibuo,jbuo,kbuo,nxuol,nyuol,nzuol,ighost);
+    [xvol,yvol,zvol] = grid_chunk(xvo,yvo,zvo,ibvo,jbvo,kbvo,nxvol,nyvol,nzvol,ighost);
+    [xwol,ywol,zwol] = grid_chunk(xwo,ywo,zwo,ibwo,jbwo,kbwo,nxwol,nywol,nzwol,ighost);
+    [xpol,ypol,zpol] = grid_chunk(xpo,ypo,zpo,ibpo,jbpo,kbpo,nxpol,nypol,nzpol,ighost);
+    
+    Fu = griddedInterpolant({xuol,yuol,zuol},uo);
+    Fv = griddedInterpolant({xvol,yvol,zvol},vo);
+    Fw = griddedInterpolant({xwol,ywol,zwol},wo);
+    Fp = griddedInterpolant({xpol,ypol,zpol},po);
+    
+    for izprocn=0:nzprocsn-1
+      for iyprocn=0:nyprocsn-1
+        for ixprocn=0:nxprocsn-1
+          if chunk_final(ixprocn+1,iyprocn+1,izprocn+1)
+            continue;
+          end
+          ibun = ibegun(ixprocn+1); nxunl = iendun(ixprocn+1)-ibegun(ixprocn+1)+1;
+          jbun = jbegun(iyprocn+1); nyunl = jendun(iyprocn+1)-jbegun(iyprocn+1)+1;
+          kbun = kbegun(izprocn+1); nzunl = kendun(izprocn+1)-kbegun(izprocn+1)+1;
+          ibvn = ibegvn(ixprocn+1); nxvnl = iendvn(ixprocn+1)-ibegvn(ixprocn+1)+1;
+          jbvn = jbegvn(iyprocn+1); nyvnl = jendvn(iyprocn+1)-jbegvn(iyprocn+1)+1;
+          kbvn = kbegvn(izprocn+1); nzvnl = kendvn(izprocn+1)-kbegvn(izprocn+1)+1;
+          ibwn = ibegwn(ixprocn+1); nxwnl = iendwn(ixprocn+1)-ibegwn(ixprocn+1)+1;
+          jbwn = jbegwn(iyprocn+1); nywnl = jendwn(iyprocn+1)-jbegwn(iyprocn+1)+1;
+          kbwn = kbegwn(izprocn+1); nzwnl = kendwn(izprocn+1)-kbegwn(izprocn+1)+1;
+          ibpn = ibegpn(ixprocn+1); nxpnl = iendpn(ixprocn+1)-ibegpn(ixprocn+1)+1;
+          jbpn = jbegpn(iyprocn+1); nypnl = jendpn(iyprocn+1)-jbegpn(iyprocn+1)+1;
+          kbpn = kbegpn(izprocn+1); nzpnl = kendpn(izprocn+1)-kbegpn(izprocn+1)+1;
+          [xunl,yunl,zunl] = grid_chunk(xun,yun,zun,ibun,jbun,kbun,nxunl,nyunl,nzunl,0);
+          [xvnl,yvnl,zvnl] = grid_chunk(xvn,yvn,zvn,ibvn,jbvn,kbvn,nxvnl,nyvnl,nzvnl,0);
+          [xwnl,ywnl,zwnl] = grid_chunk(xwn,ywn,zwn,ibwn,jbwn,kbwn,nxwnl,nywnl,nzwnl,0);
+          [xpnl,ypnl,zpnl] = grid_chunk(xpn,ypn,zpn,ibpn,jbpn,kbpn,nxpnl,nypnl,nzpnl,0);
+          
+          % u
+          ixu = find(xunl>=xuol(1) & xunl<=xuol(end));
+          iyu = find(yunl>=yuol(1) & yunl<=yuol(end));
+          izu = find(zunl>=zuol(1) & zunl<=zuol(end));
+          if ~isempty(ixu) && ~isempty(iyu) && ~isempty(izu)
+            if ~un_init(ixprocn+1,iyprocn+1,izprocn+1)
+              un{ixprocn+1,iyprocn+1,izprocn+1} = zeros(nxunl,nyunl,nzunl);
+              un_ind{ixprocn+1,iyprocn+1,izprocn+1} = false(nxunl,nyunl,nzunl);
+              un_init(ixprocn+1,iyprocn+1,izprocn+1) = true;
+              imem = imem+(nxunl*nyunl*nzunl)*(8+1);
+              maxmem = max(imem,maxmem);
+              if ~xperiodic 
+                % staggered grid goes beyond boundary in non-periodic direction:
+                % if field is coarsened, no interpolation point will be found!
+                % Thus, just leave it and mark as processed
+                un_ind{ixprocn+1,iyprocn+1,izprocn+1}(xunl<xuo(1),:,:) = true;
+                un_ind{ixprocn+1,iyprocn+1,izprocn+1}(xunl>xuo(end),:,:) = true;
+              end
+            end
+            un{ixprocn+1,iyprocn+1,izprocn+1}(ixu,iyu,izu) = Fu({xunl(ixu),yunl(iyu),zunl(izu)});
+            un_ind{ixprocn+1,iyprocn+1,izprocn+1}(ixu,iyu,izu) = true;
+          end
+          
+          % v
+          ixv = find(xvnl>=xvol(1) & xvnl<=xvol(end));
+          iyv = find(yvnl>=yvol(1) & yvnl<=yvol(end));
+          izv = find(zvnl>=zvol(1) & zvnl<=zvol(end));
+          if ~isempty(ixv) && ~isempty(iyv) && ~isempty(izv)
+            if ~vn_init(ixprocn+1,iyprocn+1,izprocn+1)
+              vn{ixprocn+1,iyprocn+1,izprocn+1} = zeros(nxvnl,nyvnl,nzvnl);
+              vn_ind{ixprocn+1,iyprocn+1,izprocn+1} = false(nxvnl,nyvnl,nzvnl);
+              vn_init(ixprocn+1,iyprocn+1,izprocn+1) = true;
+              imem = imem+(nxunl*nyunl*nzunl)*(8+1);
+              maxmem = max(imem,maxmem);
+              if ~yperiodic 
+                % staggered grid goes beyond boundary in non-periodic direction:
+                % if field is coarsened, no interpolation point will be found!
+                % Thus, just leave it and mark as processed
+                vn_ind{ixprocn+1,iyprocn+1,izprocn+1}(:,yvnl<yvo(1),:) = true;
+                vn_ind{ixprocn+1,iyprocn+1,izprocn+1}(:,yvnl>yvo(end),:) = true;
+              end
+            end
+            vn{ixprocn+1,iyprocn+1,izprocn+1}(ixv,iyv,izv) = Fv({xvnl(ixv),yvnl(iyv),zvnl(izv)});
+            vn_ind{ixprocn+1,iyprocn+1,izprocn+1}(ixv,iyv,izv) = true;
+          end
+          
+          % w
+          ixw = find(xwnl>=xwol(1) & xwnl<=xwol(end));
+          iyw = find(ywnl>=ywol(1) & ywnl<=ywol(end));
+          izw = find(zwnl>=zwol(1) & zwnl<=zwol(end));
+          if ~isempty(ixw) && ~isempty(iyw) && ~isempty(izw)
+            if ~wn_init(ixprocn+1,iyprocn+1,izprocn+1)
+              wn{ixprocn+1,iyprocn+1,izprocn+1} = zeros(nxwnl,nywnl,nzwnl);
+              wn_ind{ixprocn+1,iyprocn+1,izprocn+1} = false(nxwnl,nywnl,nzwnl);
+              wn_init(ixprocn+1,iyprocn+1,izprocn+1) = true;
+              imem = imem+(nxunl*nyunl*nzunl)*(8+1);
+              maxmem = max(imem,maxmem);
+              if ~zperiodic 
+                % staggered grid goes beyond boundary in non-periodic direction:
+                % if field is coarsened, no interpolation point will be found!
+                % Thus, just leave it and mark as processed
+                wn_ind{ixprocn+1,iyprocn+1,izprocn+1}(:,:,zwnl<zwo(1)) = true;
+                wn_ind{ixprocn+1,iyprocn+1,izprocn+1}(:,:,zwnl>zwo(end)) = true;
+              end
+            end
+            wn{ixprocn+1,iyprocn+1,izprocn+1}(ixw,iyw,izw) = Fw({xwnl(ixw),ywnl(iyw),zwnl(izw)});
+            wn_ind{ixprocn+1,iyprocn+1,izprocn+1}(ixw,iyw,izw) = true;
+          end
+          
+          % p
+          ixp = find(xpnl>=xpol(1) & xpnl<=xpol(end));
+          iyp = find(ypnl>=ypol(1) & ypnl<=ypol(end));
+          izp = find(zpnl>=zpol(1) & zpnl<=zpol(end));
+          if ~isempty(ixp) && ~isempty(iyp) && ~isempty(izp)
+            if ~pn_init(ixprocn+1,iyprocn+1,izprocn+1)
+              pn{ixprocn+1,iyprocn+1,izprocn+1} = zeros(nxpnl,nypnl,nzpnl);
+              pn_ind{ixprocn+1,iyprocn+1,izprocn+1} = false(nxpnl,nypnl,nzpnl);
+              pn_init(ixprocn+1,iyprocn+1,izprocn+1) = true;
+              imem = imem+(nxunl*nyunl*nzunl)*(8+1);
+              maxmem = max(imem,maxmem);
+            end
+            pn{ixprocn+1,iyprocn+1,izprocn+1}(ixp,iyp,izp) = Fp({xpnl(ixp),ypnl(iyp),zpnl(izp)});
+            pn_ind{ixprocn+1,iyprocn+1,izprocn+1}(ixp,iyp,izp) = true;
+          end
+          
+          if un_init(ixprocn+1,iyprocn+1,izprocn+1) && ...
+             vn_init(ixprocn+1,iyprocn+1,izprocn+1) && ...
+             wn_init(ixprocn+1,iyprocn+1,izprocn+1) && ...
+             pn_init(ixprocn+1,iyprocn+1,izprocn+1) && ...
+             all(un_ind{ixprocn+1,iyprocn+1,izprocn+1}(:)) && ...
+             all(vn_ind{ixprocn+1,iyprocn+1,izprocn+1}(:)) && ...
+             all(wn_ind{ixprocn+1,iyprocn+1,izprocn+1}(:)) && ...
+             all(pn_ind{ixprocn+1,iyprocn+1,izprocn+1}(:))
+            ichunkn = ixprocn*nyprocsn*nzprocsn+iyprocn*nzprocsn+izprocn;
+            fout = sprintf('%s/%s_%04d.%05d',dout,filebase,iseq,ichunkn);
+            write_uvwp_chunk_ucf(fout,...
+               un{ixprocn+1,iyprocn+1,izprocn+1},ibun,jbun,kbun,nxunl,nyunl,nzunl,...
+               vn{ixprocn+1,iyprocn+1,izprocn+1},ibvn,jbvn,kbvn,nxvnl,nyvnl,nzvnl,...
+               wn{ixprocn+1,iyprocn+1,izprocn+1},ibwn,jbwn,kbwn,nxwnl,nywnl,nzwnl,...
+               pn{ixprocn+1,iyprocn+1,izprocn+1},ibpn,jbpn,kbpn,nxpnl,nypnl,nzpnl,...
+                   simtime,0);
+            un{ixprocn+1,iyprocn+1,izprocn+1} = [];
+            vn{ixprocn+1,iyprocn+1,izprocn+1} = [];
+            wn{ixprocn+1,iyprocn+1,izprocn+1} = [];
+            pn{ixprocn+1,iyprocn+1,izprocn+1} = [];
+            un_ind{ixprocn+1,iyprocn+1,izprocn+1} = [];
+            vn_ind{ixprocn+1,iyprocn+1,izprocn+1} = [];
+            wn_ind{ixprocn+1,iyprocn+1,izprocn+1} = [];
+            pn_ind{ixprocn+1,iyprocn+1,izprocn+1} = [];
+            chunk_final(ixprocn+1,iyprocn+1,izprocn+1) = true;
+            imem = imem-(...
+              (nxunl*nyunl*nzunl)+...
+              (nxvnl*nyvnl*nzvnl)+...
+              (nxwnl*nywnl*nzwnl)+...
+              (nxpnl*nypnl*nzpnl) ...
+              )*(8+1);
+          end
+        end
+      end
+    end 
+    imem = imem-(numel(uo)+numel(vo)+numel(wo)+numel(po))*8;
+  end
+
+  % Check if everything has been finalized
+  for izprocn=0:nzprocsn-1
+    for iyprocn=0:nyprocsn-1
+      for ixprocn=0:nxprocsn-1
+        if ~chunk_final(ixprocn+1,iyprocn+1,izprocn+1)
+          ichunkn = ixprocn*nyprocsn*nzprocsn+iyprocn*nzprocsn+izprocn;
+          fout = sprintf('%s/%s_%04d.%05d',dout,filebase,iseq,ichunkn);
+          if flag_warn
+            warning('Chunk %2d,%2d,%2d (rank: %5d) has not been finalized! Writing anyway...',ixprocn,iyprocn,izprocn,ichunkn);
+          end
+          write_uvwp_chunk_ucf(fout,...
+             un{ixprocn+1,iyprocn+1,izprocn+1},ibun,jbun,kbun,nxunl,nyunl,nzunl,...
+             vn{ixprocn+1,iyprocn+1,izprocn+1},ibvn,jbvn,kbvn,nxvnl,nyvnl,nzvnl,...
+             wn{ixprocn+1,iyprocn+1,izprocn+1},ibwn,jbwn,kbwn,nxwnl,nywnl,nzwnl,...
+             pn{ixprocn+1,iyprocn+1,izprocn+1},ibpn,jbpn,kbpn,nxpnl,nypnl,nzpnl,...
+                 simetime,0);
+        end
+      end
+    end
+  end
+  if flag_verb
+    fprintf('Done. (maximum memory footprint: %7.0f MiB)\n',maxmem/(1024*1024));
+  end
+end

matlab/ncol_from_colmap.m

@@ -0,0 +1,26 @@
+function [ncol,ncol_rank,ncol_hybrid,ncol_dem,ncol_scal] = ncol_from_colmap(col)
+  % [ncol,ncol_rank,ncol_hybrid,ncol_dem,ncol_scal] = ncol_from_flags(irank,ihybrid,idem,iscal)
+  % Get number of columns from containers.Map object
+  % Input
+  %   col             column map which can be indexed by e.g. col('x')
+  % Output
+  %   ncol            total number of columns
+  %   ncol_rank       number of columns (rank)
+  %   ncol_hybrid     number of columns (hybrid)
+  %   ncol_dem        number of columns (DEM)
+  %   ncol_scal       number of columns (scalar)
+  
+  ncol_rank   = 0;
+  ncol_hybrid = 0;
+  ncol_dem    = 0;
+  ncol_scal   = 0;
+  if col.isKey('rank'); ncol_rank   = 1;     end
+  if col.isKey('fx');   ncol_hybrid = 21;    end
+  if col.isKey('fxc');  ncol_dem    = 6;      end
+  ii = 1;
+  while col.isKey(sprintf('s%d',ii))
+    ncol_scal = ncol_scal+2;
+    ii = ii+1;
+  end
+  ncol = ncol_rank+ncol_hybrid+ncol_dem+ncol_scal;
+end

matlab/ncol_from_flags.m

@@ -0,0 +1,25 @@
+function [ncol,ncol_rank,ncol_hybrid,ncol_dem,ncol_scal] = ncol_from_flags(irank,ihybrid,idem,iscal)
+  % [ncol,ncol_rank,ncol_hybrid,ncol_dem,ncol_scal] = ncol_from_flags(irank,ihybrid,idem,iscal)
+  % Get number of columns from flags
+  % Input
+  %   irank           rank written?
+  %   ihybrid         hybrid written?
+  %   idem            DEM written?
+  %   iscal           scalar written? (number of scalars)
+  % Output
+  %   ncol            total number of columns
+  %   ncol_rank       number of columns (rank)
+  %   ncol_hybrid     number of columns (hybrid)
+  %   ncol_dem        number of columns (DEM)
+  %   ncol_scal       number of columns (scalar)
+  
+  ncol_rank   = 0;
+  ncol_hybrid = 0;
+  ncol_dem    = 0;
+  ncol_scal   = 0;
+  if irank;   ncol_rank   = 1;       end
+  if ihybrid; ncol_hybrid = 21;      end
+  if idem;    ncol_dem    = 6;       end
+  if iscal;   ncol_scal   = 2*iscal; end
+  ncol = ncol_rank+ncol_hybrid+ncol_dem+ncol_scal;
+end

matlab/precompute_ghosts_uvwp_ucf.m

@@ -0,0 +1,366 @@
+function [] = precompute_ghosts_uvwp_ucf(hucf,dout,nghost,varargin)
+  % [] = precompute_ghosts_uvwp_ucf(hucf,dout,nghost,varargin)
+  % Constructs a given number of ghost cells per processor and saves them to
+  % mat-files.
+  % The ghost cell generation is performed with a single read operation (input sweep),
+  % which extracts the points needed by the neighbors, and a subsequent processing
+  % sweep, which exchanges the ghost cell data. The data is hold in memory using
+  % a sparse data structure (c.f. https://de.mathworks.com/matlabcentral/fileexchange/29832-n-dimensional-sparse-arrays)
+  % and is also written to file in that way, so the 'ndSparse' class is a prerequisite.
+  % This way, the ghost cells can be added to a chunk of data with a simple add
+  % operation while avoiding large consumption of disk space.
+  % Input
+  %   hucf              UCF handle (ustar,ucfmulti)
+  %   dout              output directory
+  %   nghost            number of ghost cells to compute
+  %                       periodic:       take data from neighboring processor
+  %                       non-periodic:   duplicate last datapoint
+  % ? verbosity         verbose output? (includes estimation of the memory footprint) [default: 0]
+  % ? filebase          basename of the output files [default: ghost<nghost>]
+  % ? iseq              sequence number of the output files [default: 0]
+
+  % Parse input
+  par = inputParser;
+  addParamValue(par,'verbosity',0,@isnumeric);
+  addParamValue(par,'filebase',sprintf('ghost%d',nghost),@ischar);
+  addParamValue(par,'iseq',0,@isnumeric);
+  parse(par,varargin{:});
+  flag_verb = par.Results.verbosity;
+  filebase  = par.Results.filebase;
+  iseq      = par.Results.iseq;
+  
+  % Read info from tar-archive
+  [ibegu,iendu,jbegu,jendu,kbegu,kendu,...
+   ibegv,iendv,jbegv,jendv,kbegv,kendv,...
+   ibegw,iendw,jbegw,jendw,kbegw,kendw,...
+   ibegp,iendp,jbegp,jendp,kbegp,kendp] = read_procgrid_ucf(hucf);
+  [params] = read_parameters_ucf(hucf);
+  nxp = params.mesh.nxp;
+  nyp = params.mesh.nyp;
+  nzp = params.mesh.nzp;
+  nxprocs = params.parallel.nxprocs;
+  nyprocs = params.parallel.nyprocs;
+  nzprocs = params.parallel.nzprocs;
+  xperiodic = params.geometry.xperiodic;
+  yperiodic = params.geometry.yperiodic;
+  zperiodic = params.geometry.zperiodic;
+  
+  % Buffer for boundary data of each chunk
+  ubd = cell(nxprocs,nyprocs,nzprocs);
+  vbd = cell(nxprocs,nyprocs,nzprocs);
+  wbd = cell(nxprocs,nyprocs,nzprocs);
+  pbd = cell(nxprocs,nyprocs,nzprocs);
+  
+  % Estimate memory requirement (conservative simplification)
+  if flag_verb
+    nxl_estm = floor(nxp/nxprocs);
+    nyl_estm = floor(nyp/nyprocs);
+    nzl_estm = floor(nzp/nzprocs);
+    imem = 0;
+    for ii=0:nghost-1
+      imem = imem + 4*8*(...
+              2*(nxl_estm-ii)*(nyl_estm-ii) + ...
+              2*(nyl_estm-ii)*(nzl_estm-ii) + ...
+              2*(nxl_estm-ii)*(nzl_estm-ii)) * ...
+              (nxprocs*nyprocs*nzprocs);
+    end
+    fprintf('Estimated memory requirement: %7.0f MiB\n',imem/(1024*1024));
+  end
+  
+  % Read boundary data of each chunk into memory
+  for ixproc=0:nxprocs-1
+    for iyproc=0:nyprocs-1
+      for izproc=0:nzprocs-1
+        ichunk = ixproc*nyprocs*nzprocs+iyproc*nzprocs+izproc;
+        if flag_verb
+          fprintf('Reading original chunk %5d\n',ichunk);
+        end
+        [u,ibu,jbu,kbu,nxul,nyul,nzul,...
+         v,ibv,jbv,kbv,nxvl,nyvl,nzvl,...
+         w,ibw,jbw,kbw,nxwl,nywl,nzwl,...
+         p,ibp,jbp,kbp,nxpl,nypl,nzpl,ighost] = ...
+                  read_uvwp_chunk_ucf(hucf,'rank',ichunk,'ghost',0);
+        % Set interior to zero
+        u(1+nghost:end-nghost,1+nghost:end-nghost,1+nghost:end-nghost) = zeros(nxul-2*nghost,nyul-2*nghost,nzul-2*nghost);
+        v(1+nghost:end-nghost,1+nghost:end-nghost,1+nghost:end-nghost) = zeros(nxvl-2*nghost,nyvl-2*nghost,nzvl-2*nghost);
+        w(1+nghost:end-nghost,1+nghost:end-nghost,1+nghost:end-nghost) = zeros(nxwl-2*nghost,nywl-2*nghost,nzwl-2*nghost);
+        p(1+nghost:end-nghost,1+nghost:end-nghost,1+nghost:end-nghost) = zeros(nxpl-2*nghost,nypl-2*nghost,nzpl-2*nghost);
+        % Save ghost points as sparse matrix
+        ubd{ixproc+1,iyproc+1,izproc+1} = ndSparse(u);
+        vbd{ixproc+1,iyproc+1,izproc+1} = ndSparse(v);
+        wbd{ixproc+1,iyproc+1,izproc+1} = ndSparse(w);
+        pbd{ixproc+1,iyproc+1,izproc+1} = ndSparse(p);
+      end
+    end
+  end
+  
+  % Construct ghost cells for each chunk
+  for ixproc=0:nxprocs-1
+    nxul = iendu(ixproc+1)-ibegu(ixproc+1)+1;
+    nxvl = iendv(ixproc+1)-ibegv(ixproc+1)+1;
+    nxwl = iendw(ixproc+1)-ibegw(ixproc+1)+1;
+    nxpl = iendp(ixproc+1)-ibegp(ixproc+1)+1;
+    for iyproc=0:nyprocs-1
+      nyul = jendu(iyproc+1)-jbegu(iyproc+1)+1;
+      nyvl = jendv(iyproc+1)-jbegv(iyproc+1)+1;
+      nywl = jendw(iyproc+1)-jbegw(iyproc+1)+1;
+      nypl = jendp(iyproc+1)-jbegp(iyproc+1)+1;
+      for izproc=0:nzprocs-1
+        nzul = kendu(izproc+1)-kbegu(izproc+1)+1;
+        nzvl = kendv(izproc+1)-kbegv(izproc+1)+1;
+        nzwl = kendw(izproc+1)-kbegw(izproc+1)+1;
+        nzpl = kendp(izproc+1)-kbegp(izproc+1)+1;
+        
+        ichunk = ixproc*nyprocs*nzprocs+iyproc*nzprocs+izproc;
+        if flag_verb
+          fprintf('Constructing ghost cells for chunk %5d\n',ichunk);
+        end
+        
+        % Create sparse arrays which hold the results
+        ugh = ones(nxul+2*nghost,nyul+2*nghost,nzul+2*nghost);
+        ugh(1+nghost:end-nghost,1+nghost:end-nghost,1+nghost:end-nghost) = zeros(nxul,nyul,nzul);
+        ugh = ndSparse(ugh);
+        vgh = ones(nxvl+2*nghost,nyvl+2*nghost,nzvl+2*nghost);
+        vgh(1+nghost:end-nghost,1+nghost:end-nghost,1+nghost:end-nghost) = zeros(nxvl,nyvl,nzvl);
+        vgh = ndSparse(vgh);
+        wgh = ones(nxwl+2*nghost,nywl+2*nghost,nzwl+2*nghost);
+        wgh(1+nghost:end-nghost,1+nghost:end-nghost,1+nghost:end-nghost) = zeros(nxwl,nywl,nzwl);
+        wgh = ndSparse(wgh);
+        pgh = ones(nxpl+2*nghost,nypl+2*nghost,nzpl+2*nghost);
+        pgh(1+nghost:end-nghost,1+nghost:end-nghost,1+nghost:end-nghost) = zeros(nxpl,nypl,nzpl);
+        pgh = ndSparse(pgh);
+        
+        % Fill them with ghost cell data
+        % u
+        % 1st faces (6)
+        ugh(1:nghost,           1+nghost:end-nghost,1+nghost:end-nghost) = ubd{mod(ixproc-1,nxprocs)+1,iyproc+1,izproc+1}(end-nghost+1:end,:,               :               );
+        ugh(end-nghost+1:end,   1+nghost:end-nghost,1+nghost:end-nghost) = ubd{mod(ixproc+1,nxprocs)+1,iyproc+1,izproc+1}(1:nghost,        :,               :               );
+        ugh(1+nghost:end-nghost,1:nghost,           1+nghost:end-nghost) = ubd{ixproc+1,mod(iyproc-1,nyprocs)+1,izproc+1}(:,               end-nghost+1:end,:               );
+        ugh(1+nghost:end-nghost,end-nghost+1:end,   1+nghost:end-nghost) = ubd{ixproc+1,mod(iyproc+1,nyprocs)+1,izproc+1}(:,               1:nghost,        :               );
+        ugh(1+nghost:end-nghost,1+nghost:end-nghost,1:nghost           ) = ubd{ixproc+1,iyproc+1,mod(izproc-1,nzprocs)+1}(:,               :,               end-nghost+1:end);
+        ugh(1+nghost:end-nghost,1+nghost:end-nghost,end-nghost+1:end   ) = ubd{ixproc+1,iyproc+1,mod(izproc+1,nzprocs)+1}(:,               :,               1:nghost        );
+        % 2nd edges (12)
+        ugh(1:nghost,           1:nghost,           1+nghost:end-nghost) = ubd{mod(ixproc-1,nxprocs)+1,mod(iyproc-1,nyprocs)+1,izproc+1               }(end-nghost+1:end,end-nghost+1:end,:);
+        ugh(1:nghost,           1+nghost:end-nghost,1:nghost           ) = ubd{mod(ixproc-1,nxprocs)+1,iyproc+1,               mod(izproc-1,nzprocs)+1}(end-nghost+1:end,:,               end-nghost+1:end);
+        ugh(1+nghost:end-nghost,1:nghost,           1:nghost           ) = ubd{ixproc+1,               mod(iyproc-1,nyprocs)+1,mod(izproc-1,nzprocs)+1}(:,               end-nghost+1:end,end-nghost+1:end);
+        ugh(end-nghost+1:end,   end-nghost+1:end,   1+nghost:end-nghost) = ubd{mod(ixproc+1,nxprocs)+1,mod(iyproc+1,nyprocs)+1,izproc+1               }(1:nghost,        1:nghost,        :               );
+        ugh(end-nghost+1:end,   1+nghost:end-nghost,end-nghost+1:end   ) = ubd{mod(ixproc+1,nxprocs)+1,iyproc+1,               mod(izproc+1,nzprocs)+1}(1:nghost,        :,               1:nghost        );
+        ugh(1+nghost:end-nghost,end-nghost+1:end,   end-nghost+1:end   ) = ubd{ixproc+1,               mod(iyproc+1,nyprocs)+1,mod(izproc+1,nzprocs)+1}(:,               1:nghost,        1:nghost        );
+        ugh(1:nghost,           end-nghost+1:end,   1+nghost:end-nghost) = ubd{mod(ixproc-1,nxprocs)+1,mod(iyproc+1,nyprocs)+1,izproc+1               }(end-nghost+1:end,1:nghost,        :               );
+        ugh(1:nghost,           1+nghost:end-nghost,end-nghost+1:end   ) = ubd{mod(ixproc-1,nxprocs)+1,iyproc+1,               mod(izproc+1,nzprocs)+1}(end-nghost+1:end,:,               1:nghost        );
+        ugh(1+nghost:end-nghost,1:nghost,           end-nghost+1:end   ) = ubd{ixproc+1,               mod(iyproc-1,nyprocs)+1,mod(izproc+1,nzprocs)+1}(:,               end-nghost+1:end,1:nghost        );
+        ugh(end-nghost+1:end,   1:nghost,           1+nghost:end-nghost) = ubd{mod(ixproc+1,nxprocs)+1,mod(iyproc-1,nyprocs)+1,izproc+1               }(1:nghost,        end-nghost+1:end,:               );
+        ugh(end-nghost+1:end,   1+nghost:end-nghost,1:nghost           ) = ubd{mod(ixproc+1,nxprocs)+1,iyproc+1,               mod(izproc-1,nzprocs)+1}(1:nghost,        :,               end-nghost+1:end);
+        ugh(1+nghost:end-nghost,end-nghost+1:end,   1:nghost           ) = ubd{ixproc+1,               mod(iyproc+1,nyprocs)+1,mod(izproc-1,nzprocs)+1}(:,               1:nghost,        end-nghost+1:end);
+        % 3rd corners (8)
+        ugh(1:nghost,        1:nghost,        1:nghost        ) = ubd{mod(ixproc-1,nxprocs)+1,mod(iyproc-1,nyprocs)+1,mod(izproc-1,nzprocs)+1}(end-nghost+1:end,end-nghost+1:end,end-nghost+1:end);
+        ugh(end-nghost+1:end,1:nghost,        1:nghost        ) = ubd{mod(ixproc+1,nxprocs)+1,mod(iyproc-1,nyprocs)+1,mod(izproc-1,nzprocs)+1}(1:nghost,        end-nghost+1:end,end-nghost+1:end);
+        ugh(1:nghost,        end-nghost+1:end,1:nghost        ) = ubd{mod(ixproc-1,nxprocs)+1,mod(iyproc+1,nyprocs)+1,mod(izproc-1,nzprocs)+1}(end-nghost+1:end,1:nghost,        end-nghost+1:end);
+        ugh(1:nghost,        1:nghost,        end-nghost+1:end) = ubd{mod(ixproc-1,nxprocs)+1,mod(iyproc-1,nyprocs)+1,mod(izproc+1,nzprocs)+1}(end-nghost+1:end,end-nghost+1:end,1:nghost        );
+        ugh(end-nghost+1:end,end-nghost+1:end,1:nghost        ) = ubd{mod(ixproc+1,nxprocs)+1,mod(iyproc+1,nyprocs)+1,mod(izproc-1,nzprocs)+1}(1:nghost,        1:nghost,        end-nghost+1:end);
+        ugh(end-nghost+1:end,1:nghost,        end-nghost+1:end) = ubd{mod(ixproc+1,nxprocs)+1,mod(iyproc-1,nyprocs)+1,mod(izproc+1,nzprocs)+1}(1:nghost,        end-nghost+1:end,1:nghost        );
+        ugh(1:nghost,        end-nghost+1:end,end-nghost+1:end) = ubd{mod(ixproc-1,nxprocs)+1,mod(iyproc+1,nyprocs)+1,mod(izproc+1,nzprocs)+1}(end-nghost+1:end,1:nghost,        1:nghost        );
+        ugh(end-nghost+1:end,end-nghost+1:end,end-nghost+1:end) = ubd{mod(ixproc+1,nxprocs)+1,mod(iyproc+1,nyprocs)+1,mod(izproc+1,nzprocs)+1}(1:nghost,        1:nghost,        1:nghost        );
+        % Correct if not periodic
+        if ~xperiodic && ixproc==0
+          ugh(1:nghost,        :,                  :                  ) = repmat(ugh(nghost+1,:,:),[nghost,1,1]);
+          ugh(1:nghost,        1+nghost:end-nghost,1+nghost:end-nghost) = repmat(ubd{ixproc+1,iyproc+1,izproc+1}(1,:,:),[nghost,1,1]);
+        end
+        if ~xperiodic && ixproc==nxprocs-1
+          ugh(end-nghost+1:end,:,                  :                  ) = repmat(ugh(end-nghost,:,:),[nghost,1,1]);
+          ugh(end-nghost+1:end,1+nghost:end-nghost,1+nghost:end-nghost) = repmat(ubd{ixproc+1,iyproc+1,izproc+1}(end,:,:),[nghost,1,1]);
+        end
+        if ~yperiodic && iyproc==0
+          ugh(:,                  1:nghost,        :                  ) = repmat(ugh(:,nghost+1,:),[1,nghost,1]);
+          ugh(1+nghost:end-nghost,1:nghost,        1+nghost:end-nghost) = repmat(ubd{ixproc+1,iyproc+1,izproc+1}(:,1,:),[1,nghost,1]);
+        end
+        if ~yperiodic && iyproc==nyprocs-1
+          ugh(:,                  end-nghost+1:end,:                  ) = repmat(ugh(:,end-nghost,:),[1,nghost,1]);
+          ugh(1+nghost:end-nghost,end-nghost+1:end,1+nghost:end-nghost) = repmat(ubd{ixproc+1,iyproc+1,izproc+1}(:,end,:),[1,nghost,1]);
+        end
+        if ~zperiodic && izproc==0
+          ugh(:,                  :,                  1:nghost        ) = repmat(ugh(:,:,nghost+1),[1,1,nghost]);
+          ugh(1+nghost:end-nghost,1+nghost:end-nghost,1:nghost        ) = repmat(ubd{ixproc+1,iyproc+1,izproc+1}(:,:,1),[1,1,nghost]);
+        end
+        if ~zperiodic && izproc==nzprocs-1
+          ugh(:,                  :,                  end-nghost+1:end) = repmat(ugh(:,:,end-nghost),[1,1,nghost]);
+          ugh(1+nghost:end-nghost,1+nghost:end-nghost,end-nghost+1:end) = repmat(ubd{ixproc+1,iyproc+1,izproc+1}(:,:,end),[1,1,nghost]);
+        end
+        
+        % Fill them with ghost cell data
+        % v
+        % 1st faces (6)
+        vgh(1:nghost,           1+nghost:end-nghost,1+nghost:end-nghost) = vbd{mod(ixproc-1,nxprocs)+1,iyproc+1,izproc+1}(end-nghost+1:end,:,               :               );
+        vgh(end-nghost+1:end,   1+nghost:end-nghost,1+nghost:end-nghost) = vbd{mod(ixproc+1,nxprocs)+1,iyproc+1,izproc+1}(1:nghost,        :,               :               );
+        vgh(1+nghost:end-nghost,1:nghost,           1+nghost:end-nghost) = vbd{ixproc+1,mod(iyproc-1,nyprocs)+1,izproc+1}(:,               end-nghost+1:end,:               );
+        vgh(1+nghost:end-nghost,end-nghost+1:end,   1+nghost:end-nghost) = vbd{ixproc+1,mod(iyproc+1,nyprocs)+1,izproc+1}(:,               1:nghost,        :               );
+        vgh(1+nghost:end-nghost,1+nghost:end-nghost,1:nghost           ) = vbd{ixproc+1,iyproc+1,mod(izproc-1,nzprocs)+1}(:,               :,               end-nghost+1:end);
+        vgh(1+nghost:end-nghost,1+nghost:end-nghost,end-nghost+1:end   ) = vbd{ixproc+1,iyproc+1,mod(izproc+1,nzprocs)+1}(:,               :,               1:nghost        );
+        % 2nd edges (12)
+        vgh(1:nghost,           1:nghost,           1+nghost:end-nghost) = vbd{mod(ixproc-1,nxprocs)+1,mod(iyproc-1,nyprocs)+1,izproc+1               }(end-nghost+1:end,end-nghost+1:end,:);
+        vgh(1:nghost,           1+nghost:end-nghost,1:nghost           ) = vbd{mod(ixproc-1,nxprocs)+1,iyproc+1,               mod(izproc-1,nzprocs)+1}(end-nghost+1:end,:,               end-nghost+1:end);
+        vgh(1+nghost:end-nghost,1:nghost,           1:nghost           ) = vbd{ixproc+1,               mod(iyproc-1,nyprocs)+1,mod(izproc-1,nzprocs)+1}(:,               end-nghost+1:end,end-nghost+1:end);
+        vgh(end-nghost+1:end,   end-nghost+1:end,   1+nghost:end-nghost) = vbd{mod(ixproc+1,nxprocs)+1,mod(iyproc+1,nyprocs)+1,izproc+1               }(1:nghost,        1:nghost,        :               );
+        vgh(end-nghost+1:end,   1+nghost:end-nghost,end-nghost+1:end   ) = vbd{mod(ixproc+1,nxprocs)+1,iyproc+1,               mod(izproc+1,nzprocs)+1}(1:nghost,        :,               1:nghost        );
+        vgh(1+nghost:end-nghost,end-nghost+1:end,   end-nghost+1:end   ) = vbd{ixproc+1,               mod(iyproc+1,nyprocs)+1,mod(izproc+1,nzprocs)+1}(:,               1:nghost,        1:nghost        );
+        vgh(1:nghost,           end-nghost+1:end,   1+nghost:end-nghost) = vbd{mod(ixproc-1,nxprocs)+1,mod(iyproc+1,nyprocs)+1,izproc+1               }(end-nghost+1:end,1:nghost,        :               );
+        vgh(1:nghost,           1+nghost:end-nghost,end-nghost+1:end   ) = vbd{mod(ixproc-1,nxprocs)+1,iyproc+1,               mod(izproc+1,nzprocs)+1}(end-nghost+1:end,:,               1:nghost        );
+        vgh(1+nghost:end-nghost,1:nghost,           end-nghost+1:end   ) = vbd{ixproc+1,               mod(iyproc-1,nyprocs)+1,mod(izproc+1,nzprocs)+1}(:,               end-nghost+1:end,1:nghost        );
+        vgh(end-nghost+1:end,   1:nghost,           1+nghost:end-nghost) = vbd{mod(ixproc+1,nxprocs)+1,mod(iyproc-1,nyprocs)+1,izproc+1               }(1:nghost,        end-nghost+1:end,:               );
+        vgh(end-nghost+1:end,   1+nghost:end-nghost,1:nghost           ) = vbd{mod(ixproc+1,nxprocs)+1,iyproc+1,               mod(izproc-1,nzprocs)+1}(1:nghost,        :,               end-nghost+1:end);
+        vgh(1+nghost:end-nghost,end-nghost+1:end,   1:nghost           ) = vbd{ixproc+1,               mod(iyproc+1,nyprocs)+1,mod(izproc-1,nzprocs)+1}(:,               1:nghost,        end-nghost+1:end);
+        % 3rd corners (8)
+        vgh(1:nghost,        1:nghost,        1:nghost        ) = vbd{mod(ixproc-1,nxprocs)+1,mod(iyproc-1,nyprocs)+1,mod(izproc-1,nzprocs)+1}(end-nghost+1:end,end-nghost+1:end,end-nghost+1:end);
+        vgh(end-nghost+1:end,1:nghost,        1:nghost        ) = vbd{mod(ixproc+1,nxprocs)+1,mod(iyproc-1,nyprocs)+1,mod(izproc-1,nzprocs)+1}(1:nghost,        end-nghost+1:end,end-nghost+1:end);
+        vgh(1:nghost,        end-nghost+1:end,1:nghost        ) = vbd{mod(ixproc-1,nxprocs)+1,mod(iyproc+1,nyprocs)+1,mod(izproc-1,nzprocs)+1}(end-nghost+1:end,1:nghost,        end-nghost+1:end);
+        vgh(1:nghost,        1:nghost,        end-nghost+1:end) = vbd{mod(ixproc-1,nxprocs)+1,mod(iyproc-1,nyprocs)+1,mod(izproc+1,nzprocs)+1}(end-nghost+1:end,end-nghost+1:end,1:nghost        );
+        vgh(end-nghost+1:end,end-nghost+1:end,1:nghost        ) = vbd{mod(ixproc+1,nxprocs)+1,mod(iyproc+1,nyprocs)+1,mod(izproc-1,nzprocs)+1}(1:nghost,        1:nghost,        end-nghost+1:end);
+        vgh(end-nghost+1:end,1:nghost,        end-nghost+1:end) = vbd{mod(ixproc+1,nxprocs)+1,mod(iyproc-1,nyprocs)+1,mod(izproc+1,nzprocs)+1}(1:nghost,        end-nghost+1:end,1:nghost        );
+        vgh(1:nghost,        end-nghost+1:end,end-nghost+1:end) = vbd{mod(ixproc-1,nxprocs)+1,mod(iyproc+1,nyprocs)+1,mod(izproc+1,nzprocs)+1}(end-nghost+1:end,1:nghost,        1:nghost        );
+        vgh(end-nghost+1:end,end-nghost+1:end,end-nghost+1:end) = vbd{mod(ixproc+1,nxprocs)+1,mod(iyproc+1,nyprocs)+1,mod(izproc+1,nzprocs)+1}(1:nghost,        1:nghost,        1:nghost        );
+        % Correct if not periodic
+        if ~xperiodic && ixproc==0
+          vgh(1:nghost,        :,                  :                  ) = repmat(vgh(nghost+1,:,:),[nghost,1,1]);
+          vgh(1:nghost,        1+nghost:end-nghost,1+nghost:end-nghost) = repmat(vbd{ixproc+1,iyproc+1,izproc+1}(1,:,:),[nghost,1,1]);
+        end
+        if ~xperiodic && ixproc==nxprocs-1
+          vgh(end-nghost+1:end,:,                  :                  ) = repmat(vgh(end-nghost,:,:),[nghost,1,1]);
+          vgh(end-nghost+1:end,1+nghost:end-nghost,1+nghost:end-nghost) = repmat(vbd{ixproc+1,iyproc+1,izproc+1}(end,:,:),[nghost,1,1]);
+        end
+        if ~yperiodic && iyproc==0
+          vgh(:,                  1:nghost,        :                  ) = repmat(vgh(:,nghost+1,:),[1,nghost,1]);
+          vgh(1+nghost:end-nghost,1:nghost,        1+nghost:end-nghost) = repmat(vbd{ixproc+1,iyproc+1,izproc+1}(:,1,:),[1,nghost,1]);
+        end
+        if ~yperiodic && iyproc==nyprocs-1
+          vgh(:,                  end-nghost+1:end,:                  ) = repmat(vgh(:,end-nghost,:),[1,nghost,1]);
+          vgh(1+nghost:end-nghost,end-nghost+1:end,1+nghost:end-nghost) = repmat(vbd{ixproc+1,iyproc+1,izproc+1}(:,end,:),[1,nghost,1]);
+        end
+        if ~zperiodic && izproc==0
+          vgh(:,                  :,                  1:nghost        ) = repmat(vgh(:,:,nghost+1),[1,1,nghost]);
+          vgh(1+nghost:end-nghost,1+nghost:end-nghost,1:nghost        ) = repmat(vbd{ixproc+1,iyproc+1,izproc+1}(:,:,1),[1,1,nghost]);
+        end
+        if ~zperiodic && izproc==nzprocs-1
+          vgh(:,                  :,                  end-nghost+1:end) = repmat(vgh(:,:,end-nghost),[1,1,nghost]);
+          vgh(1+nghost:end-nghost,1+nghost:end-nghost,end-nghost+1:end) = repmat(vbd{ixproc+1,iyproc+1,izproc+1}(:,:,end),[1,1,nghost]);
+        end
+        
+        % Fill them with ghost cell data
+        % w
+        % 1st faces (6)
+        wgh(1:nghost,           1+nghost:end-nghost,1+nghost:end-nghost) = wbd{mod(ixproc-1,nxprocs)+1,iyproc+1,izproc+1}(end-nghost+1:end,:,               :               );
+        wgh(end-nghost+1:end,   1+nghost:end-nghost,1+nghost:end-nghost) = wbd{mod(ixproc+1,nxprocs)+1,iyproc+1,izproc+1}(1:nghost,        :,               :               );
+        wgh(1+nghost:end-nghost,1:nghost,           1+nghost:end-nghost) = wbd{ixproc+1,mod(iyproc-1,nyprocs)+1,izproc+1}(:,               end-nghost+1:end,:               );
+        wgh(1+nghost:end-nghost,end-nghost+1:end,   1+nghost:end-nghost) = wbd{ixproc+1,mod(iyproc+1,nyprocs)+1,izproc+1}(:,               1:nghost,        :               );
+        wgh(1+nghost:end-nghost,1+nghost:end-nghost,1:nghost           ) = wbd{ixproc+1,iyproc+1,mod(izproc-1,nzprocs)+1}(:,               :,               end-nghost+1:end);
+        wgh(1+nghost:end-nghost,1+nghost:end-nghost,end-nghost+1:end   ) = wbd{ixproc+1,iyproc+1,mod(izproc+1,nzprocs)+1}(:,               :,               1:nghost        );
+        % 2nd edges (12)
+        wgh(1:nghost,           1:nghost,           1+nghost:end-nghost) = wbd{mod(ixproc-1,nxprocs)+1,mod(iyproc-1,nyprocs)+1,izproc+1               }(end-nghost+1:end,end-nghost+1:end,:);
+        wgh(1:nghost,           1+nghost:end-nghost,1:nghost           ) = wbd{mod(ixproc-1,nxprocs)+1,iyproc+1,               mod(izproc-1,nzprocs)+1}(end-nghost+1:end,:,               end-nghost+1:end);
+        wgh(1+nghost:end-nghost,1:nghost,           1:nghost           ) = wbd{ixproc+1,               mod(iyproc-1,nyprocs)+1,mod(izproc-1,nzprocs)+1}(:,               end-nghost+1:end,end-nghost+1:end);
+        wgh(end-nghost+1:end,   end-nghost+1:end,   1+nghost:end-nghost) = wbd{mod(ixproc+1,nxprocs)+1,mod(iyproc+1,nyprocs)+1,izproc+1               }(1:nghost,        1:nghost,        :               );
+        wgh(end-nghost+1:end,   1+nghost:end-nghost,end-nghost+1:end   ) = wbd{mod(ixproc+1,nxprocs)+1,iyproc+1,               mod(izproc+1,nzprocs)+1}(1:nghost,        :,               1:nghost        );
+        wgh(1+nghost:end-nghost,end-nghost+1:end,   end-nghost+1:end   ) = wbd{ixproc+1,               mod(iyproc+1,nyprocs)+1,mod(izproc+1,nzprocs)+1}(:,               1:nghost,        1:nghost        );
+        wgh(1:nghost,           end-nghost+1:end,   1+nghost:end-nghost) = wbd{mod(ixproc-1,nxprocs)+1,mod(iyproc+1,nyprocs)+1,izproc+1               }(end-nghost+1:end,1:nghost,        :               );
+        wgh(1:nghost,           1+nghost:end-nghost,end-nghost+1:end   ) = wbd{mod(ixproc-1,nxprocs)+1,iyproc+1,               mod(izproc+1,nzprocs)+1}(end-nghost+1:end,:,               1:nghost        );
+        wgh(1+nghost:end-nghost,1:nghost,           end-nghost+1:end   ) = wbd{ixproc+1,               mod(iyproc-1,nyprocs)+1,mod(izproc+1,nzprocs)+1}(:,               end-nghost+1:end,1:nghost        );
+        wgh(end-nghost+1:end,   1:nghost,           1+nghost:end-nghost) = wbd{mod(ixproc+1,nxprocs)+1,mod(iyproc-1,nyprocs)+1,izproc+1               }(1:nghost,        end-nghost+1:end,:               );
+        wgh(end-nghost+1:end,   1+nghost:end-nghost,1:nghost           ) = wbd{mod(ixproc+1,nxprocs)+1,iyproc+1,               mod(izproc-1,nzprocs)+1}(1:nghost,        :,               end-nghost+1:end);
+        wgh(1+nghost:end-nghost,end-nghost+1:end,   1:nghost           ) = wbd{ixproc+1,               mod(iyproc+1,nyprocs)+1,mod(izproc-1,nzprocs)+1}(:,               1:nghost,        end-nghost+1:end);
+        % 3rd corners (8)
+        wgh(1:nghost,        1:nghost,        1:nghost        ) = wbd{mod(ixproc-1,nxprocs)+1,mod(iyproc-1,nyprocs)+1,mod(izproc-1,nzprocs)+1}(end-nghost+1:end,end-nghost+1:end,end-nghost+1:end);
+        wgh(end-nghost+1:end,1:nghost,        1:nghost        ) = wbd{mod(ixproc+1,nxprocs)+1,mod(iyproc-1,nyprocs)+1,mod(izproc-1,nzprocs)+1}(1:nghost,        end-nghost+1:end,end-nghost+1:end);
+        wgh(1:nghost,        end-nghost+1:end,1:nghost        ) = wbd{mod(ixproc-1,nxprocs)+1,mod(iyproc+1,nyprocs)+1,mod(izproc-1,nzprocs)+1}(end-nghost+1:end,1:nghost,        end-nghost+1:end);
+        wgh(1:nghost,        1:nghost,        end-nghost+1:end) = wbd{mod(ixproc-1,nxprocs)+1,mod(iyproc-1,nyprocs)+1,mod(izproc+1,nzprocs)+1}(end-nghost+1:end,end-nghost+1:end,1:nghost        );
+        wgh(end-nghost+1:end,end-nghost+1:end,1:nghost        ) = wbd{mod(ixproc+1,nxprocs)+1,mod(iyproc+1,nyprocs)+1,mod(izproc-1,nzprocs)+1}(1:nghost,        1:nghost,        end-nghost+1:end);
+        wgh(end-nghost+1:end,1:nghost,        end-nghost+1:end) = wbd{mod(ixproc+1,nxprocs)+1,mod(iyproc-1,nyprocs)+1,mod(izproc+1,nzprocs)+1}(1:nghost,        end-nghost+1:end,1:nghost        );
+        wgh(1:nghost,        end-nghost+1:end,end-nghost+1:end) = wbd{mod(ixproc-1,nxprocs)+1,mod(iyproc+1,nyprocs)+1,mod(izproc+1,nzprocs)+1}(end-nghost+1:end,1:nghost,        1:nghost        );
+        wgh(end-nghost+1:end,end-nghost+1:end,end-nghost+1:end) = wbd{mod(ixproc+1,nxprocs)+1,mod(iyproc+1,nyprocs)+1,mod(izproc+1,nzprocs)+1}(1:nghost,        1:nghost,        1:nghost        );
+        % Correct if not periodic
+        if ~xperiodic && ixproc==0
+          wgh(1:nghost,        :,                  :                  ) = repmat(wgh(nghost+1,:,:),[nghost,1,1]);
+          wgh(1:nghost,        1+nghost:end-nghost,1+nghost:end-nghost) = repmat(wbd{ixproc+1,iyproc+1,izproc+1}(1,:,:),[nghost,1,1]);
+        end
+        if ~xperiodic && ixproc==nxprocs-1
+          wgh(end-nghost+1:end,:,                  :                  ) = repmat(wgh(end-nghost,:,:),[nghost,1,1]);
+          wgh(end-nghost+1:end,1+nghost:end-nghost,1+nghost:end-nghost) = repmat(wbd{ixproc+1,iyproc+1,izproc+1}(end,:,:),[nghost,1,1]);
+        end
+        if ~yperiodic && iyproc==0
+          wgh(:,                  1:nghost,        :                  ) = repmat(wgh(:,nghost+1,:),[1,nghost,1]);
+          wgh(1+nghost:end-nghost,1:nghost,        1+nghost:end-nghost) = repmat(wbd{ixproc+1,iyproc+1,izproc+1}(:,1,:),[1,nghost,1]);
+        end
+        if ~yperiodic && iyproc==nyprocs-1
+          wgh(:,                  end-nghost+1:end,:                  ) = repmat(wgh(:,end-nghost,:),[1,nghost,1]);
+          wgh(1+nghost:end-nghost,end-nghost+1:end,1+nghost:end-nghost) = repmat(wbd{ixproc+1,iyproc+1,izproc+1}(:,end,:),[1,nghost,1]);
+        end
+        if ~zperiodic && izproc==0
+          wgh(:,                  :,                  1:nghost        ) = repmat(wgh(:,:,nghost+1),[1,1,nghost]);
+          wgh(1+nghost:end-nghost,1+nghost:end-nghost,1:nghost        ) = repmat(wbd{ixproc+1,iyproc+1,izproc+1}(:,:,1),[1,1,nghost]);
+        end
+        if ~zperiodic && izproc==nzprocs-1
+          wgh(:,                  :,                  end-nghost+1:end) = repmat(wgh(:,:,end-nghost),[1,1,nghost]);
+          wgh(1+nghost:end-nghost,1+nghost:end-nghost,end-nghost+1:end) = repmat(wbd{ixproc+1,iyproc+1,izproc+1}(:,:,end),[1,1,nghost]);
+        end
+        
+        % Fill them with ghost cell data
+        % p
+        % 1st faces (6)
+        pgh(1:nghost,           1+nghost:end-nghost,1+nghost:end-nghost) = pbd{mod(ixproc-1,nxprocs)+1,iyproc+1,izproc+1}(end-nghost+1:end,:,               :               );
+        pgh(end-nghost+1:end,   1+nghost:end-nghost,1+nghost:end-nghost) = pbd{mod(ixproc+1,nxprocs)+1,iyproc+1,izproc+1}(1:nghost,        :,               :               );
+        pgh(1+nghost:end-nghost,1:nghost,           1+nghost:end-nghost) = pbd{ixproc+1,mod(iyproc-1,nyprocs)+1,izproc+1}(:,               end-nghost+1:end,:               );
+        pgh(1+nghost:end-nghost,end-nghost+1:end,   1+nghost:end-nghost) = pbd{ixproc+1,mod(iyproc+1,nyprocs)+1,izproc+1}(:,               1:nghost,        :               );
+        pgh(1+nghost:end-nghost,1+nghost:end-nghost,1:nghost           ) = pbd{ixproc+1,iyproc+1,mod(izproc-1,nzprocs)+1}(:,               :,               end-nghost+1:end);
+        pgh(1+nghost:end-nghost,1+nghost:end-nghost,end-nghost+1:end   ) = pbd{ixproc+1,iyproc+1,mod(izproc+1,nzprocs)+1}(:,               :,               1:nghost        );
+        % 2nd edges (12)
+        pgh(1:nghost,           1:nghost,           1+nghost:end-nghost) = pbd{mod(ixproc-1,nxprocs)+1,mod(iyproc-1,nyprocs)+1,izproc+1               }(end-nghost+1:end,end-nghost+1:end,:);
+        pgh(1:nghost,           1+nghost:end-nghost,1:nghost           ) = pbd{mod(ixproc-1,nxprocs)+1,iyproc+1,               mod(izproc-1,nzprocs)+1}(end-nghost+1:end,:,               end-nghost+1:end);
+        pgh(1+nghost:end-nghost,1:nghost,           1:nghost           ) = pbd{ixproc+1,               mod(iyproc-1,nyprocs)+1,mod(izproc-1,nzprocs)+1}(:,               end-nghost+1:end,end-nghost+1:end);
+        pgh(end-nghost+1:end,   end-nghost+1:end,   1+nghost:end-nghost) = pbd{mod(ixproc+1,nxprocs)+1,mod(iyproc+1,nyprocs)+1,izproc+1               }(1:nghost,        1:nghost,        :               );
+        pgh(end-nghost+1:end,   1+nghost:end-nghost,end-nghost+1:end   ) = pbd{mod(ixproc+1,nxprocs)+1,iyproc+1,               mod(izproc+1,nzprocs)+1}(1:nghost,        :,               1:nghost        );
+        pgh(1+nghost:end-nghost,end-nghost+1:end,   end-nghost+1:end   ) = pbd{ixproc+1,               mod(iyproc+1,nyprocs)+1,mod(izproc+1,nzprocs)+1}(:,               1:nghost,        1:nghost        );
+        pgh(1:nghost,           end-nghost+1:end,   1+nghost:end-nghost) = pbd{mod(ixproc-1,nxprocs)+1,mod(iyproc+1,nyprocs)+1,izproc+1               }(end-nghost+1:end,1:nghost,        :               );
+        pgh(1:nghost,           1+nghost:end-nghost,end-nghost+1:end   ) = pbd{mod(ixproc-1,nxprocs)+1,iyproc+1,               mod(izproc+1,nzprocs)+1}(end-nghost+1:end,:,               1:nghost        );
+        pgh(1+nghost:end-nghost,1:nghost,           end-nghost+1:end   ) = pbd{ixproc+1,               mod(iyproc-1,nyprocs)+1,mod(izproc+1,nzprocs)+1}(:,               end-nghost+1:end,1:nghost        );
+        pgh(end-nghost+1:end,   1:nghost,           1+nghost:end-nghost) = pbd{mod(ixproc+1,nxprocs)+1,mod(iyproc-1,nyprocs)+1,izproc+1               }(1:nghost,        end-nghost+1:end,:               );
+        pgh(end-nghost+1:end,   1+nghost:end-nghost,1:nghost           ) = pbd{mod(ixproc+1,nxprocs)+1,iyproc+1,               mod(izproc-1,nzprocs)+1}(1:nghost,        :,               end-nghost+1:end);
+        pgh(1+nghost:end-nghost,end-nghost+1:end,   1:nghost           ) = pbd{ixproc+1,               mod(iyproc+1,nyprocs)+1,mod(izproc-1,nzprocs)+1}(:,               1:nghost,        end-nghost+1:end);
+        % 3rd corners (8)
+        pgh(1:nghost,        1:nghost,        1:nghost        ) = pbd{mod(ixproc-1,nxprocs)+1,mod(iyproc-1,nyprocs)+1,mod(izproc-1,nzprocs)+1}(end-nghost+1:end,end-nghost+1:end,end-nghost+1:end);
+        pgh(end-nghost+1:end,1:nghost,        1:nghost        ) = pbd{mod(ixproc+1,nxprocs)+1,mod(iyproc-1,nyprocs)+1,mod(izproc-1,nzprocs)+1}(1:nghost,        end-nghost+1:end,end-nghost+1:end);
+        pgh(1:nghost,        end-nghost+1:end,1:nghost        ) = pbd{mod(ixproc-1,nxprocs)+1,mod(iyproc+1,nyprocs)+1,mod(izproc-1,nzprocs)+1}(end-nghost+1:end,1:nghost,        end-nghost+1:end);
+        pgh(1:nghost,        1:nghost,        end-nghost+1:end) = pbd{mod(ixproc-1,nxprocs)+1,mod(iyproc-1,nyprocs)+1,mod(izproc+1,nzprocs)+1}(end-nghost+1:end,end-nghost+1:end,1:nghost        );
+        pgh(end-nghost+1:end,end-nghost+1:end,1:nghost        ) = pbd{mod(ixproc+1,nxprocs)+1,mod(iyproc+1,nyprocs)+1,mod(izproc-1,nzprocs)+1}(1:nghost,        1:nghost,        end-nghost+1:end);
+        pgh(end-nghost+1:end,1:nghost,        end-nghost+1:end) = pbd{mod(ixproc+1,nxprocs)+1,mod(iyproc-1,nyprocs)+1,mod(izproc+1,nzprocs)+1}(1:nghost,        end-nghost+1:end,1:nghost        );
+        pgh(1:nghost,        end-nghost+1:end,end-nghost+1:end) = pbd{mod(ixproc-1,nxprocs)+1,mod(iyproc+1,nyprocs)+1,mod(izproc+1,nzprocs)+1}(end-nghost+1:end,1:nghost,        1:nghost        );
+        pgh(end-nghost+1:end,end-nghost+1:end,end-nghost+1:end) = pbd{mod(ixproc+1,nxprocs)+1,mod(iyproc+1,nyprocs)+1,mod(izproc+1,nzprocs)+1}(1:nghost,        1:nghost,        1:nghost        );
+        % Correct if not periodic
+        if ~xperiodic && ixproc==0
+          pgh(1:nghost,        :,                  :                  ) = repmat(pgh(nghost+1,:,:),[nghost,1,1]);
+          pgh(1:nghost,        1+nghost:end-nghost,1+nghost:end-nghost) = repmat(pbd{ixproc+1,iyproc+1,izproc+1}(1,:,:),[nghost,1,1]);
+        end
+        if ~xperiodic && ixproc==nxprocs-1
+          pgh(end-nghost+1:end,:,                  :                  ) = repmat(pgh(end-nghost,:,:),[nghost,1,1]);
+          pgh(end-nghost+1:end,1+nghost:end-nghost,1+nghost:end-nghost) = repmat(pbd{ixproc+1,iyproc+1,izproc+1}(end,:,:),[nghost,1,1]);
+        end
+        if ~yperiodic && iyproc==0
+          pgh(:,                  1:nghost,        :                  ) = repmat(pgh(:,nghost+1,:),[1,nghost,1]);
+          pgh(1+nghost:end-nghost,1:nghost,        1+nghost:end-nghost) = repmat(pbd{ixproc+1,iyproc+1,izproc+1}(:,1,:),[1,nghost,1]);
+        end
+        if ~yperiodic && iyproc==nyprocs-1
+          pgh(:,                  end-nghost+1:end,:                  ) = repmat(pgh(:,end-nghost,:),[1,nghost,1]);
+          pgh(1+nghost:end-nghost,end-nghost+1:end,1+nghost:end-nghost) = repmat(pbd{ixproc+1,iyproc+1,izproc+1}(:,end,:),[1,nghost,1]);
+        end
+        if ~zperiodic && izproc==0
+          pgh(:,                  :,                  1:nghost        ) = repmat(pgh(:,:,nghost+1),[1,1,nghost]);
+          pgh(1+nghost:end-nghost,1+nghost:end-nghost,1:nghost        ) = repmat(pbd{ixproc+1,iyproc+1,izproc+1}(:,:,1),[1,1,nghost]);
+        end
+        if ~zperiodic && izproc==nzprocs-1
+          pgh(:,                  :,                  end-nghost+1:end) = repmat(pgh(:,:,end-nghost),[1,1,nghost]);
+          pgh(1+nghost:end-nghost,1+nghost:end-nghost,end-nghost+1:end) = repmat(pbd{ixproc+1,iyproc+1,izproc+1}(:,:,end),[1,1,nghost]);
+        end
+        
+        fout = sprintf('%s/%s_%04d.%05d.mat',dout,filebase,iseq,ichunk);
+        save(fout,'ugh','vgh','wgh','pgh');
+      end
+    end
+  end
+end

matlab/read_domain_legacy.m

@@ -0,0 +1,16 @@
+function [a,b,c,d,e,f] = read_domain_legacy(file)
+  % [a,b,c,d,e,f] = read_domain_legacy(file)
+  % Reads domain boundaries from 'current_domain.tmp'
+  % Input
+  %  file         path to 'current_domain.tmp'
+  % Output
+  % a,b,c,d,e,f   domain boundaries
+  
+  border=load(file);
+  a=border(1);
+  b=border(2);
+  c=border(3);
+  d=border(4);
+  e=border(5);
+  f=border(6);
+end

matlab/read_field_chunk_legacy.m

@@ -0,0 +1,157 @@
+function [data,ib,jb,kb,nxl,nyl,nzl,ighost] = read_field_chunk_legacy(file,fproc,ighost,field,varargin)
+    % [data,ib,jb,kb,nxl,nyl,nzl,ighost] = read_field_chunk_legacy(file,fproc,ighost,field,varargin)
+    % Reads arbitrary field from one processor chunk.
+    % Input
+    %   file                path to chunk
+    %   fproc               path to 'current_proc_grid.tmp'
+    %   ighost              data written with ghost cells?
+    %   field               field to be read 
+    %                         {'u','v','w','p','s1','s2',...}
+    % ? ghost               keep ghost cells? (default: 1)
+    % ? verbosity           verbose output? (default: 0)
+    % ? nparam              number of parameters in header (default: 10)
+    % ? precision           precision of data (default: 'float64')
+    %                        'float32'
+    %                        'float64'
+    % ? endian              endianess of file (default: 'a')
+    % ? reclen              FORTRAN record length in bytes (default: 4)
+    % Output
+    %   data                partial field with dim(nxl+2*ighost,nyl+2*ighost,nzl+2*ighost)
+    %   ib,jb,kb            global index of first grid point of the partial field w/o ghost cells
+    %   nxl,nyl,nzl         local field size w/o ghost cells
+    %   ighost              ghost cell flag
+
+    % Parse optional input arguments
+    par = inputParser;
+    addParamValue(par,'ghost',1,@isnumeric);
+    addParamValue(par,'verbosity',0,@isnumeric);
+    addParamValue(par,'nparam',10,@isnumeric);
+    addParamValue(par,'precision','float64',@ischar);
+    addParamValue(par,'endian','a',@ischar);
+    addParamValue(par,'reclen',4,@isnumeric);
+    parse(par,varargin{:});
+    keepghost = par.Results.ghost;
+    verbosity = par.Results.verbosity;
+    nparam    = par.Results.nparam;
+    precision = par.Results.precision;
+    endian    = par.Results.endian;
+    reclen    = par.Results.reclen;
+    
+    % Parse field
+    if ischar(field)
+      switch field(1)
+      case 'u'; ifield=1; fbase='uvwp';
+      case 'v'; ifield=2; fbase='uvwp';
+      case 'w'; ifield=3; fbase='uvwp';
+      case 'p'; ifield=4; fbase='uvwp';
+      case 's'; ifield=str2double(field(2:end)); fbase='scal';
+      end
+    else
+      error('field must be of type char');
+    end
+    
+    % First read processor grid from tmp file
+    [ibegu,iendu,jbegu,jendu,kbegu,kendu,...
+     ibegv,iendv,jbegv,jendv,kbegv,kendv,...
+     ibegw,iendw,jbegw,jendw,kbegw,kendw,...
+     ibegp,iendp,jbegp,jendp,kbegp,kendp,...
+     nxprocs,nyprocs,nzprocs] = read_procgrid_legacy(fproc);
+
+    % Determine processor rank from filename
+    [idxbeg,idxend] = regexp(file,'[\d]+$');
+    if isempty(idxbeg)
+      error('Invalid file: does not contain rank. %s',fuvwp);
+    end
+    iproc = str2double(file(idxbeg:idxend));
+    
+    % Determine local array sizes
+    ixproc = floor(iproc/(nyprocs*nzprocs));
+    iyproc = mod(floor(iproc/nzprocs),nyprocs);
+    izproc = mod(iproc,nzprocs);
+    
+    ibu  = ibegu(ixproc+1);
+    jbu  = jbegu(iyproc+1);
+    kbu  = kbegu(izproc+1);
+    nxul = iendu(ixproc+1)-ibu+1;
+    nyul = jendu(iyproc+1)-jbu+1;
+    nzul = kendu(izproc+1)-kbu+1;
+    
+    ibv  = ibegv(ixproc+1);
+    jbv  = jbegv(iyproc+1);
+    kbv  = kbegv(izproc+1);
+    nxvl = iendv(ixproc+1)-ibv+1;
+    nyvl = jendv(iyproc+1)-jbv+1;
+    nzvl = kendv(izproc+1)-kbv+1;
+    
+    ibw  = ibegw(ixproc+1);
+    jbw  = jbegw(iyproc+1);
+    kbw  = kbegw(izproc+1);
+    nxwl = iendw(ixproc+1)-ibw+1;
+    nywl = jendw(iyproc+1)-jbw+1;
+    nzwl = kendw(izproc+1)-kbw+1;
+    
+    ibp  = ibegp(ixproc+1);
+    jbp  = jbegp(iyproc+1);
+    kbp  = kbegp(izproc+1);
+    nxpl = iendp(ixproc+1)-ibp+1;
+    nypl = jendp(iyproc+1)-jbp+1;
+    nzpl = kendp(izproc+1)-kbp+1;
+    
+    % Convert 'precision' into bytes
+    switch precision
+      case 'float32'; nprecision = 4;
+      case 'float64'; nprecision = 8;
+      otherwise; error('Invalid precision: %s',precision)
+    end
+    
+    % Open uvwp file
+    fid = fopen(file,'r',endian);
+    if fid<0
+      error('File not found: %s',file);
+    end
+    
+    % Determine header size
+    nheader = 4*reclen+(4+nparam)*nprecision;
+    
+    % Skip datasets until correct one is reached
+    for iset=1:ifield
+      switch fbase
+      case 'uvwp'
+        if iset==1; nxl=nxul; nyl=nyul; nzl=nzul; end
+        if iset==2; nxl=nxvl; nyl=nyvl; nzl=nzvl; end
+        if iset==3; nxl=nxwl; nyl=nywl; nzl=nzwl; end
+        if iset==4; nxl=nxpl; nyl=nypl; nzl=nzpl; end
+      case 'scal'
+        nxl=nxpl; nyl=nypl; nzl=nzpl;
+      end
+      ndata = (nxl+2*ighost)*(nyl+2*ighost)*(nzl+2*ighost);
+      if iset~=ifield
+        nskip = nheader+2*reclen+ndata*nprecision;
+        fseek(fid,nskip,'cof');
+      else
+        fseek(fid,nheader+reclen,'cof');
+        data = fread(fid,ndata,precision);
+        data = reshape(data,nxl+2*ighost,nyl+2*ighost,nzl+2*ighost);
+      end
+    end
+      
+    % Get processor bounds
+    switch fbase
+    case 'uvwp'
+      if ifield==1; ib=ibu; jb=jbu; kb=kbu; nxl=nxul; nyl=nyul; nzl=nzul; end
+      if ifield==2; ib=ibv; jb=jbv; kb=kbv; nxl=nxvl; nyl=nyvl; nzl=nzvl; end
+      if ifield==3; ib=ibw; jb=jbw; kb=kbw; nxl=nxwl; nyl=nywl; nzl=nzwl; end
+      if ifield==4; ib=ibp; jb=jbp; kb=kbp; nxl=nxpl; nyl=nypl; nzl=nzpl; end
+    case 'scal'
+      ib=ibp; jb=jbp; kb=kbp; nxl=nxpl; nyl=nypl; nzl=nzpl;
+    end
+    
+    % Close file
+    fclose(fid);
+        
+    % Remove ghosts if necessary
+    if ighost && ~keepghost
+      data = data(2:end-1,2:end-1,2:end-1);
+      ighost = 0;
+    end
+end

matlab/read_field_chunk_ucf.m

@@ -0,0 +1,87 @@
+function [data,ib,jb,kb,nxl,nyl,nzl,ighost] = read_field_chunk_ucf(file,field,varargin)
+    % [data,ib,jb,kb,nxl,nyl,nzl,ighost] = read_field_chunk_ucf(file,field,varargin)
+    % Reads single field from one processor chunk.
+    % Input
+    %   file                file name (if tar-mode: ustar handle)
+    %   field               field to be read 
+    %                         {'u','v','w','p','s1','s2',...}
+    %                         or integer with dataset index (not in tar-mode)
+    % ? step                index of timestep to be read (default: 1)
+    % ? ghost               keep ghost cells? (default: yes)
+    % ? verbosity           verbose output? (default: no)
+    % ? debug               debug output? (default: no)
+    % ? tarmode             read from tar-file? (default: 0)
+    % ? rank                rank of processor, ignored if not in tarmode (default: 0)
+    % Output
+    %   data                partial field with dim(nxl+2*ighost,nyl+2*ighost,nzl+2*ighost)
+    %   ib,jb,kb            global index of first grid point of the partial field w/o ghost cells
+    %   nxl,nyl,nzl         local field size w/o ghost cells
+    %   ighost              ghost cell flag
+
+    % Parse optional input arguments
+    par = inputParser;
+    addParamValue(par,'step',1,@isnumeric);
+    addParamValue(par,'ghost',1,@isnumeric);
+    addParamValue(par,'verbosity',0,@isnumeric);
+    addParamValue(par,'debug',0,@isnumeric);
+    addParamValue(par,'tarmode',0,@isnumeric);  % deprecated
+    addParamValue(par,'rank',0,@isnumeric);
+    parse(par,varargin{:});
+    istep     = par.Results.step;
+    keepghost = par.Results.ghost;
+    
+    % Parse field
+    if ischar(field)
+      switch field(1)
+      case 'u'; ifield=1; fbase='uvwp';
+      case 'v'; ifield=2; fbase='uvwp';
+      case 'w'; ifield=3; fbase='uvwp';
+      case 'p'; ifield=4; fbase='uvwp';
+      case 's'; ifield=str2double(field(2:end)); fbase='scal';
+      end
+    elseif isnumeric(field)
+      if par.Results.tarmode
+        error('field cannot be numeric, if tar-mode is used.');
+      end
+      ifield=field; fbase='';
+    else
+      error('field must either be numeric or string.');
+    end
+    
+    % Open file
+    obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug);
+    switch class(file)
+    case 'char'
+      obj.open(file);
+    case {'ustar','ucfmulti'}
+      subfile = sprintf('%s.%05d',fbase,par.Results.rank);
+      obj.opentar(file.pointer(subfile));
+    otherwise
+      error('Input file type not supported: %s',class(file));
+    end
+    
+    % Read raw data
+    if ~obj.validateType('field')
+      error('read error: no field data.'); 
+    end
+    [data,params] = obj.readSet(istep,ifield);
+    params = cast(params,'double');
+    ighost = params(1);
+    ib     = params(2);
+    jb     = params(3);
+    kb     = params(4);
+    nxl    = params(5);
+    nyl    = params(6);
+    nzl    = params(7);
+    nx     = params(8);
+    ny     = params(9);
+    nz     = params(10);
+    data   = reshape(data,nxl+2*ighost,nyl+2*ighost,nzl+2*ighost);
+    if ighost && ~keepghost
+      data = data(2:end-1,2:end-1,2:end-1);
+      ighost = 0;
+    end
+
+    % Close UCF file
+    obj.close();
+end

matlab/read_grid_legacy.m

@@ -0,0 +1,21 @@
+function [xu,yu,zu,xv,yv,zv,xw,yw,zw,xp,yp,zp,...
+  nxu,nyu,nzu,nxv,nyv,nzv,nxw,nyw,nzw,nxp,nyp,nzp] = read_grid_legacy(file,a,b,c,d,e,f,x_periodic,y_periodic,z_periodic)
+  % [xu,yu,zu,xv,yv,zv,xw,yw,zw,xp,yp,zp,...
+  %  nxu,nyu,nzu,nxv,nyv,nzv,nxw,nyw,nzw,nxp,nyp,nzp] = read_grid_legacy(file,a,b,c,d,e,f,x_periodic,y_periodic,z_periodic)
+  % Reconstructs grid from 'current_points.tmp'.
+  % Input
+  %   file            path to 'current_points.tmp'
+  %   a,b,c,d,e,f     domain boundaries
+  %   x/y/z_periodic  periodicity
+  % Output
+  % xu,yu,zu,...      grid vectors
+  % nxu,nyu,nzu,...   number of points
+  
+  nn=load(file);
+  nxp=nn(1);
+  nyp=nn(2);
+  nzp=nn(3);
+  [nxu,nyu,nzu,nxv,nyv,nzv,nxw,nyw,nzw,...
+       xu,yu,zu,xv,yv,zv,xw,yw,zw,xp,yp,zp]=...
+              generate_grid(a,b,c,d,e,f,nxp,nyp,nzp,x_periodic,y_periodic,z_periodic);
+end

matlab/read_grid_ucf.m

@@ -0,0 +1,93 @@
+function [xu,yu,zu,xv,yv,zv,xw,yw,zw,xp,yp,zp,xs,ys,zs] = read_grid_ucf(file,varargin)
+    % [xu,yu,zu,xv,yv,zv,xw,yw,zw,xp,yp,zp,xs,ys,zs] = read_grid_ucf(file,varargin)
+    % Reads staggered grid from ucf file.
+    % Input
+    %   file              file to be read
+    % ? verbosity         verbose output? (default: 0)
+    % ? debug             debug output? (default: 0)
+    % ? tarmode           read from tar-file? (default: 0)
+    % Output
+    %   xu,yu,zu          velocity grid u
+    %   xv,yv,zv          velocity grid v
+    %   xw,yw,zw          velocity grid w
+    %   xp,yp,zp          pressure grid
+    %   xs,ys,zs          scalar grid
+
+    % Parse optional input arguments
+    par = inputParser;
+    addParamValue(par,'verbosity',0,@isnumeric);
+    addParamValue(par,'debug',0,@isnumeric);
+    addParamValue(par,'tarmode',0,@isnumeric);  % deprecated
+    parse(par,varargin{:});
+    
+    % Open UCF file and read
+    obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug);
+    switch class(file)
+    case 'char'
+      obj.open(file);
+    case {'ustar','ucfmulti'}
+      ptr = file.pointer('grid.bin');
+      obj.opentar(ptr);
+    otherwise
+      error('Input file type not supported: %s',class(file));
+    end
+    
+    % Define sets to be read
+    isDualMesh=(obj.UCFVersion>=2);
+    if isDualMesh
+      sets = {'u','v','w','p','s'};
+    else
+      sets = {'u','v','w','p'};
+    end
+    nset = numel(sets);
+  
+    % Read raw data
+    x = cell(1,nset);
+    y = cell(1,nset);
+    z = cell(1,nset);
+    for iset=1:nset
+      [data,params] = obj.readSet(1,iset);
+      params = cast(params,'double');
+      nx{iset} = params(1);
+      ny{iset} = params(2);
+      nz{iset} = params(3);
+      x{iset}  = data(1:nx{iset});
+      y{iset}  = data(nx{iset}+1:nx{iset}+ny{iset});
+      z{iset}  = data(nx{iset}+ny{iset}+1:nx{iset}+ny{iset}+nz{iset});
+    end
+    
+    % Close UCF file
+    obj.close();
+    
+    % Reassign content from loop
+    iset = find(strcmp(sets,'u'));
+    xu = x{iset};
+    yu = y{iset};
+    zu = z{iset};
+    
+    iset = find(strcmp(sets,'v'));
+    xv = x{iset};
+    yv = y{iset};
+    zv = z{iset};
+    
+    iset = find(strcmp(sets,'w'));
+    xw = x{iset};
+    yw = y{iset};
+    zw = z{iset};
+    
+    iset = find(strcmp(sets,'p'));
+    xp = x{iset};
+    yp = y{iset};
+    zp = z{iset};
+    
+    if isDualMesh
+      iset = find(strcmp(sets,'s'));
+      xs = x{iset};
+      ys = y{iset};
+      zs = z{iset};
+    else
+      xs = xp;
+      ys = yp;
+      zs = zp;
+    end
+end

matlab/read_parameters_ucf.m

@@ -0,0 +1,32 @@
+function [params] = read_parameters_ucf(file,varargin)
+  % [params] = read_parameters_ucf(file)
+  % Reads the entire parameters_XXXX.asc as written by UCF file format
+  % into a struct.
+  % Input
+  %   file          file name (if tar-mode: ustar handle)
+  % ? verbosity     verbose screen output? (default: 0)
+  % ? tarmode       read from tar-file? (default: 0)
+  % Output
+  %   params        struct with content of parameters_XXXX.asc
+  
+  % Parse optional input arguments
+  par = inputParser;
+  addParamValue(par,'verbosity',0,@isnumeric);
+  addParamValue(par,'tarmode',0,@isnumeric);  % deprecated
+  parse(par,varargin{:});
+  % Open file
+  obj = ini('verbosity',par.Results.verbosity);
+    switch class(file)
+    case 'char'
+      obj.open(file);
+    case {'ustar','ucfmulti'}
+      ptr = file.pointer('parameters.asc');
+      obj.opentar(ptr);
+    otherwise
+      error('Input file type not supported: %s',class(file));
+    end
+  % Get parsed content
+  params = obj.getContent();
+  % Close file
+  obj.close();
+end

matlab/read_particle_balancing_ucf.m

@@ -0,0 +1,47 @@
+function [iproc,nmaster,nslave,stime] = read_particle_balancing_ucf(file,varargin)
+    % [pp,col] = read_particles_ucf(file,varargin)
+    % Reads particle data from UCF file
+    % Input
+    %   file              file to be read
+    % Optional input (key/value pair)
+    %   timestep          timestep to be read (default: all)
+    %   verbosity         verbose output? (default: no)
+    %   debug             debug output? (default: no)
+    % Output
+    %   pp                particle balancing data
+    %   stime             simulation time with dim(1,nt)
+    
+    % Parse optional input arguments
+    par = inputParser;
+    addParamValue(par,'timestep',1,@isnumeric);
+    addParamValue(par,'verbosity',0,@isnumeric);
+    addParamValue(par,'debug',0,@isnumeric);
+    parse(par,varargin{:});
+    istep = par.Results.timestep;
+    
+    % Open file
+    obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug);
+    obj.open(file);
+ 
+    % Get range of steps to be read
+    if istep>obj.getNumTimesteps()
+      error('Timestep out of range.');
+    end
+    
+    % Get simulation time
+    stime = obj.getSimulationTime(istep);
+    
+    % Read data
+    [data,params] = obj.readSet(istep,1);
+    params = cast(params,'double');
+    nxprocs = params(1);
+    nyprocs = params(2);
+    nzprocs = params(3);
+    data    = cast(reshape(data,3,nxprocs*nyprocs*nzprocs),'double');
+    iproc   = data(1,:);
+    nmaster = data(2,:);
+    nslave  = data(3,:);
+    
+    % Close file
+    obj.close();
+end

matlab/read_particles_ucf.m

@@ -0,0 +1,101 @@
+function [pp,col,stime] = read_particles_ucf(file,varargin)
+    % [pp,col] = read_particles_ucf(file,varargin)
+    % Reads particle data from UCF file
+    % Input
+    %   file              file name (if tar-mode: ustar handle)
+    % ? step              timestep to be read (default: 0 [all])
+    % ? format            MATLAB format (default: 'array')
+    %                       'array'     plain array with dim(ncol.np,nt)
+    %                       'struct'    structure array with short fieldnames
+    %                       'paraview'  structure array with H5Part fieldnames
+    %                       'cell'      cell array with dim(1,nt) and plain arrays with dim(ncol,np) inside
+    % ? verbosity         verbose output? (default: no)
+    % ? debug             debug output? (default: no)
+    % ? tarmode           read from tar-file? (default: 0)
+    % ? append            read an append file? Only matters for tar archives (default: 0)
+    % Output
+    %   pp                particle data in specified format
+    %   col               container map 'char'->'double' which maps quantity names to column index
+    %   stime             simulation time with dim(1,nt)
+    
+    % Parse optional input arguments
+    par = inputParser;
+    addParamValue(par,'step',0,@isnumeric);
+    addParamValue(par,'format','array',@ischar);
+    addParamValue(par,'verbosity',0,@isnumeric);
+    addParamValue(par,'debug',0,@isnumeric);
+    addParamValue(par,'tarmode',0,@isnumeric);  % deprecated (automatically checked now)
+    addParamValue(par,'append',0,@isnumeric);
+    parse(par,varargin{:});
+    istep     = par.Results.step;
+    mlformat  = par.Results.format;
+    isappend  = par.Results.append;
+    
+    % Open file
+    obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug);
+    switch class(file)
+    case 'char'
+      obj.open(file);
+    case {'ustar','ucfmulti'}
+      if isappend
+        ptr = file.pointer('append_particles.bin');
+      else
+        ptr = file.pointer('particles.bin');
+      end
+      obj.opentar(ptr);
+    otherwise
+      error('Input file type not supported: %s',class(file));
+    end
+
+    if ~obj.validateType('particle')
+      error('read error: no particle data.'); 
+    end
+ 
+    % Get range of steps to be read
+    if istep==0
+      steps = uint32(1:obj.getNumTimesteps());
+    else
+      steps = uint32(istep);
+    end
+    nstepout = numel(steps);
+    
+    % Get simulation time
+    stime = obj.getSimulationTime();
+    stime = stime(steps);
+    
+    % Read raw data
+    for istep=nstepout:-1:1
+      [data,params] = obj.readSet(steps(istep),1);
+      params = cast(params,'double');
+      np          = params(1);
+      ncol        = params(2);
+      ncol_rank   = params(3);
+      ncol_hybrid = params(4);
+      ncol_dem    = params(5);
+      ncol_scalar = params(6);
+      data = reshape(data,ncol,np);
+      nscal = ncol_scalar/2;
+      col = colmap_from_flags(ncol_rank,ncol_hybrid,ncol_dem,nscal);
+    
+      % Parse data to specified format
+      switch mlformat
+      case 'struct'
+        pp(istep) = convert_particles_array2struct(data,col);
+      case 'paraview'
+        col = convert_colmap_matlab2paraview(col);
+        pp(istep) = convert_particles_array2struct(data,col);
+      case {'cell','array'}
+        pp{istep} = data;
+      otherwise
+        error('Unknown format: %s',mlformat);
+      end
+    end
+    
+    % Convert cell array to 3-dim array, if mlformat=='array'
+    if strcmp(mlformat,'array')
+      pp = cat(3,pp{:});
+    end
+    
+    % Close file
+    obj.close();
+end

matlab/read_procgrid_legacy.m

@@ -0,0 +1,83 @@
+function [ibegu,iendu,jbegu,jendu,kbegu,kendu,...
+          ibegv,iendv,jbegv,jendv,kbegv,kendv,...
+          ibegw,iendw,jbegw,jendw,kbegw,kendw,...
+          ibegp,iendp,jbegp,jendp,kbegp,kendp,...
+          nxprocs,nyprocs,nzprocs] = read_procgrid_legacy(file)
+    % [ibegu,iendu,jbegu,jendu,kbegu,kendu,...
+    %  ibegv,iendv,jbegv,jendv,kbegv,kendv,...
+    %  ibegw,iendw,jbegw,jendw,kbegw,kendw,...
+    %  ibegp,iendp,jbegp,jendp,kbegp,kendp,...
+    %  nxprocs,nyprocs,nzprocs] = read_procgrid_legacy(file)
+    % Reads processor grids from 'current_proc_grid.tmp'.
+    % Input
+    %   file              path to 'current_proc_grid.tmp'
+    % Output
+    %  ibegu,iendu,jbegu,jendu,kbegu,kendu    processor grid u
+    %  ibegv,iendv,jbegv,jendv,kbegv,kendv    processor grid v
+    %  ibegw,iendw,jbegw,jendw,kbegw,kendw    processor grid w
+    %  ibegp,iendp,jbegp,jendp,kbegp,kendp    processor grid p
+    %  nxprocs,nyprocs,nzprocs                number of processors
+    
+    proc_grid=load(file);
+    nxprocs = proc_grid(1,1);
+    nyprocs = proc_grid(1,2);
+    nzprocs = proc_grid(1,3);
+    
+    ibegu = zeros(nxprocs,1);
+    iendu = zeros(nxprocs,1);
+    ibegv = zeros(nxprocs,1);
+    iendv = zeros(nxprocs,1);
+    ibegw = zeros(nxprocs,1);
+    iendw = zeros(nxprocs,1);
+    ibegp = zeros(nxprocs,1);
+    iendp = zeros(nxprocs,1);
+    
+    jbegu = zeros(nyprocs,1);
+    jendu = zeros(nyprocs,1);
+    jbegv = zeros(nyprocs,1);
+    jendv = zeros(nyprocs,1);
+    jbegw = zeros(nyprocs,1);
+    jendw = zeros(nyprocs,1);
+    jbegp = zeros(nyprocs,1);
+    jendp = zeros(nyprocs,1);
+    
+    kbegu = zeros(nzprocs,1);
+    kendu = zeros(nzprocs,1);
+    kbegv = zeros(nzprocs,1);
+    kendv = zeros(nzprocs,1);
+    kbegw = zeros(nzprocs,1);
+    kendw = zeros(nzprocs,1);
+    kbegp = zeros(nzprocs,1);
+    kendp = zeros(nzprocs,1);
+
+    for ii=1:nxprocs
+      ibegu(ii)=proc_grid(1+ii,1);
+      iendu(ii)=proc_grid(1+ii,2);
+      ibegv(ii)=proc_grid(1+ii,3);
+      iendv(ii)=proc_grid(1+ii,4);
+      ibegw(ii)=proc_grid(1+ii,5);
+      iendw(ii)=proc_grid(1+ii,6);
+      ibegp(ii)=proc_grid(1+ii,7);
+      iendp(ii)=proc_grid(1+ii,8);
+    end
+    for ii=1:nyprocs
+      jbegu(ii)=proc_grid(1+nxprocs+ii,1);
+      jendu(ii)=proc_grid(1+nxprocs+ii,2);
+      jbegv(ii)=proc_grid(1+nxprocs+ii,3);
+      jendv(ii)=proc_grid(1+nxprocs+ii,4);
+      jbegw(ii)=proc_grid(1+nxprocs+ii,5);
+      jendw(ii)=proc_grid(1+nxprocs+ii,6);
+      jbegp(ii)=proc_grid(1+nxprocs+ii,7);
+      jendp(ii)=proc_grid(1+nxprocs+ii,8);
+    end
+    for ii=1:nzprocs
+      kbegu(ii)=proc_grid(1+nxprocs+nyprocs+ii,1);
+      kendu(ii)=proc_grid(1+nxprocs+nyprocs+ii,2);
+      kbegv(ii)=proc_grid(1+nxprocs+nyprocs+ii,3);
+      kendv(ii)=proc_grid(1+nxprocs+nyprocs+ii,4);
+      kbegw(ii)=proc_grid(1+nxprocs+nyprocs+ii,5);
+      kendw(ii)=proc_grid(1+nxprocs+nyprocs+ii,6);
+      kbegp(ii)=proc_grid(1+nxprocs+nyprocs+ii,7);
+      kendp(ii)=proc_grid(1+nxprocs+nyprocs+ii,8);
+    end
+end

matlab/read_procgrid_ucf.m

@@ -0,0 +1,124 @@
+function [ibegu,iendu,jbegu,jendu,kbegu,kendu,...
+          ibegv,iendv,jbegv,jendv,kbegv,kendv,...
+          ibegw,iendw,jbegw,jendw,kbegw,kendw,...
+          ibegp,iendp,jbegp,jendp,kbegp,kendp,...
+          ibegs,iends,jbegs,jends,kbegs,kends] = read_procgrid_ucf(file,varargin)
+    % [ibegu,iendu,jbegu,jendu,kbegu,kendu,...
+    %  ibegv,iendv,jbegv,jendv,kbegv,kendv,...
+    %  ibegw,iendw,jbegw,jendw,kbegw,kendw,...
+    %  ibegp,iendp,jbegp,jendp,kbegp,kendp,...
+    %  ibegs,iends,jbegs,jends,kbegs,kends] = read_procgrid_ucf(file,varargin)
+    % Reads processor grids.
+    % Input
+    %   file              file name (if tar-mode: ustar handle)
+    % ? verbosity         verbose output? (default: 0)
+    % ? tarmode           read from tar-file? (default: 0)
+    % Output
+    %  ibegu,iendu,jbegu,jendu,kbegu,kendu    processor grid u
+    %  ibegv,iendv,jbegv,jendv,kbegv,kendv    processor grid v
+    %  ibegw,iendw,jbegw,jendw,kbegw,kendw    processor grid w
+    %  ibegp,iendp,jbegp,jendp,kbegp,kendp    processor grid p
+    %  ibegs,iends,jbegs,jends,kbegs,kends    processor grid s  
+    
+    % Parse optional input arguments
+    par = inputParser;
+    addParamValue(par,'verbosity',0,@isnumeric);
+    addParamValue(par,'debug',0,@isnumeric);
+    addParamValue(par,'tarmode',0,@isnumeric);
+    parse(par,varargin{:});
+    
+    % Open file
+    obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug);
+    switch class(file)
+    case 'char'
+      obj.open(file);
+    case {'ustar','ucfmulti'}
+      ptr = file.pointer('proc.bin');
+      obj.opentar(ptr);
+    otherwise
+      error('Input file type not supported: %s',class(file));
+    end
+    
+    % Define sets to be read
+    isDualMesh=(obj.UCFVersion>=2);
+    if isDualMesh
+      sets = {'u','v','w','p','s'};
+    else
+      sets = {'u','v','w','p'};
+    end
+    nset = numel(sets);
+    
+    % Read raw data
+    ibeg = cell(1,nset);
+    iend = cell(1,nset);
+    jbeg = cell(1,nset);
+    jend = cell(1,nset);
+    kbeg = cell(1,nset);
+    kend = cell(1,nset);
+    for iset=1:nset
+      [data,params] = obj.readSet(1,iset);
+      params = cast(params,'double');
+      nxprocs = params(1);
+      nyprocs = params(2);
+      nzprocs = params(3);
+      ibeg{iset} = data(1:nxprocs);
+      iend{iset} = data(nxprocs+1:2*nxprocs);
+      jbeg{iset} = data(2*nxprocs+1:2*nxprocs+nyprocs);
+      jend{iset} = data(2*nxprocs+nyprocs+1:2*nxprocs+2*nyprocs);
+      kbeg{iset} = data(2*nxprocs+2*nyprocs+1:2*nxprocs+2*nyprocs+nzprocs);
+      kend{iset} = data(2*nxprocs+2*nyprocs+nzprocs+1:2*nxprocs+2*nyprocs+2*nzprocs);
+    end
+    
+    % Close UCF file
+    obj.close();
+    
+    % Reassign content from loop
+    iset = find(strcmp(sets,'u'));
+    ibegu = cast(ibeg{iset},'double');
+    iendu = cast(iend{iset},'double');
+    jbegu = cast(jbeg{iset},'double');
+    jendu = cast(jend{iset},'double');
+    kbegu = cast(kbeg{iset},'double');
+    kendu = cast(kend{iset},'double');
+    
+    iset = find(strcmp(sets,'v'));
+    ibegv = cast(ibeg{iset},'double');
+    iendv = cast(iend{iset},'double');
+    jbegv = cast(jbeg{iset},'double');
+    jendv = cast(jend{iset},'double');
+    kbegv = cast(kbeg{iset},'double');
+    kendv = cast(kend{iset},'double');
+    
+    iset = find(strcmp(sets,'w'));
+    ibegw = cast(ibeg{iset},'double');
+    iendw = cast(iend{iset},'double');
+    jbegw = cast(jbeg{iset},'double');
+    jendw = cast(jend{iset},'double');
+    kbegw = cast(kbeg{iset},'double');
+    kendw = cast(kend{iset},'double');
+    
+    iset = find(strcmp(sets,'p'));
+    ibegp = cast(ibeg{iset},'double');
+    iendp = cast(iend{iset},'double');
+    jbegp = cast(jbeg{iset},'double');
+    jendp = cast(jend{iset},'double');
+    kbegp = cast(kbeg{iset},'double');
+    kendp = cast(kend{iset},'double');
+    
+    if isDualMesh
+      iset = find(strcmp(sets,'s'));
+      ibegs = cast(ibeg{iset},'double');
+      iends = cast(iend{iset},'double');
+      jbegs = cast(jbeg{iset},'double');
+      jends = cast(jend{iset},'double');
+      kbegs = cast(kbeg{iset},'double');
+      kends = cast(kend{iset},'double');
+    else
+      ibegs = ibegp;
+      iends = iendp;
+      jbegs = jbegp;
+      jends = jendp;
+      kbegs = kbegp;
+      kends = kendp;
+    end
+end

matlab/read_scal_chunk_ucf.m

@@ -0,0 +1,71 @@
+function [s,ib,jb,kb,nxl,nyl,nzl,ighost] = read_scal_chunk_ucf(file,varargin)
+    % [s,ib,jb,kb,nxl,nyl,nzl,ighost] = read_scal_chunk_ucf(file,varargin)
+    % Reads scalar fields from one processor chunk.
+    % Input
+    %   file                file name (if tar-mode: ustar handle)
+    % ? step                index of timestep to be read (default: 1)
+    % ? ghost               keep ghost cells? (default: 1)
+    % ? verbosity           verbose output? (default: 0)
+    % ? debug               debug output? (default: 0)
+    % ? tarmode             read from tar-file? (default: 0)
+    % ? rank                rank of processor, ignored if not in tarmode (default: 0)
+    % Output
+    %   s                   partial fields with dim(nxl+2*ighost,nyl+2*ighost,nzl+2*ighost,nscal)
+    %   ib,jb,kb,...        global index of first grid point of the partial field w/o ghost cells
+    %   nxl,nyl,nzl,...     local field size w/o ghost cells
+    %   ighost              ghost cell flag
+
+    % Parse optional input arguments
+    par = inputParser;
+    addParamValue(par,'step',1,@isnumeric);
+    addParamValue(par,'ghost',1,@isnumeric);
+    addParamValue(par,'verbosity',0,@isnumeric);
+    addParamValue(par,'debug',0,@isnumeric);
+    addParamValue(par,'tarmode',0,@isnumeric);  % deprecated
+    addParamValue(par,'rank',0,@isnumeric);
+    parse(par,varargin{:});
+    istep     = par.Results.step;
+    keepghost = par.Results.ghost;
+  
+    % Open file
+    obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug);
+    switch class(file)
+    case 'char'
+      obj.open(file);
+    case {'ustar','ucfmulti'}
+      subfile = sprintf('scal.%05d',par.Results.rank);
+      obj.opentar(file.pointer(subfile));
+    otherwise
+      error('Input file type not supported: %s',class(file));
+    end
+  
+    if ~obj.validateType('field')
+      error('read error: no field data.'); 
+    end
+    nscal = obj.getNumDatasets(istep);
+
+    % Read raw data
+    s = cell(1,nscal);
+    for iset=1:nscal
+      [data,params] = obj.readSet(istep,iset);
+      params = cast(params,'double');
+      ighost = params(1);
+      ib     = params(2);
+      jb     = params(3);
+      kb     = params(4);
+      nxl    = params(5);
+      nyl    = params(6);
+      nzl    = params(7);
+      s{iset} = reshape(data,nxl+2*ighost,nyl+2*ighost,nzl+2*ighost);
+      if ighost && ~keepghost
+        s{iset} = s{iset}(2:end-1,2:end-1,2:end-1);
+        ighost = 0;
+      end
+    end
+    
+    % Close UCF file
+    obj.close();
+    
+    % Convert cell array to 4-D array
+    s = cat(4,s{:});
+end

matlab/read_scal_complete_ucf.m

@@ -0,0 +1,131 @@
+function [s] = read_scal_complete_ucf(file,varargin)
+    % [s] = read_scal_complete_ucf(file,varargin)
+    % Reads u,v,w,p from all processor chunks and combines them to a single field.
+    % Input
+    %   file              file name (if tar-mode: ustar handle)
+    % ? step              index of timestep to be read (default: 1)
+    % ? verbosity         verbose output? (default: 0)
+    % ? debug             debug output? (default: 0)
+    % ? tarmode           read from tar-file? (default: 0)
+    % Output
+    %   s                 complete scalar fields with dim(nx,ny,nz,nscal)
+
+    % Parse optional input arguments
+    par = inputParser;
+    addParamValue(par,'step',1,@isnumeric);
+    addParamValue(par,'verbosity',0,@isnumeric);
+    addParamValue(par,'debug',0,@isnumeric);
+    addParamValue(par,'tarmode',0,@isnumeric);  % deprecated
+    parse(par,varargin{:});
+    istep = par.Results.step;
+
+    % Open first file
+    obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug);
+    switch class(file)
+    case 'char'
+      [fdir,fbase,fext] = fileparts(file);
+      fname = sprintf('%s/%s.%05d',fdir,fbase,0);
+      obj.open(fname);
+    case {'ustar','ucfmulti'}
+      fname = sprintf('scal.%05d',0);
+      obj.opentar(file.pointer(fname));
+    otherwise
+      error('Input file type not supported: %s',class(file));
+    end
+
+    if ~obj.validateType('field')
+      error('read error: no field data.');
+    end
+    nscal = obj.getNumDatasets(istep);
+
+    % Read raw data
+    q = cell(1,nscal);
+    for iset=1:nscal
+      [data,params] = obj.readSet(istep,iset);
+      params = cast(params,'double');
+      ighost = params(1);
+      ib  = params(2);
+      jb  = params(3);
+      kb  = params(4);
+      nxl = params(5);
+      nyl = params(6);
+      nzl = params(7);
+      nx  = params(8);
+      ny  = params(9);
+      nz  = params(10);
+      q{iset}  = reshape(data,nxl+2*ighost,nyl+2*ighost,nzl+2*ighost);
+      if ighost
+        q{iset} = q{iset}(2:end-1,2:end-1,2:end-1);
+      end
+    end
+
+    % Reassign content from loop and create global arrays
+    s = zeros(nx,ny,nz,nscal,class(q{1}));
+    for iscal=1:nscal
+      s(ib:ib+nxl-1,jb:jb+nyl-1,kb:kb+nzl-1,iscal) = q{iscal};
+    end
+
+    % Close the first file
+    obj.close();
+
+    % Now loop consecutively through files
+    ifile = 1;
+    switch class(file)
+    case 'char'
+      loopCondition = @(x) exist(x,'file');
+      fname = sprintf('%s/%s.%05d',fdir,fbase,ifile);
+    case {'ustar','ucfmulti'}
+      loopCondition = @(x) file.isSubfile(x);
+      fname = sprintf('scal.%05d',ifile);
+    end
+
+    while loopCondition(fname)
+      % Open file
+      obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug);
+      switch class(file)
+      case 'char'
+        obj.open(fname);
+      case {'ustar','ucfmulti'}
+        obj.opentar(file.pointer(fname));
+      end
+
+      if ~obj.validateType('field')
+        error('read error: no field data.');
+      end
+
+      % Read raw data
+      q  = cell(1,nscal);
+      for iset=1:nscal
+        [data,params] = obj.readSet(istep,iset);
+        params = cast(params,'double');
+        ighost = params(1);
+        ib     = params(2);
+        jb     = params(3);
+        kb     = params(4);
+        nxl    = params(5);
+        nyl    = params(6);
+        nzl    = params(7);
+        q{iset} = reshape(data,nxl+2*ighost,nyl+2*ighost,nzl+2*ighost);
+        if ighost
+          q{iset} = q{iset}(2:end-1,2:end-1,2:end-1);
+        end
+      end
+
+      % Reassign content from loop and create global arrays
+      for iscal=1:nscal
+        s(ib:ib+nxl-1,jb:jb+nyl-1,kb:kb+nzl-1,iscal) = q{iscal};
+      end
+
+      % Close file
+      obj.close();
+
+      % Move on to next file
+      ifile = ifile+1;
+      switch class(file)
+      case 'char'
+        fname = sprintf('%s/%s.%05d',fdir,fbase,ifile);
+      case {'ustar','ucfmulti'}
+        fname = sprintf('scal.%05d',ifile);
+      end
+    end
+end

matlab/read_statistics_channel_scal_ucf.m

@@ -0,0 +1,41 @@
+function [tbeg,tend,nstat,sm,ss,su,sv] = read_statistics_channel_scal_ucf(file,varargin)
+
+    % Parse optional input arguments
+    par = inputParser;
+    addParamValue(par,'verbosity',0,@isnumeric);
+    addParamValue(par,'debug',0,@isnumeric);
+    parse(par,varargin{:});
+
+    % Define sets to be read
+    sets = {'um','uu','vv','ww','uv','uub','uvb'};
+    nset = numel(sets);
+
+    % Open UCF file and read
+    obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug);
+    obj.open(file);
+    tend = obj.getSimulationTime(1);
+
+    % Read parameters of first set
+    params = obj.readParameters(1,1);
+
+    % Parse parameters
+    tbeg   = typecast(params(1),'double');
+    nstat  = cast(params(2),'double');
+    ny     = cast(params(3),'double');
+    nscal  = cast(params(4),'double');  % number of scal
+    nset   = cast(params(5),'double');  % number of stats
+
+    sm = zeros(ny,nscal);
+    ss = zeros(ny,nscal);
+    su = zeros(ny,nscal);
+    sv = zeros(ny,nscal);
+    for iscal=1:nscal
+      sm(:,iscal) = obj.readSet(1,(iscal-1)*4+1)/nstat;
+      ss(:,iscal) = obj.readSet(1,(iscal-1)*4+2)/nstat;
+      su(:,iscal) = obj.readSet(1,(iscal-1)*4+3)/nstat;
+      sv(:,iscal) = obj.readSet(1,(iscal-1)*4+4)/nstat;
+    end
+
+    % Close file
+    obj.close();
+end

matlab/read_statistics_channel_ucf.m

@@ -0,0 +1,84 @@
+function [tbeg,tend,nstat,um,uu,vv,ww,uv,uub,uvb,varargout] = read_statistics_channel_ucf(file,varargin)
+    % [tbeg,tend,nstat,um,uu,vv,ww,uv,uub,uvb,varargout] = read_statistics_channel_ucf(file,varargin)
+    % Reads statistics file for channel data in UCF format.
+    % Input
+    %   file              file name (if tar-mode: ustar handle)
+    % ? verbosity         verbose output? (default: no)
+    % ? debug             debug output? (default: no)
+    % ? pure              read an statistics_pure file? Only matters for tar archives (default: 0)
+    % Output
+    %   tbeg              time at beginning of sampling
+    %   tend              time at end of sampling
+    %   nstat             number of samples
+    %   um                mean streamwise velocity
+    %   uu,vv,ww          <u'u'>, <v'v'>, <w'w'>
+    %   uv                <u'v'>
+    %   uub               <uu>
+    %   uvb               <uv>
+    % ? fxp,fyp,fzp       particle force in Eulerian frame
+
+    % Parse optional input arguments
+    par = inputParser;
+    addParamValue(par,'verbosity',0,@isnumeric);
+    addParamValue(par,'debug',0,@isnumeric);
+    addParamValue(par,'pure',0,@isnumeric);
+    parse(par,varargin{:});
+    ispure  = par.Results.pure;
+
+    % Define sets to be read
+    sets = {'um','uu','vv','ww','uv','uub','uvb'};
+    nset = numel(sets);
+
+    % Open UCF file and read
+    obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug);
+    switch class(file)
+    case 'char'
+      obj.open(file);
+    case {'ustar','ucfmulti'}
+      if ispure
+        ptr = file.pointer('statistics_pure.bin');
+      else
+        ptr = file.pointer('statistics.bin');
+      end
+      obj.opentar(ptr);
+    otherwise
+      error('Input file type not supported: %s',class(file));
+    end
+    tend = obj.getSimulationTime();
+
+    % Read first dataset
+    [um,params] = obj.readSet(1,1);
+
+    % Parse parameters
+    tbeg   = typecast(params(1),'double');
+    nstat  = cast(params(2),'double');
+    nset1  = cast(params(4),'double');  % channel stats
+    nset2  = cast(params(5),'double');  % force eulerian
+
+    if nset1~=7
+      error('Invalid file.');
+    end
+    um  = um/nstat;
+    uu  = obj.readSet(1,2)/nstat;
+    vv  = obj.readSet(1,3)/nstat;
+    ww  = obj.readSet(1,4)/nstat;
+    uv  = obj.readSet(1,5)/nstat;
+    uub = obj.readSet(1,6)/nstat;
+    uvb = obj.readSet(1,7)/nstat;
+
+    % If eulerian force data, read that too
+    if nset2==3
+      if par.Results.verbosity
+        fprintf('Eulerian force data found.\n');
+      end
+      fxp = obj.readSet(1,8);
+      fyp = obj.readSet(1,9);
+      fzp = obj.readSet(1,10);
+      varargout{1} = fxp/nstat;
+      varargout{2} = fyp/nstat;
+      varargout{3} = fzp/nstat;
+    end
+
+    % Close file
+    obj.close();
+end

matlab/read_uvwp_chunk_legacy.m

@@ -0,0 +1,149 @@
+function [u,ibu,jbu,kbu,nxul,nyul,nzul,...
+    v,ibv,jbv,kbv,nxvl,nyvl,nzvl,...
+    w,ibw,jbw,kbw,nxwl,nywl,nzwl,...
+    p,ibp,jbp,kbp,nxpl,nypl,nzpl,ighost] = read_uvwp_chunk_legacy(fuvwp,fproc,ighost,varargin)
+    % [u,ibu,jbu,kbu,nxul,nyul,nzul,...
+    %  v,ibv,jbv,kbv,nxvl,nyvl,nzvl,...
+    %  w,ibw,jbw,kbw,nxwl,nywl,nzwl,...
+    %  p,ibp,jbp,kbp,nxpl,nypl,nzpl,ighost] = read_uvwp_chunk_legacy(fuvwp,fproc,ighost,varargin)
+    % Reads u,v,w,p from one processor chunk.
+    % Input
+    %   fuvwp               path to uvwp chunk
+    %   fproc               path to 'current_proc_grid.tmp'
+    %   ighost              data written with ghost cells?
+    % ? ghost               keep ghost cells? (default: 1)
+    % ? verbosity           verbose output? (default: 0)
+    % ? nparam              number of parameters in header (default: 10)
+    % ? precision           precision of data (default: 'float64')
+    %                        'float32'
+    %                        'float64'
+    % ? endian              endianess of file (default: 'a')
+    % ? reclen              FORTRAN record length in bytes (default: 4)
+    % Output
+    %   u,v,w,p             partial fields with dim(nxl,nyl,nzl)+2*ighost
+    %   ibu,jbu,kbu,...     global index of first grid point of the partial field w/o ghost cells
+    %   nxul,nyul,nzul,...  local field size w/o ghost cells
+    %   ighost              ghost cell flag
+
+    % Parse optional input arguments
+    par = inputParser;
+    addParamValue(par,'ghost',1,@isnumeric);
+    addParamValue(par,'verbosity',0,@isnumeric);
+    addParamValue(par,'nparam',10,@isnumeric);
+    addParamValue(par,'precision','float64',@ischar);
+    addParamValue(par,'endian','a',@ischar);
+    addParamValue(par,'reclen',4,@isnumeric);
+    parse(par,varargin{:});
+    keepghost = par.Results.ghost;
+    verbosity = par.Results.verbosity;
+    nparam    = par.Results.nparam;
+    precision = par.Results.precision;
+    endian    = par.Results.endian;
+    reclen    = par.Results.reclen;
+    
+    % First read processor grid from tmp file
+    [ibegu,iendu,jbegu,jendu,kbegu,kendu,...
+     ibegv,iendv,jbegv,jendv,kbegv,kendv,...
+     ibegw,iendw,jbegw,jendw,kbegw,kendw,...
+     ibegp,iendp,jbegp,jendp,kbegp,kendp,...
+     nxprocs,nyprocs,nzprocs] = read_procgrid_legacy(fproc);
+
+    % Determine processor rank from filename
+    [idxbeg,idxend] = regexp(fuvwp,'[\d]+$');
+    if isempty(idxbeg)
+      error('Invalid file: does not contain rank. %s',fuvwp);
+    end
+    iproc = str2double(fuvwp(idxbeg:idxend));
+    
+    % Determine local array sizes
+    ixproc = floor(iproc/(nyprocs*nzprocs));
+    iyproc = mod(floor(iproc/nzprocs),nyprocs);
+    izproc = mod(iproc,nzprocs);
+    
+    ibu  = ibegu(ixproc+1);
+    jbu  = jbegu(iyproc+1);
+    kbu  = kbegu(izproc+1);
+    nxul = iendu(ixproc+1)-ibu+1;
+    nyul = jendu(iyproc+1)-jbu+1;
+    nzul = kendu(izproc+1)-kbu+1;
+    
+    ibv  = ibegv(ixproc+1);
+    jbv  = jbegv(iyproc+1);
+    kbv  = kbegv(izproc+1);
+    nxvl = iendv(ixproc+1)-ibv+1;
+    nyvl = jendv(iyproc+1)-jbv+1;
+    nzvl = kendv(izproc+1)-kbv+1;
+    
+    ibw  = ibegw(ixproc+1);
+    jbw  = jbegw(iyproc+1);
+    kbw  = kbegw(izproc+1);
+    nxwl = iendw(ixproc+1)-ibw+1;
+    nywl = jendw(iyproc+1)-jbw+1;
+    nzwl = kendw(izproc+1)-kbw+1;
+    
+    ibp  = ibegp(ixproc+1);
+    jbp  = jbegp(iyproc+1);
+    kbp  = kbegp(izproc+1);
+    nxpl = iendp(ixproc+1)-ibp+1;
+    nypl = jendp(iyproc+1)-jbp+1;
+    nzpl = kendp(izproc+1)-kbp+1;
+    
+    % Convert 'precision' into bytes
+    switch precision
+      case 'float32'; nprecision = 4;
+      case 'float64'; nprecision = 8;
+      otherwise; error('Invalid precision: %s',precision)
+    end
+    
+    % Open uvwp file
+    fid = fopen(fuvwp,'r',endian);
+    if fid<0
+      error('File not found: %s',fuvwp);
+    end
+    
+    % Skip headers and read fields
+    nheader = 4*reclen+(4+nparam)*nprecision;
+    fseek(fid,nheader+reclen,'bof');
+    nx = (nxul+2*ighost);
+    ny = (nyul+2*ighost);
+    nz = (nzul+2*ighost);
+    u = fread(fid,nx*ny*nz,precision);
+    u = reshape(u,nx,ny,nz);
+    fseek(fid,nheader+2*reclen,'cof');
+    nx = (nxvl+2*ighost);
+    ny = (nyvl+2*ighost);
+    nz = (nzvl+2*ighost);
+    v = fread(fid,nx*ny*nz,precision);
+    v = reshape(v,nx,ny,nz);
+    fseek(fid,nheader+2*reclen,'cof');
+    nx = (nxwl+2*ighost);
+    ny = (nywl+2*ighost);
+    nz = (nzwl+2*ighost);
+    w = fread(fid,nx*ny*nz,precision);
+    w = reshape(w,nx,ny,nz);
+    fseek(fid,nheader+2*reclen,'cof');
+    nx = (nxpl+2*ighost);
+    ny = (nypl+2*ighost);
+    nz = (nzpl+2*ighost);
+    p = fread(fid,nx*ny*nz,precision);
+    p = reshape(p,nx,ny,nz);
+    fseek(fid,reclen,'cof');
+    
+    % Test for eof
+    fread(fid,1,'char');
+    if ~feof(fid)
+      warning('End-of-file not reached: data might be corrupt.');
+    end
+    
+    % Close file
+    fclose(fid);
+        
+    % Remove ghosts if necessary
+    if ighost && ~keepghost
+      u = u(2:end-1,2:end-1,2:end-1);
+      v = v(2:end-1,2:end-1,2:end-1);
+      w = w(2:end-1,2:end-1,2:end-1);
+      p = p(2:end-1,2:end-1,2:end-1);
+      ighost = 0;
+    end
+end

matlab/read_uvwp_chunk_ucf.m

@@ -0,0 +1,116 @@
+function [u,ibu,jbu,kbu,nxul,nyul,nzul,...
+    v,ibv,jbv,kbv,nxvl,nyvl,nzvl,...
+    w,ibw,jbw,kbw,nxwl,nywl,nzwl,...
+    p,ibp,jbp,kbp,nxpl,nypl,nzpl,ighost] = read_uvwp_chunk_ucf(file,varargin)
+    % [u,ibu,jbu,kbu,nxul,nyul,nzul,...
+    %  v,ibv,jbv,kbv,nxvl,nyvl,nzvl,...
+    %  w,ibw,jbw,kbw,nxwl,nywl,nzwl,...
+    %  p,ibp,jbp,kbp,nxpl,nypl,nzpl,ighost] = read_uvwp_chunk_ucf(file,varargin)
+    % Reads u,v,w,p from one processor chunk.
+    % Input
+    %   file                file name (if tar-mode: ustar handle)
+    % ? step                index of timestep to be read (default: 1)
+    % ? ghost               keep ghost cells? (default: 1)
+    % ? verbosity           verbose output? (default: 0)
+    % ? debug               debug output? (default: 0)
+    % ? tarmode             read from tar-file? (default: 0)
+    % ? rank                rank of processor, ignored if not in tarmode (default: 0)
+    % Output
+    %   u,v,w,p             partial fields with dim(nxl,nyl,nzl)+2*ighost
+    %   ibu,jbu,kbu,...     global index of first grid point of the partial field w/o ghost cells
+    %   nxul,nyul,nzul,...  local field size w/o ghost cells
+    %   ighost              ghost cell flag
+
+    % Parse optional input arguments
+    par = inputParser;
+    addParamValue(par,'step',1,@isnumeric);
+    addParamValue(par,'ghost',1,@isnumeric);
+    addParamValue(par,'verbosity',0,@isnumeric);
+    addParamValue(par,'debug',0,@isnumeric);
+    addParamValue(par,'tarmode',0,@isnumeric);  % deprecated
+    addParamValue(par,'rank',0,@isnumeric);
+    parse(par,varargin{:});
+    istep     = par.Results.step;
+    keepghost = par.Results.ghost;
+
+    % Define sets to be read
+    sets = {'u','v','w','p'};
+    nset = numel(sets);
+
+    % Open file
+    obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug);
+    switch class(file)
+    case 'char'
+      obj.open(file);
+    case {'ustar','ucfmulti'}
+      subfile = sprintf('uvwp.%05d',par.Results.rank);
+      obj.opentar(file.pointer(subfile));
+    otherwise
+      error('Input file type not supported: %s',class(file));
+    end
+
+    % Read raw data
+    ib   = cell(1,nset);
+    jb   = cell(1,nset);
+    kb   = cell(1,nset);
+    nxl  = cell(1,nset);
+    nyl  = cell(1,nset);
+    nzl  = cell(1,nset);
+    q    = cell(1,nset);
+    for iset=1:nset
+      [data,params] = obj.readSet(istep,iset);
+      params = cast(params,'double');
+      ighost    = params(1);
+      ib{iset}  = params(2);
+      jb{iset}  = params(3);
+      kb{iset}  = params(4);
+      nxl{iset} = params(5);
+      nyl{iset} = params(6);
+      nzl{iset} = params(7);
+      q{iset} = reshape(data,nxl{iset}+2*ighost,nyl{iset}+2*ighost,nzl{iset}+2*ighost);
+      if ighost && ~keepghost
+        q{iset} = q{iset}(2:end-1,2:end-1,2:end-1);
+        ighost = 0;
+      end
+    end
+
+    % Close UCF file
+    obj.close();
+
+    % Reassign content from loop
+    iset = find(strcmp(sets,'u'));
+    u    = q{iset};
+    ibu  = ib{iset};
+    jbu  = jb{iset};
+    kbu  = kb{iset};
+    nxul = nxl{iset};
+    nyul = nyl{iset};
+    nzul = nzl{iset};
+
+    iset = find(strcmp(sets,'v'));
+    v    = q{iset};
+    ibv  = ib{iset};
+    jbv  = jb{iset};
+    kbv  = kb{iset};
+    nxvl = nxl{iset};
+    nyvl = nyl{iset};
+    nzvl = nzl{iset};
+
+    iset = find(strcmp(sets,'w'));
+    w    = q{iset};
+    ibw  = ib{iset};
+    jbw  = jb{iset};
+    kbw  = kb{iset};
+    nxwl = nxl{iset};
+    nywl = nyl{iset};
+    nzwl = nzl{iset};
+
+    iset = find(strcmp(sets,'p'));
+    p    = q{iset};
+    ibp  = ib{iset};
+    jbp  = jb{iset};
+    kbp  = kb{iset};
+    nxpl = nxl{iset};
+    nypl = nyl{iset};
+    nzpl = nzl{iset};
+end

matlab/read_uvwp_complete_legacy.m

@@ -0,0 +1,72 @@
+function [u,v,w,p] = read_uvwp_complete_legacy(wkdir,ifield,nmyid,...
+                                            x_periodic,y_periodic,z_periodic,...
+                                            ighost,varargin)
+
+
+par = inputParser;
+addParameter(par,'resdir',[wkdir 'result/']);
+
+parse(par,varargin{:});
+keepghost = 0; % see ToDo
+resdir    = par.Results.resdir;
+
+% tmp files with simulation parameters
+fdomain = [wkdir 'current_domain.tmp'];
+fproc   = [wkdir 'current_proc_grid.tmp'];
+fpoints = [wkdir 'current_points.tmp'];
+
+% read domain
+[a,b,c,d,e,f]   = read_domain_legacy(fdomain);
+
+% load the grid
+[ xu,yu,zu,xv,yv,zv   ,...
+  xw,yw,zw,xp,yp,zp   ,...
+  nxu,nyu,nzu,nxv,nyv ,...
+  nzv,nxw,nyw,nzw,nxp ,...
+  nyp,nzp] = read_grid_legacy(fpoints,a,b,c,d,e,f,...
+                              x_periodic,y_periodic,z_periodic);
+
+% initialize zero arrays
+u = zeros(nxu,nyu,nzu);
+v = zeros(nxv,nyv,nzv);
+w = zeros(nxw,nyw,nzw);
+p = zeros(nxp,nyp,nzp);
+
+for myid = 0:nmyid-1
+
+  fuvwp=[resdir 'uvwp_' sprintf('%04d',ifield) '.' sprintf('%05d',myid)];
+
+  [utmp,ibu,jbu,kbu,nxul,nyul,nzul,...
+   vtmp,ibv,jbv,kbv,nxvl,nyvl,nzvl,...
+   wtmp,ibw,jbw,kbw,nxwl,nywl,nzwl,...
+   ptmp,ibp,jbp,kbp,nxpl,nypl,nzpl,~] = ...
+                read_uvwp_chunk_legacy(fuvwp,fproc,ighost,'ghost',keepghost);
+
+  ieu = ibu+nxul-1;  iev = ibv+nxvl-1; iew = ibw+nxwl-1; iep = ibp+nxpl-1;
+  jeu = jbu+nyul-1;  jev = jbv+nyvl-1; jew = jbw+nywl-1; jep = jbp+nypl-1;
+  keu = kbu+nzul-1;  kev = kbv+nzvl-1; kew = kbw+nzwl-1; kep = kbp+nzpl-1;
+
+  u(ibu:ieu,jbu:jeu,kbu:keu) = utmp;
+  v(ibv:iev,jbv:jev,kbv:kev) = vtmp;
+  w(ibw:iew,jbw:jew,kbw:kew) = wtmp;
+  p(ibp:iep,jbp:jep,kbp:kep) = ptmp;
+
+end
+
+end
+
+%-------------------------------------------------------------------------------
+% ToDo!
+%-------------------------------------------------------------------------------
+% addParameter(par,'nparam',10);
+% addParameter(par,'precision','float64');
+% addParameter(par,'endian','a');
+% addParameter(par,'reclen',4);
+% nparam    = par.Results.nparam;
+% precision = par.Results.precision;
+% endian    = par.Results.endian;
+% reclen    = par.Results.reclen;
+
+% addParameter(par,'keepghost',0); % for now we can only read the full field
+                                   % no ghosts..
+% keepghost = par.Results.keepghost;

matlab/read_uvwp_complete_ucf.m

@@ -0,0 +1,183 @@
+function [u,v,w,p] = read_uvwp_complete_ucf(file,varargin)
+    % [u,v,w,p] = read_uvwp_complete_ucf(file,varargin)
+    % Reads u,v,w,p from all processor chunks and combines them to a single field.
+    % Input
+    %   file              file name (if tar-mode: ustar handle)
+    % ? step              index of timestep to be read (default: 1)
+    % ? verbosity         verbose output? (default: no)
+    % ? debug             debug output? (default: no)
+    % ? tarmode           read from tar-file? (default: 0)
+    % Output
+    %   u,v,w,p           complete fields with dim(nx,ny,nz)
+
+    % Parse optional input arguments
+    par = inputParser;
+    addParamValue(par,'step',1,@isnumeric);
+    addParamValue(par,'verbosity',0,@isnumeric);
+    addParamValue(par,'debug',0,@isnumeric);
+    addParamValue(par,'tarmode',0,@isnumeric); % deprecated
+    parse(par,varargin{:});
+    istep = par.Results.step;
+
+    % Define sets to be read
+    sets = {'u','v','w','p'};
+    nset = numel(sets);
+
+    % Open first file
+    obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug);
+    switch class(file)
+    case 'char'
+      [fdir,fbase,fext] = fileparts(file);
+      fname = sprintf('%s/%s.%05d',fdir,fbase,0);
+      obj.open(fname);
+    case {'ustar','ucfmulti'}
+      fname = sprintf('uvwp.%05d',0);
+      obj.opentar(file.pointer(fname));
+    otherwise
+      error('Input file type not supported: %s',class(file));
+    end
+    
+    if ~obj.validateType('field')
+      error('read error: no field data.');
+    end
+
+    % Read raw data
+    q  = cell(1,nset);
+    ib  = cell(1,nset);
+    jb  = cell(1,nset);
+    kb  = cell(1,nset);
+    nxl = cell(1,nset);
+    nyl = cell(1,nset);
+    nzl = cell(1,nset);
+    nx  = cell(1,nset);
+    ny  = cell(1,nset);
+    nz  = cell(1,nset);
+    for iset=1:nset
+      [data,params] = obj.readSet(istep,iset);
+      params = cast(params,'double');
+      ighost   = params(1);
+      ib{iset} = params(2);
+      jb{iset} = params(3);
+      kb{iset} = params(4);
+      nxl{iset}= params(5);
+      nyl{iset}= params(6);
+      nzl{iset}= params(7);
+      nx{iset} = params(8);
+      ny{iset} = params(9);
+      nz{iset} = params(10);
+      q{iset}  = reshape(data,nxl{iset}+2*ighost,nyl{iset}+2*ighost,nzl{iset}+2*ighost);
+      if ighost
+        q{iset} = q{iset}(2:end-1,2:end-1,2:end-1);
+      end
+    end
+
+    % Reassign content from loop and create global arrays
+    iset = find(strcmp(sets,'u'));
+    u = zeros(nx{iset},ny{iset},nz{iset},class(q{iset}));
+    u(ib{iset}:ib{iset}+nxl{iset}-1,...
+      jb{iset}:jb{iset}+nyl{iset}-1,...
+      kb{iset}:kb{iset}+nzl{iset}-1) = q{iset};
+
+    iset = find(strcmp(sets,'v'));
+    v = zeros(nx{iset},ny{iset},nz{iset},class(q{iset}));
+    v(ib{iset}:ib{iset}+nxl{iset}-1,...
+      jb{iset}:jb{iset}+nyl{iset}-1,...
+      kb{iset}:kb{iset}+nzl{iset}-1) = q{iset};
+
+    iset = find(strcmp(sets,'w'));
+    w = zeros(nx{iset},ny{iset},nz{iset},class(q{iset}));
+    w(ib{iset}:ib{iset}+nxl{iset}-1,...
+      jb{iset}:jb{iset}+nyl{iset}-1,...
+      kb{iset}:kb{iset}+nzl{iset}-1) = q{iset};
+
+    iset = find(strcmp(sets,'p'));
+    p = zeros(nx{iset},ny{iset},nz{iset},class(q{iset}));
+    p(ib{iset}:ib{iset}+nxl{iset}-1,...
+      jb{iset}:jb{iset}+nyl{iset}-1,...
+      kb{iset}:kb{iset}+nzl{iset}-1) = q{iset};
+
+    % Close the first file
+    obj.close();
+
+    % Now loop consecutively through files
+    ifile = 1;
+    switch class(file)
+    case 'char'
+      loopCondition = @(x) exist(x,'file');
+      fname = sprintf('%s/%s.%05d',fdir,fbase,ifile);
+    case {'ustar','ucfmulti'}
+      loopCondition = @(x) file.isSubfile(x);
+      fname = sprintf('uvwp.%05d',ifile);
+    end
+    while loopCondition(fname)
+      % Open file
+      obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug);
+      switch class(file)
+      case 'char'
+        obj.open(fname);
+      case {'ustar','ucfmulti'}
+        obj.opentar(file.pointer(fname));
+      end
+      
+      if ~obj.validateType('field')
+        error('read error: no field data.');
+      end
+
+      % Read raw data
+      q  = cell(1,nset);
+      ib  = cell(1,nset);
+      jb  = cell(1,nset);
+      kb  = cell(1,nset);
+      nxl = cell(1,nset);
+      nyl = cell(1,nset);
+      nzl = cell(1,nset);
+      for iset=1:nset
+        [data,params] = obj.readSet(istep,iset);
+        params = cast(params,'double');
+        ighost   = params(1);
+        ib{iset} = params(2);
+        jb{iset} = params(3);
+        kb{iset} = params(4);
+        nxl{iset}= params(5);
+        nyl{iset}= params(6);
+        nzl{iset}= params(7);
+        q{iset}  = reshape(data,nxl{iset}+2*ighost,nyl{iset}+2*ighost,nzl{iset}+2*ighost);
+        if ighost
+          q{iset} = q{iset}(2:end-1,2:end-1,2:end-1);
+        end
+      end
+
+      % Reassign content from loop and create global arrays
+      iset = find(strcmp(sets,'u'));
+      u(ib{iset}:ib{iset}+nxl{iset}-1,...
+        jb{iset}:jb{iset}+nyl{iset}-1,...
+        kb{iset}:kb{iset}+nzl{iset}-1) = q{iset};
+
+      iset = find(strcmp(sets,'v'));
+      v(ib{iset}:ib{iset}+nxl{iset}-1,...
+        jb{iset}:jb{iset}+nyl{iset}-1,...
+        kb{iset}:kb{iset}+nzl{iset}-1) = q{iset};
+
+      iset = find(strcmp(sets,'w'));
+      w(ib{iset}:ib{iset}+nxl{iset}-1,...
+        jb{iset}:jb{iset}+nyl{iset}-1,...
+        kb{iset}:kb{iset}+nzl{iset}-1) = q{iset};
+
+      iset = find(strcmp(sets,'p'));
+      p(ib{iset}:ib{iset}+nxl{iset}-1,...
+        jb{iset}:jb{iset}+nyl{iset}-1,...
+        kb{iset}:kb{iset}+nzl{iset}-1) = q{iset};
+
+      % Close file
+      obj.close();
+
+      % Move on to next file
+      ifile = ifile+1;
+      switch class(file)
+      case 'char'
+        fname = sprintf('%s/%s.%05d',fdir,fbase,ifile);
+      case {'ustar','ucfmulti'}
+        fname = sprintf('uvwp.%05d',ifile);
+      end
+    end
+end

matlab/write_particles_ucf.m

@@ -0,0 +1,46 @@
+function [] = write_particles_ucf(file,pp,col,time,varargin)
+    % [] = write_particles_ucf(file,pp,col,time,varargin)
+    % Writes particles data into a UCF file.
+    % Input
+    %   file                file to be written (the extension will be replaced by the proc rank)
+    %   pp                  particle data in 'array' format with dim(ncol,np,nt)
+    %   col                 particle column map. CAUTION: data is not rearranged according to colmap yet!!!
+    %   time                simulation time with dim(nt)
+    % ? endian              endianess of the file as used by fopen (default: 'n')
+    % ? verbosity           verbose output? (default: no)
+    % ? debug               debug output? (default: no)
+
+    % Parse variable input
+    par = inputParser;
+    addParamValue(par,'endian','n',@ischar);
+    addParamValue(par,'verbosity',0,@isnumeric);
+    addParamValue(par,'debug',0,@isnumeric);
+    parse(par,varargin{:});
+
+    ncol = size(pp,1);
+    np   = size(pp,2);
+    [ncolref,ncol_rank,ncol_hybrid,ncol_dem,ncol_scal] = ncol_from_colmap(col);
+    nt = size(pp,3);
+
+    % Check for consistency
+    if ncol~=col.Count || ncol~=ncolref
+      error('Particle data does not match column map.');
+    end
+    if nt~=numel(time)
+      error('Number of time steps must match the number of time values given. %d,%d',nt,numel(time));
+    end
+
+    % Create file
+    obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug);
+    obj.create(file,'type','particle','endian',par.Results.endian);
+    % Add particle data step by step
+    for it=1:nt
+      obj.appendStep(time(it));
+      params = int64([np,ncol,ncol_rank,ncol_hybrid,ncol_dem,ncol_scal]);
+      data = pp(:,:,it);
+      obj.appendSet(data,params);
+    end
+
+    % Close file
+    obj.close();
+end

matlab/write_scal_chunk_ucf.m

@@ -0,0 +1,44 @@
+function [] = write_scal_chunk_ucf(file,s,ib,jb,kb,nx,ny,nz,time,ighost,varargin)
+    % [] = write_scal_chunk_ucf(file,s,ibp,jbp,kbp,nxp,nyp,nzp,time,ighost,varargin)
+    % Writes a partial uvwp field to UCF chunk.
+    % Input
+    %   file                file to be written (the extension will be replaced by the proc rank)
+    %   s                   partial scalar fields with dim(nx,ny,nz,nscal)
+    %   ib,jb,kb            global index of first data point of field w/o ghost cells
+    %   nx,ny,nz            global grid size
+    %   time                simulation time
+    %   ighost              does partial field contain ghost cells?
+    % Optional input (key/value pair)
+    %   rank                processor rank as array [myid,my_col,my_row,my_pln]
+    %   endian              endianess of the file as used by fopen (default: 'n')
+    %   verbosity           verbose output? (default: no)
+    %   debug               debug output? (default: no)
+
+    % Parse variable input
+    par = inputParser;
+    addParamValue(par,'rank',[0,0,0,0],@(x)isnumeric(x)&&numel(x)==4);
+    addParamValue(par,'endian','n',@ischar);
+    addParamValue(par,'verbosity',0,@isnumeric);
+    addParamValue(par,'debug',0,@isnumeric);
+    parse(par,varargin{:});
+
+    % Determine number of scalar fields
+    nscal = size(s,4);
+
+    % Create file and write to it
+    obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug);
+    obj.create(file,'type','field','rank',par.Results.rank,'endian',par.Results.endian);
+    obj.appendStep(time);
+
+    % Write scalar field by field
+    for iscal=1:nscal
+      nxl = size(s,1)-2*ighost;
+      nyl = size(s,2)-2*ighost;
+      nzl = size(s,3)-2*ighost;
+      params = int64([ighost,ib,jb,kb,nxl,nyl,nzl,nx,ny,nz]);
+      obj.appendSet(s(:,:,:,iscal),params);
+    end
+
+    % Close file
+    obj.close();
+end

matlab/write_scal_complete_ucf.m

@@ -0,0 +1,75 @@
+function [] = write_scal_complete_ucf(file,s,ibegp,iendp,jbegp,jendp,kbegp,kendp,time,varargin)
+    % [] = write_scal_complete_ucf(file,s,ibegp,iendp,jbegp,jendp,kbegp,kendp,time,varargin)
+    % Writes a complete scalar field to UCF chunks.
+    % Input
+    %   file                file to be written (the extension will be replaced by the proc rank)
+    %   s                   full scalar field with dim(nx,ny,nz,nscal)
+    %   ibegp,jbegp,kbegp   arrays with length(n.proc) with processor grid
+    %   iendp,jendp,kendp   """
+    %   time                simulation time
+    % Optional input (key/value pair)
+    %   endian              endianess of the file as used by fopen (default: 'n')
+    %   verbosity           verbose output? (default: no)
+    %   debug               debug output? (default: no)
+
+    % Parse variable input
+    par = inputParser;
+    addParamValue(par,'endian','n',@ischar);
+    addParamValue(par,'verbosity',0,@isnumeric);
+    addParamValue(par,'debug',0,@isnumeric);
+    parse(par,varargin{:});
+
+    % Parse filename
+    [fdir,fbase,fext] = fileparts(file);
+    fbasepath = fullfile(fdir,fbase);
+
+    nxprocs = numel(ibegp);
+    nyprocs = numel(jbegp);
+    nzprocs = numel(kbegp);
+
+    nx = size(s,1);
+    ny = size(s,2);
+    nz = size(s,3);
+    nscal = size(s,4);
+
+    for ixproc=0:nxprocs-1
+      for iyproc=0:nyprocs-1
+        for izproc=0:nzprocs-1
+          [iproc] = locfun_proc_id(ixproc,iyproc,izproc,nxprocs,nyprocs,nzprocs);
+          fname = sprintf('%s.%05d',fbasepath,iproc);
+
+          % Extract local indices
+          ighost = 0;
+          ib = ibegp(ixproc+1);
+          ie = iendp(ixproc+1);
+          jb = jbegp(iyproc+1);
+          je = jendp(iyproc+1);
+          kb = kbegp(izproc+1);
+          ke = kendp(izproc+1);
+          nxl = ie-ib+1;
+          nyl = je-jb+1;
+          nzl = ke-kb+1;
+
+          % Create a new chunk file
+          obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug);
+          obj.create(fname,'type','field','rank',[iproc,ixproc,iyproc,izproc],'endian',par.Results.endian);
+          obj.appendStep(time);
+
+          % Write data field by field
+          for iscal=1:nscal
+            params = int64([ighost,ib,jb,kb,nxl,nyl,nzl,nx,ny,nz]);
+            data = s(ib:ie,jb:je,kb:ke,iscal);
+            obj.appendSet(data,params);
+          end
+
+          % Close chunk file
+          obj.close();
+        end
+      end
+    end
+end
+
+function [ind] = locfun_proc_id(ii,jj,kk,nx,ny,nz)
+  % local version of 'sub2ind_zero_row'
+  ind=ii*ny*nz+jj*nz+kk;
+end

matlab/write_uvwp_chunk_ucf.m

@@ -0,0 +1,70 @@
+function [] = write_uvwp_chunk_ucf(file,...
+    u,ibu,jbu,kbu,nxu,nyu,nzu,...
+    v,ibv,jbv,kbv,nxv,nyv,nzv,...
+    w,ibw,jbw,kbw,nxw,nyw,nzw,...
+    p,ibp,jbp,kbp,nxp,nyp,nzp,...
+    time,ighost,varargin)
+    % [] = write_uvwp_chunk_ucf(file,...
+    %  u,ibu,jbu,kbu,nxu,nyu,nzu,,...
+    %  v,ibv,jbv,kbv,nxv,nyv,nzv,,...
+    %  w,ibw,jbw,kbw,nxw,nyw,nzw,,...
+    %  p,ibp,jbp,kbp,nxp,nyp,nzp,,...
+    %  time,ighost,varargin)
+    % Writes a partial uvwp field to UCF chunk.
+    % Input
+    %   file                file to be written (the extension will be replaced by the proc rank)
+    %   u,v,w,p             partial u,v,w,p fields with dim(nx,ny,nz)
+    %   ib,jb,kb            global index of first data point of field w/o ghost cells
+    %   nx,ny,nz            global grid size
+    %   time                simulation time
+    %   ighost              does partial field contain ghost cells?
+    % Optional input (key/value pair)
+    %   rank                processor rank as array [myid,my_col,my_row,my_pln]
+    %   endian              endianess of the file as used by fopen (default: 'n')
+    %   verbosity           verbose output? (default: no)
+    %   debug               debug output? (default: no)
+
+    % Parse variable input
+    par = inputParser;
+    addParamValue(par,'rank',[0,0,0,0],@(x)isnumeric(x)&&numel(x)==4);
+    addParamValue(par,'endian','n',@ischar);
+    addParamValue(par,'verbosity',0,@isnumeric);
+    addParamValue(par,'debug',0,@isnumeric);
+    parse(par,varargin{:});
+
+    % Create file and write to it
+    obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug);
+    obj.create(file,'type','field','rank',par.Results.rank,'endian',par.Results.endian);
+    obj.appendStep(time);
+
+    % Write u
+    nxul = size(u,1)-2*ighost;
+    nyul = size(u,2)-2*ighost;
+    nzul = size(u,3)-2*ighost;
+    params = int64([ighost,ibu,jbu,kbu,nxul,nyul,nzul,nxu,nyu,nzu]);
+    obj.appendSet(u,params);
+
+    % Write v
+    nxvl = size(v,1)-2*ighost;
+    nyvl = size(v,2)-2*ighost;
+    nzvl = size(v,3)-2*ighost;
+    params = int64([ighost,ibv,jbv,kbv,nxvl,nyvl,nzvl,nxv,nyv,nzv]);
+    obj.appendSet(v,params);
+
+    % Write w
+    nxwl = size(w,1)-2*ighost;
+    nywl = size(w,2)-2*ighost;
+    nzwl = size(w,3)-2*ighost;
+    params = int64([ighost,ibw,jbw,kbw,nxwl,nywl,nzwl,nxw,nyw,nzw]);
+    obj.appendSet(w,params);
+
+    % Write p
+    nxpl = size(p,1)-2*ighost;
+    nypl = size(p,2)-2*ighost;
+    nzpl = size(p,3)-2*ighost;
+    params = int64([ighost,ibp,jbp,kbp,nxpl,nypl,nzpl,nxp,nyp,nzp]);
+    obj.appendSet(p,params);
+
+    % Close file
+    obj.close();
+end

matlab/write_uvwp_complete_ucf.m

@@ -0,0 +1,106 @@
+function [] = write_uvwp_complete_ucf(file,...
+    u,ibegu,jbegu,kbegu,iendu,jendu,kendu,...
+    v,ibegv,jbegv,kbegv,iendv,jendv,kendv,...
+    w,ibegw,jbegw,kbegw,iendw,jendw,kendw,...
+    p,ibegp,jbegp,kbegp,iendp,jendp,kendp,...
+    time,varargin)
+    % [] = write_uvwp_complete_ucf(file,...
+    %  u,ibegu,jbegu,kbegu,iendu,jendu,kendu,...
+    %  v,ibegv,jbegv,kbegv,iendv,jendv,kendv,...
+    %  w,ibegw,jbegw,kbegw,iendw,jendw,kendw,...
+    %  p,ibegp,jbegp,kbegp,iendp,jendp,kendp,...
+    %  time,varargin)
+    % Writes a complete uvwp field to UCF chunks.
+    % Input
+    %   file                file to be written (the extension will be replaced by the proc rank)
+    %   u,v,w,p             full u,v,w,p fields with dim(nx,ny,nz)
+    %   ibeg,jbeg,kbeg      arrays with length(n.proc) with processor grid
+    %   iend,jend,kend      """
+    %   time                simulation time
+    % Optional input (key/value pair)
+    %   endian              endianess of the file as used by fopen (default: 'n')
+    %   verbosity           verbose output? (default: no)
+    %   debug               debug output? (default: no)
+
+    % Parse variable input
+    par = inputParser;
+    addParamValue(par,'endian','n',@ischar);
+    addParamValue(par,'verbosity',0,@isnumeric);
+    addParamValue(par,'debug',0,@isnumeric);
+    parse(par,varargin{:});
+
+    % Parse filename
+    [fdir,fbase,fext] = fileparts(file);
+    fbasepath = fullfile(fdir,fbase);
+
+    nxprocs = numel(ibegu);
+    nyprocs = numel(jbegu);
+    nzprocs = numel(kbegu);
+
+    nxu = size(u,1); nxv = size(v,1); nxw = size(w,1); nxp = size(p,1);
+    nyu = size(u,2); nyv = size(v,2); nyw = size(w,2); nyp = size(p,2);
+    nzu = size(u,3); nzv = size(v,3); nzw = size(w,3); nzp = size(p,3);
+
+    for ixproc=0:nxprocs-1
+      for iyproc=0:nyprocs-1
+        for izproc=0:nzprocs-1
+          [iproc] = locfun_proc_id(ixproc,iyproc,izproc,nxprocs,nyprocs,nzprocs);
+          fname = sprintf('%s.%05d',fbasepath,iproc);
+
+          % Extract local indices
+          ibu = ibegu(ixproc+1); ibv = ibegv(ixproc+1); ibw = ibegw(ixproc+1); ibp = ibegp(ixproc+1);
+          ieu = iendu(ixproc+1); iev = iendv(ixproc+1); iew = iendw(ixproc+1); iep = iendp(ixproc+1);
+          jbu = jbegu(iyproc+1); jbv = jbegv(iyproc+1); jbw = jbegw(iyproc+1); jbp = jbegp(iyproc+1);
+          jeu = jendu(iyproc+1); jev = jendv(iyproc+1); jew = jendw(iyproc+1); jep = jendp(iyproc+1);
+          kbu = kbegu(izproc+1); kbv = kbegv(izproc+1); kbw = kbegw(izproc+1); kbp = kbegp(izproc+1);
+          keu = kendu(izproc+1); kev = kendv(izproc+1); kew = kendw(izproc+1); kep = kendp(izproc+1);
+
+          % Create a new chunk file
+          obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug);
+          obj.create(fname,'type','field','rank',[iproc,ixproc,iyproc,izproc],'endian',par.Results.endian);
+          obj.appendStep(time);
+          ighost = 0;
+
+          % Write u
+          data = u(ibu:ieu,jbu:jeu,kbu:keu);
+          nxul = size(data,1)-2*ighost;
+          nyul = size(data,2)-2*ighost;
+          nzul = size(data,3)-2*ighost;
+          params = int64([ighost,ibu,jbu,kbu,nxul,nyul,nzul,nxu,nyu,nzu]);
+          obj.appendSet(data,params);
+
+          % Write v
+          data = v(ibv:iev,jbv:jev,kbv:kev);
+          nxvl = size(data,1)-2*ighost;
+          nyvl = size(data,2)-2*ighost;
+          nzvl = size(data,3)-2*ighost;
+          params = int64([ighost,ibv,jbv,kbv,nxvl,nyvl,nzvl,nxv,nyv,nzv]);
+          obj.appendSet(data,params);
+
+          % Write w
+          data = w(ibw:iew,jbw:jew,kbw:kew);
+          nxwl = size(data,1)-2*ighost;
+          nywl = size(data,2)-2*ighost;
+          nzwl = size(data,3)-2*ighost;
+          params = int64([ighost,ibw,jbw,kbw,nxwl,nywl,nzwl,nxw,nyw,nzw]);
+          obj.appendSet(data,params);
+
+          % Write p
+          data = p(ibp:iep,jbp:jep,kbp:kep);
+          nxpl = size(data,1)-2*ighost;
+          nypl = size(data,2)-2*ighost;
+          nzpl = size(data,3)-2*ighost;
+          params = int64([ighost,ibp,jbp,kbp,nxpl,nypl,nzpl,nxp,nyp,nzp]);
+          obj.appendSet(data,params);
+
+          % Close chunk file
+          obj.close();
+        end
+      end
+    end
+end
+
+function [ind] = locfun_proc_id(ii,jj,kk,nx,ny,nz)
+  % local version of 'sub2ind_zero_row'
+  ind=ii*ny*nz+jj*nz+kk;
+end

matlab/write_uvwp_newmesh_ucf.m

@@ -0,0 +1,133 @@
+function [] = write_uvwp_newmesh_ucf(file,...
+    u,v,w,p,...
+    xuo,yuo,zuo,xvo,yvo,zvo,xwo,ywo,zwo,xpo,ypo,zpo,...
+    xun,yun,zun,xvn,yvn,zvn,xwn,ywn,zwn,xpn,ypn,zpn,...
+    ibegu,jbegu,kbegu,iendu,jendu,kendu,...
+    ibegv,jbegv,kbegv,iendv,jendv,kendv,...
+    ibegw,jbegw,kbegw,iendw,jendw,kendw,...
+    ibegp,jbegp,kbegp,iendp,jendp,kendp,...
+    time,varargin)
+    % [] = write_uvwp_newmesh_ucf(file,...
+    %     u,v,w,p,...
+    %     xuo,yuo,zuo,xvo,yvo,zvo,xwo,ywo,zwo,xpo,ypo,zpo,...
+    %     xun,yun,zun,xvn,yvn,zvn,xwn,ywn,zwn,xpn,ypn,zpn,...
+    %     ibegu,jbegu,kbegu,iendu,jendu,kendu,...
+    %     ibegv,jbegv,kbegv,iendv,jendv,kendv,...
+    %     ibegw,jbegw,kbegw,iendw,jendw,kendw,...
+    %     ibegp,jbegp,kbegp,iendp,jendp,kendp,...
+    %     time,varargin)
+    % Writes a complete uvwp field to UCF chunks with a different mesh.
+    % Caution: extrapolation might be performed (periodicity is not checked)
+    % Input
+    %   file                      output file
+    %   u,v,w,p                   complete uvwp fields on old grid
+    %   xuo,yuo,zuo,...           old mesh
+    %   xun,yun,zun,...           new mesh
+    %   ibegu,jbegu,kbegu,...     new processor grid
+    %   time                      simulation time
+    % Optional input (key/value pair)
+    %   method              interpolation method (default: 'cubic')
+    %   endian              endianess of the file as used by fopen (default: 'n')
+    %   verbosity           verbose output? (default: no)
+    %   debug               debug output? (default: no)
+
+    % Parse variable input
+    par = inputParser;
+    addParamValue(par,'method','cubic',@ischar);
+    addParamValue(par,'endian','n',@ischar);
+    addParamValue(par,'verbosity',0,@isnumeric);
+    addParamValue(par,'debug',0,@isnumeric);
+    parse(par,varargin{:});
+
+    % Parse filename
+    [fdir,fbase,fext] = fileparts(file);
+    fbasepath = fullfile(fdir,fbase);
+
+    nxprocs = numel(ibegu);
+    nyprocs = numel(jbegu);
+    nzprocs = numel(kbegu);
+
+    nxu = numel(xun); nxv = numel(xvn); nxw = numel(xwn); nxp = numel(xpn);
+    nyu = numel(yun); nyv = numel(yvn); nyw = numel(ywn); nyp = numel(ypn);
+    nzu = numel(zun); nzv = numel(zvn); nzw = numel(zwn); nzp = numel(zpn);
+
+    % Create interpolants
+    giu = griddedInterpolant({xuo,yuo,zuo},u,par.Results.method);
+    giv = griddedInterpolant({xvo,yvo,zvo},v,par.Results.method);
+    giw = griddedInterpolant({xwo,ywo,zwo},w,par.Results.method);
+    gip = griddedInterpolant({xpo,ypo,zpo},p,par.Results.method);
+
+    for ixproc=0:nxprocs-1
+      for iyproc=0:nyprocs-1
+        for izproc=0:nzprocs-1
+          [iproc] = locfun_proc_id(ixproc,iyproc,izproc,nxprocs,nyprocs,nzprocs);
+          fname = sprintf('%s.%05d',fbasepath,iproc);
+
+          % Get processor boundaries
+          ibu = ibegu(ixproc+1); ibv = ibegv(ixproc+1); ibw = ibegw(ixproc+1); ibp = ibegp(ixproc+1);
+          ieu = iendu(ixproc+1); iev = iendv(ixproc+1); iew = iendw(ixproc+1); iep = iendp(ixproc+1);
+          jbu = jbegu(iyproc+1); jbv = jbegv(iyproc+1); jbw = jbegw(iyproc+1); jbp = jbegp(iyproc+1);
+          jeu = jendu(iyproc+1); jev = jendv(iyproc+1); jew = jendw(iyproc+1); jep = jendp(iyproc+1);
+          kbu = kbegu(izproc+1); kbv = kbegv(izproc+1); kbw = kbegw(izproc+1); kbp = kbegp(izproc+1);
+          keu = kendu(izproc+1); kev = kendv(izproc+1); kew = kendw(izproc+1); kep = kendp(izproc+1);
+
+          % Create output file
+          obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug);
+          obj.create(fname,'type','field','rank',[iproc,ixproc,iyproc,izproc],'endian',par.Results.endian);
+          obj.appendStep(time);
+          ighost = 0;
+
+          % Interpolate to refined chunk and write
+          data = giu({xun(ibu:ieu),yun(jbu:jeu),zun(kbu:keu)});
+          if any(isnan(data))
+            warning('NaN detected: u, proc #%d, %d occurences',iproc,sum(isnan(data(:))));
+          end
+          nxul = size(data,1)-2*ighost;
+          nyul = size(data,2)-2*ighost;
+          nzul = size(data,3)-2*ighost;
+          params = int64([ighost,ibu,jbu,kbu,nxul,nyul,nzul,nxu,nyu,nzu]);
+          obj.appendSet(data,params);
+                    
+          % v
+          data = giv({xvn(ibv:iev),yvn(jbv:jev),zvn(kbv:kev)});
+          if any(isnan(data))
+            warning('NaN detected: v, proc #%d, %d occurences',iproc,sum(isnan(data(:))));
+          end
+          nxvl = size(data,1)-2*ighost;
+          nyvl = size(data,2)-2*ighost;
+          nzvl = size(data,3)-2*ighost;
+          params = int64([ighost,ibv,jbv,kbv,nxvl,nyvl,nzvl,nxv,nyv,nzv]);
+          obj.appendSet(data,params);
+
+          % w
+          data = giw({xwn(ibw:iew),ywn(jbw:jew),zwn(kbw:kew)});
+          if any(isnan(data))
+            warning('NaN detected: w, proc #%d, %d occurences',iproc,sum(isnan(data(:))));
+          end
+          nxwl = size(data,1)-2*ighost;
+          nywl = size(data,2)-2*ighost;
+          nzwl = size(data,3)-2*ighost;
+          params = int64([ighost,ibw,jbw,kbw,nxwl,nywl,nzwl,nxw,nyw,nzw]);
+          obj.appendSet(data,params);
+          
+          % p
+          data = gip({xpn(ibp:iep),ypn(jbp:jep),zpn(kbp:kep)});
+          if any(isnan(data))
+            warning('NaN detected: p, proc #%d, %d occurences',iproc,sum(isnan(data(:))));
+          end
+          nxpl = size(data,1)-2*ighost;
+          nypl = size(data,2)-2*ighost;
+          nzpl = size(data,3)-2*ighost;
+          params = int64([ighost,ibp,jbp,kbp,nxpl,nypl,nzpl,nxp,nyp,nzp]);
+          obj.appendSet(data,params);
+
+          obj.close();
+        end
+      end
+    end
+end
+
+function [ind] = locfun_proc_id(ii,jj,kk,nx,ny,nz)
+  % local version of 'sub2ind_zero_row'
+  ind=ii*ny*nz+jj*nz+kk;
+end

python/ibmppp/__init__.py

@@ -0,0 +1 @@
+from .ibmppp import *

python/ratarindex.py

@@ -0,0 +1,496 @@
+#!/usr/bin/env python3
+
+import os, re, sys, stat, tarfile, argparse
+import itertools
+from collections import namedtuple
+from timeit import default_timer as timer
+
+
+printDebug = 1
+
+def overrides( parentClass ):
+    def overrider( method ):
+        assert( method.__name__ in dir( parentClass ) )
+        return method
+    return overrider
+
+
+FileInfo = namedtuple( "FileInfo", "offset size mtime mode type linkname uid gid istar" )
+
+
+class IndexedTar( object ):
+    """
+    This class reads once through a whole TAR archive and stores TAR file offsets for all packed files
+    in an index to support fast seeking to a given file.
+    """
+
+    __slots__ = (
+        'tarFileName',
+        'fileIndex',
+        'mountRecursively',
+        'cacheFolder',
+        'possibleIndexFilePaths',
+        'indexFileName',
+    )
+
+    # these allowed backends also double as extensions for the index file to look for
+    availableSerializationBackends = [
+        'pickle',
+        'pickle2',
+        'pickle3',
+        'custom',
+        'cbor',
+        'msgpack',
+        'rapidjson',
+        'ujson',
+        'simplejson'
+    ]
+    availableCompressions = [
+        '', # no compression
+        'lz4',
+        'gz',
+    ]
+
+    def __init__( self, pathToTar = None, fileObject = None, writeIndex = False,
+                  recursive = False, serializationBackend = None ):
+        self.tarFileName = os.path.normpath( pathToTar )
+        # Stores the file hierarchy in a dictionary with keys being either the file and containing file metainformation
+        # or keys being a folder name and containing recursively defined dictionary.
+        self.fileIndex = {}
+        self.mountRecursively = recursive
+
+        self.cacheFolder = os.path.expanduser( "~/.ratarmount" ) # will be used for storing if current path is read-only
+        self.possibleIndexFilePaths = [
+            self.tarFileName + ".index",
+            self.cacheFolder + "/" + self.tarFileName.replace( "/", "_" ) + ".index"
+        ]
+
+        if serializationBackend not in self.supportedIndexExtensions():
+            serializationBackend = 'custom'
+            print( "[Warning] Serialization backend not supported. Defaulting to '" + serializationBackend + "'!" )
+
+        # this is the actual index file, which will be used in the end, and by default
+        self.indexFileName = self.possibleIndexFilePaths[0] + "." + serializationBackend
+
+        if fileObject is not None:
+            if writeIndex:
+                print( "Can't write out index for file object input. Ignoring this option." )
+            self.createIndex( fileObject )
+        else:
+            # first try loading the index for the given serialization backend
+            if serializationBackend is not None:
+                for indexPath in self.possibleIndexFilePaths:
+                    indexPathWitExt = indexPath + "." + serializationBackend
+
+                    if self.indexIsLoaded():
+                        break
+
+                    if os.path.isfile( indexPathWitExt ):
+                        if os.path.getsize( indexPathWitExt ) == 0:
+                            os.remove( indexPathWitExt )
+                        else:
+                            writeIndex = False 
+
+            if not self.indexIsLoaded():
+                with open( self.tarFileName, 'rb' ) as file:
+                    self.createIndex( file )
+
+                if writeIndex:
+                    for indexPath in self.possibleIndexFilePaths:
+                        indexPath += "." + serializationBackend
+
+                        try:
+                            folder = os.path.dirname( indexPath )
+                            if not os.path.exists( folder ):
+                                os.mkdir( folder )
+
+                            f = open( indexPath, 'wb' )
+                            f.close()
+                            os.remove( indexPath )
+                            self.indexFileName = indexPath
+
+                            break
+                        except IOError:
+                            if printDebug >= 2:
+                                print( "Could not create file:", indexPath )
+
+                    try:
+                        self.writeIndex( self.indexFileName )
+                    except IOError:
+                        print( "[Info] Could not write TAR index to file." )
+
+    @staticmethod
+    def supportedIndexExtensions():
+        return [ '.'.join( combination ).strip( '.' )
+                 for combination in itertools.product( IndexedTar.availableSerializationBackends,
+                                                       IndexedTar.availableCompressions ) ]
+    @staticmethod
+    def dump( toDump, file ):
+        if isinstance( toDump, dict ):
+            file.write( b'\x01' ) # magic code meaning "start dictionary object"
+
+            for key, value in toDump.items():
+                file.write( b'\x03' ) # magic code meaning "serialized key value pair"
+                IndexedTar.dump( key, file )
+                IndexedTar.dump( value, file )
+
+            file.write( b'\x02' ) # magic code meaning "close dictionary object"
+
+        elif isinstance( toDump, FileInfo ):
+            import msgpack
+            serialized = msgpack.dumps( toDump )
+            file.write( b'\x05' ) # magic code meaning "msgpack object"
+            file.write( len( serialized ).to_bytes( 4, byteorder = 'little' ) )
+            file.write( serialized )
+
+        elif isinstance( toDump, str ):
+            serialized = toDump.encode()
+            file.write( b'\x04' ) # magic code meaning "string object"
+            file.write( len( serialized ).to_bytes( 4, byteorder = 'little' ) )
+            file.write( serialized )
+
+        else:
+            print( "Ignoring unsupported type to write:", toDump )
+
+    @staticmethod
+    def load( file ):
+        elementType = file.read( 1 )
+
+        if elementType == b'\x01': # start of dictionary
+            result = {}
+
+            dictElementType = file.read( 1 )
+            while len( dictElementType ) != 0:
+                if dictElementType == b'\x02':
+                    break
+
+                elif dictElementType == b'\x03':
+                    import msgpack
+
+                    keyType = file.read( 1 )
+                    if keyType != b'\x04': # key must be string object
+                        raise Exception( 'Custom TAR index loader: invalid file format' )
+                    size = int.from_bytes( file.read( 4 ), byteorder = 'little' )
+                    key = file.read( size ).decode()
+
+                    valueType = file.read( 1 )
+                    if valueType == b'\x05': # msgpack object
+                        size = int.from_bytes( file.read( 4 ), byteorder = 'little' )
+                        serialized = file.read( size )
+                        value = FileInfo( *msgpack.loads( serialized ) )
+
+                    elif valueType == b'\x01': # dict object
+                        import io
+                        file.seek( -1, io.SEEK_CUR )
+                        value = IndexedTar.load( file )
+
+                    else:
+                        raise Exception( 'Custom TAR index loader: invalid file format ' +
+                            '(expected msgpack or dict but got' +
+                            str( int.from_bytes( valueType, byteorder = 'little' ) ) + ')' )
+
+                    result[key] = value
+
+                else:
+                    raise Exception( 'Custom TAR index loader: invalid file format ' +
+                        '(expected end-of-dict or key-value pair but got' +
+                        str( int.from_bytes( dictElementType, byteorder = 'little' ) ) + ')' )
+
+                dictElementType = file.read( 1 )
+
+            return result
+
+        else:
+            raise Exception( 'Custom TAR index loader: invalid file format' )
+
+    def getFileInfo( self, path, listDir = False ):
+        # go down file hierarchy tree along the given path
+        p = self.fileIndex
+        for name in os.path.normpath( path ).split( os.sep ):
+            if not name:
+                continue
+            if not name in p:
+                return
+            p = p[name]
+
+        def repackDeserializedNamedTuple( p ):
+            if isinstance( p, list ) and len( p ) == len( FileInfo._fields ):
+                return FileInfo( *p )
+            elif isinstance( p, dict ) and len( p ) == len( FileInfo._fields ) and \
+                 'uid' in p and isinstance( p['uid'], int ):
+                # a normal directory dict must only have dict or FileInfo values, so if the value to the 'uid'
+                # key is an actual int, then it is sure it is a deserialized FileInfo object and not a file named 'uid'
+                print( "P ===", p )
+                print( "FileInfo ===", FileInfo( **p ) )
+                return FileInfo( **p )
+            return p
+
+        p = repackDeserializedNamedTuple( p )
+
+        # if the directory contents are not to be printed and it is a directory, return the "file" info of "."
+        if not listDir and isinstance( p, dict ):
+            if '.' in p:
+                p = p['.']
+            else:
+                return FileInfo(
+                    offset   = 0, # not necessary for directory anyways
+                    size     = 1, # might be misleading / non-conform
+                    mtime    = 0,
+                    mode     = 0o555 | stat.S_IFDIR,
+                    type     = tarfile.DIRTYPE,
+                    linkname = "",
+                    uid      = 0,
+                    gid      = 0,
+                    istar    = False
+                )
+
+        return repackDeserializedNamedTuple( p )
+
+    def isDir( self, path ):
+        return True if isinstance( self.getFileInfo( path, listDir = True ), dict ) else False
+
+    def exists( self, path ):
+        path = os.path.normpath( path )
+        return self.isDir( path ) or isinstance( self.getFileInfo( path ), FileInfo )
+
+    def setFileInfo( self, path, fileInfo ):
+        """
+        path: the full path to the file with leading slash (/) for which to set the file info
+        """
+        assert( isinstance( fileInfo, FileInfo ) )
+
+        pathHierarchy = os.path.normpath( path ).split( os.sep )
+        if len( pathHierarchy ) == 0:
+            return
+
+        # go down file hierarchy tree along the given path
+        p = self.fileIndex
+        for name in pathHierarchy[:-1]:
+            if not name:
+                continue
+            assert( isinstance( p, dict ) )
+            p = p.setdefault( name, {} )
+
+        # create a new key in the dictionary of the parent folder
+        p.update( { pathHierarchy[-1] : fileInfo } )
+
+    def setDirInfo( self, path, dirInfo, dirContents = {} ):
+        """
+        path: the full path to the file with leading slash (/) for which to set the folder info
+        """
+        assert( isinstance( dirInfo, FileInfo ) )
+        assert( isinstance( dirContents, dict ) )
+
+        pathHierarchy = os.path.normpath( path ).strip( os.sep ).split( os.sep )
+        if len( pathHierarchy ) == 0:
+            return
+
+        # go down file hierarchy tree along the given path
+        p = self.fileIndex
+        for name in pathHierarchy[:-1]:
+            if not name:
+                continue
+            assert( isinstance( p, dict ) )
+            p = p.setdefault( name, {} )
+
+        # create a new key in the dictionary of the parent folder
+        p.update( { pathHierarchy[-1] : dirContents } )
+        p[pathHierarchy[-1]].update( { '.' : dirInfo } )
+
+    def createIndex( self, fileObject ):
+        if printDebug >= 1:
+            print( "Creating offset dictionary for", "<file object>" if self.tarFileName is None else self.tarFileName, "..." )
+        t0 = timer()
+
+        self.fileIndex = {}
+        try:
+            loadedTarFile = tarfile.open( fileobj = fileObject, mode = 'r:' )
+        except tarfile.ReadError as exception:
+            print( "Archive can't be opened! This might happen for compressed TAR archives, which currently is not supported." )
+            raise exception
+
+        for tarInfo in loadedTarFile:
+            mode = tarInfo.mode
+            if tarInfo.isdir() : mode |= stat.S_IFDIR
+            if tarInfo.isfile(): mode |= stat.S_IFREG
+            if tarInfo.issym() : mode |= stat.S_IFLNK
+            if tarInfo.ischr() : mode |= stat.S_IFCHR
+            if tarInfo.isfifo(): mode |= stat.S_IFIFO
+            fileInfo = FileInfo(
+                offset   = tarInfo.offset_data,
+                size     = tarInfo.size       ,
+                mtime    = tarInfo.mtime      ,
+                mode     = mode               ,
+                type     = tarInfo.type       ,
+                linkname = tarInfo.linkname   ,
+                uid      = tarInfo.uid        ,
+                gid      = tarInfo.gid        ,
+                istar    = False
+            )
+
+            # open contained tars for recursive mounting
+            indexedTar = None
+            if self.mountRecursively and tarInfo.isfile() and tarInfo.name.endswith( ".tar" ):
+                oldPos = fileObject.tell()
+                if oldPos != tarInfo.offset_data:
+                    fileObject.seek( tarInfo.offset_data )
+                indexedTar = IndexedTar( tarInfo.name, fileObject = fileObject, writeIndex = False )
+                fileObject.seek( fileObject.tell() ) # might be especially necessary if the .tar is not actually a tar!
+
+            # Add a leading '/' as a convention where '/' represents the TAR root folder
+            # Partly, done because fusepy specifies paths in a mounted directory like this
+            path = os.path.normpath( "/" + tarInfo.name )
+
+            # test whether the TAR file could be loaded and if so "mount" it recursively
+            if indexedTar is not None and indexedTar.indexIsLoaded():
+                # actually apply the recursive tar mounting
+                extractedName = re.sub( r"\.tar$", "", path )
+                if not self.exists( extractedName ):
+                    path = extractedName
+
+                mountMode = ( fileInfo.mode & 0o777 ) | stat.S_IFDIR
+                if mountMode & stat.S_IRUSR != 0: mountMode |= stat.S_IXUSR
+                if mountMode & stat.S_IRGRP != 0: mountMode |= stat.S_IXGRP
+                if mountMode & stat.S_IROTH != 0: mountMode |= stat.S_IXOTH
+                fileInfo = fileInfo._replace( mode = mountMode, istar = True )
+
+                if self.exists( path ):
+                    print( "[Warning]", path, "already exists in database and will be overwritten!" )
+
+                # merge fileIndex from recursively loaded TAR into our Indexes
+                self.setDirInfo( path, fileInfo, indexedTar.fileIndex )
+
+            elif path != '/':
+                # just a warning and check for the path already existing
+                if self.exists( path ):
+                    fileInfo = self.getFileInfo( path, listDir = False )
+                    if fileInfo.istar:
+                        # move recursively mounted TAR directory to original .tar name if there is a name-clash,
+                        # e.g., when foo/ also exists in the TAR but foo.tar would be mounted to foo/.
+                        # In this case, move that mount to foo.tar/
+                        self.setFileInfo( path + ".tar", fileInfo, self.getFileInfo( path, listDir = True ) )
+                    else:
+                        print( "[Warning]", path, "already exists in database and will be overwritten!" )
+
+                # simply store the file or directory information from current TAR item
+                if tarInfo.isdir():
+                    self.setDirInfo( path, fileInfo, {} )
+                else:
+                    self.setFileInfo( path, fileInfo )
+
+        t1 = timer()
+        if printDebug >= 1:
+            print( "Creating offset dictionary for", "<file object>" if self.tarFileName is None else self.tarFileName, "took {:.2f}s".format( t1 - t0 ) )
+
+    def serializationBackendFromFileName( self, fileName ):
+        splitName = fileName.split( '.' )
+
+        if len( splitName ) > 2 and '.'.join( splitName[-2:] ) in self.supportedIndexExtensions():
+            return '.'.join( splitName[-2:] )
+        elif splitName[-1] in self.supportedIndexExtensions():
+            return splitName[-1]
+        return None
+
+    def indexIsLoaded( self ):
+        return True if self.fileIndex else False
+
+    def writeIndex( self, outFileName ):
+        """
+        outFileName: full file name with backend extension. Depending on the extension the serialization is chosen.
+        """
+
+        serializationBackend = self.serializationBackendFromFileName( outFileName )
+
+        if printDebug >= 1:
+            print( "Writing out TAR index using", serializationBackend, "to", outFileName, "..." )
+        t0 = timer()
+
+        fileMode = 'wt' if 'json' in serializationBackend else 'wb'
+
+        if serializationBackend.endswith( '.lz4' ):
+            import lz4.frame
+            wrapperOpen = lambda x : lz4.frame.open( x, fileMode )
+        elif serializationBackend.endswith( '.gz' ):
+            import gzip
+            wrapperOpen = lambda x : gzip.open( x, fileMode )
+        else:
+            wrapperOpen = lambda x : open( x, fileMode )
+        serializationBackend = serializationBackend.split( '.' )[0]
+
+        # libraries tested but not working:
+        #  - marshal: can't serialize namedtuples
+        #  - hickle: for some reason, creates files almost 64x larger as pickle!? And also takes similarly longer
+        #  - yaml: almost a 10 times slower and more memory usage and deserializes everything including ints to string
+
+        with wrapperOpen( outFileName ) as outFile:
+            if serializationBackend == 'pickle2':
+                import pickle
+                pickle.dump( self.fileIndex, outFile )
+                pickle.dump( self.fileIndex, outFile, protocol = 2 )
+
+            # default serialization because it has the fewest dependencies and because it was legacy default
+            elif serializationBackend == 'pickle3' or \
+                 serializationBackend == 'pickle' or \
+                 serializationBackend is None:
+                import pickle
+                pickle.dump( self.fileIndex, outFile )
+                pickle.dump( self.fileIndex, outFile, protocol = 3 ) # 3 is default protocol
+
+            elif serializationBackend == 'simplejson':
+                import simplejson
+                simplejson.dump( self.fileIndex, outFile, namedtuple_as_object = True )
+
+            elif serializationBackend == 'custom':
+                IndexedTar.dump( self.fileIndex, outFile )
+
+            elif serializationBackend in [ 'msgpack', 'cbor', 'rapidjson', 'ujson' ]:
+                import importlib
+                module = importlib.import_module( serializationBackend )
+                getattr( module, 'dump' )( self.fileIndex, outFile )
+
+            else:
+                print( "Tried to save index with unsupported extension backend:", serializationBackend, "!" )
+
+        t1 = timer()
+        if printDebug >= 1:
+            print( "Writing out TAR index to", outFileName, "took {:.2f}s".format( t1 - t0 ),
+                   "and is sized", os.stat( outFileName ).st_size, "B" )
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        formatter_class = argparse.ArgumentDefaultsHelpFormatter,
+        description = '''\
+        Create index for random access to files inside the tar which will be saved to <path to tar>.index.<backend>[.<compression]. If it can't be saved there, it will be saved in ~/.ratarmount/<path to tar: '/' -> '_'>.index.<backend>[.<compression].
+        ''' )
+
+    parser.add_argument( '-d', '--debug', type = int, default = 1,
+                         help = 'sets the debugging level. Higher means more output. Currently 3 is the highest' )
+
+    parser.add_argument( '-r', '--recursive', action='store_true', default = False,
+                         help = 'index TAR archives inside the mounted TAR recursively.' )
+
+    parser.add_argument( '-s', '--serialization-backend', type = str, default = 'custom',
+                         help = 'specify which library to use for writing out the TAR index. Supported keywords: (' +
+                                ','.join( IndexedTar.availableSerializationBackends ) + ')[.(' +
+                                ','.join( IndexedTar.availableCompressions ).strip( ',' ) + ')]' )
+
+    parser.add_argument( 'tarfilepath', metavar = 'tar-file-path',
+                         type = argparse.FileType( 'r' ), nargs = 1,
+                         help = 'the path to the TAR archive to be mounted' )
+
+    args = parser.parse_args()
+
+    tarToMount = os.path.abspath( args.tarfilepath[0].name )
+    try:
+        tarfile.open( tarToMount, mode = 'r:' )
+    except tarfile.ReadError:
+        print( "Archive", tarToMount, "can't be opened!",
+               "This might happen for compressed TAR archives, which currently is not supported." )
+        exit( 1 )
+
+    printDebug = args.debug
+
+    IndexedTar( pathToTar = tarToMount, 
+                writeIndex = True,
+                recursive = args.recursive,
+                serializationBackend = args.serialization_backend )

python/ratarmount.py

@@ -0,0 +1,706 @@
+#!/usr/bin/env python3
+
+import os, re, sys, stat, tarfile, fuse, argparse
+import itertools
+from collections import namedtuple
+from timeit import default_timer as timer
+
+
+printDebug = 1
+
+def overrides( parentClass ):
+    def overrider( method ):
+        assert( method.__name__ in dir( parentClass ) )
+        return method
+    return overrider
+
+
+FileInfo = namedtuple( "FileInfo", "offset size mtime mode type linkname uid gid istar" )
+
+
+class IndexedTar( object ):
+    """
+    This class reads once through a whole TAR archive and stores TAR file offsets for all packed files
+    in an index to support fast seeking to a given file.
+    """
+
+    __slots__ = (
+        'tarFileName',
+        'fileIndex',
+        'mountRecursively',
+        'cacheFolder',
+        'possibleIndexFilePaths',
+        'indexFileName',
+    )
+
+    # these allowed backends also double as extensions for the index file to look for
+    availableSerializationBackends = [
+        'pickle',
+        'pickle2',
+        'pickle3',
+        'custom',
+        'cbor',
+        'msgpack',
+        'rapidjson',
+        'ujson',
+        'simplejson'
+    ]
+    availableCompressions = [
+        '', # no compression
+        'lz4',
+        'gz',
+    ]
+
+    def __init__( self, pathToTar = None, fileObject = None, writeIndex = False, clearIndexCache = False,
+                  recursive = False, serializationBackend = None ):
+        self.tarFileName = os.path.normpath( pathToTar )
+        # Stores the file hierarchy in a dictionary with keys being either the file and containing file metainformation
+        # or keys being a folder name and containing recursively defined dictionary.
+        self.fileIndex = {}
+        self.mountRecursively = recursive
+
+        self.cacheFolder = os.path.expanduser( "~/.ratarmount" ) # will be used for storing if current path is read-only
+        self.possibleIndexFilePaths = [
+            self.tarFileName + ".index",
+            self.cacheFolder + "/" + self.tarFileName.replace( "/", "_" ) + ".index"
+        ]
+
+        if serializationBackend not in self.supportedIndexExtensions():
+            serializationBackend = 'custom'
+            print( "[Warning] Serialization backend not supported. Defaulting to '" + serializationBackend + "'!" )
+
+        # this is the actual index file, which will be used in the end, and by default
+        self.indexFileName = self.possibleIndexFilePaths[0] + "." + serializationBackend
+
+        if clearIndexCache:
+            for indexPath in self.possibleIndexFilePaths:
+                for extension in self.supportedIndexExtensions():
+                    indexPathWitExt = indexPath + "." + extension
+                    if os.path.isfile( indexPathWitExt ):
+                        os.remove( indexPathWitExt )
+
+        if fileObject is not None:
+            if writeIndex:
+                print( "Can't write out index for file object input. Ignoring this option." )
+            self.createIndex( fileObject )
+        else:
+            # first try loading the index for the given serialization backend
+            if serializationBackend is not None:
+                for indexPath in self.possibleIndexFilePaths:
+                    indexPathWitExt = indexPath + "." + serializationBackend
+
+                    if self.indexIsLoaded():
+                        break
+
+                    if os.path.isfile( indexPathWitExt ):
+                        if os.path.getsize( indexPathWitExt ) == 0:
+                            os.remove( indexPathWitExt )
+                        else:
+                            self.loadIndex( indexPathWitExt )
+
+            # try loading the index from one of the pre-configured paths
+            for indexPath in self.possibleIndexFilePaths:
+                for extension in self.supportedIndexExtensions():
+                    indexPathWitExt = indexPath + "." + extension
+
+                    if self.indexIsLoaded():
+                        break
+
+                    if os.path.isfile( indexPathWitExt ):
+                        if os.path.getsize( indexPathWitExt ) == 0:
+                            os.remove( indexPathWitExt )
+                        else:
+                            self.loadIndex( indexPathWitExt )
+
+            if not self.indexIsLoaded():
+                with open( self.tarFileName, 'rb' ) as file:
+                    self.createIndex( file )
+
+                if writeIndex:
+                    for indexPath in self.possibleIndexFilePaths:
+                        indexPath += "." + serializationBackend
+
+                        try:
+                            folder = os.path.dirname( indexPath )
+                            if not os.path.exists( folder ):
+                                os.mkdir( folder )
+
+                            f = open( indexPath, 'wb' )
+                            f.close()
+                            os.remove( indexPath )
+                            self.indexFileName = indexPath
+
+                            break
+                        except IOError:
+                            if printDebug >= 2:
+                                print( "Could not create file:", indexPath )
+
+                    try:
+                        self.writeIndex( self.indexFileName )
+                    except IOError:
+                        print( "[Info] Could not write TAR index to file. Subsequent mounts might be slow!" )
+
+    @staticmethod
+    def supportedIndexExtensions():
+        return [ '.'.join( combination ).strip( '.' )
+                 for combination in itertools.product( IndexedTar.availableSerializationBackends,
+                                                       IndexedTar.availableCompressions ) ]
+    @staticmethod
+    def dump( toDump, file ):
+        if isinstance( toDump, dict ):
+            file.write( b'\x01' ) # magic code meaning "start dictionary object"
+
+            for key, value in toDump.items():
+                file.write( b'\x03' ) # magic code meaning "serialized key value pair"
+                IndexedTar.dump( key, file )
+                IndexedTar.dump( value, file )
+
+            file.write( b'\x02' ) # magic code meaning "close dictionary object"
+
+        elif isinstance( toDump, FileInfo ):
+            import msgpack
+            serialized = msgpack.dumps( toDump )
+            file.write( b'\x05' ) # magic code meaning "msgpack object"
+            file.write( len( serialized ).to_bytes( 4, byteorder = 'little' ) )
+            file.write( serialized )
+
+        elif isinstance( toDump, str ):
+            serialized = toDump.encode()
+            file.write( b'\x04' ) # magic code meaning "string object"
+            file.write( len( serialized ).to_bytes( 4, byteorder = 'little' ) )
+            file.write( serialized )
+
+        else:
+            print( "Ignoring unsupported type to write:", toDump )
+
+    @staticmethod
+    def load( file ):
+        elementType = file.read( 1 )
+
+        if elementType == b'\x01': # start of dictionary
+            result = {}
+
+            dictElementType = file.read( 1 )
+            while len( dictElementType ) != 0:
+                if dictElementType == b'\x02':
+                    break
+
+                elif dictElementType == b'\x03':
+                    import msgpack
+
+                    keyType = file.read( 1 )
+                    if keyType != b'\x04': # key must be string object
+                        raise Exception( 'Custom TAR index loader: invalid file format' )
+                    size = int.from_bytes( file.read( 4 ), byteorder = 'little' )
+                    key = file.read( size ).decode()
+
+                    valueType = file.read( 1 )
+                    if valueType == b'\x05': # msgpack object
+                        size = int.from_bytes( file.read( 4 ), byteorder = 'little' )
+                        serialized = file.read( size )
+                        value = FileInfo( *msgpack.loads( serialized ) )
+
+                    elif valueType == b'\x01': # dict object
+                        import io
+                        file.seek( -1, io.SEEK_CUR )
+                        value = IndexedTar.load( file )
+
+                    else:
+                        raise Exception( 'Custom TAR index loader: invalid file format ' +
+                            '(expected msgpack or dict but got' +
+                            str( int.from_bytes( valueType, byteorder = 'little' ) ) + ')' )
+
+                    result[key] = value
+
+                else:
+                    raise Exception( 'Custom TAR index loader: invalid file format ' +
+                        '(expected end-of-dict or key-value pair but got' +
+                        str( int.from_bytes( dictElementType, byteorder = 'little' ) ) + ')' )
+
+                dictElementType = file.read( 1 )
+
+            return result
+
+        else:
+            raise Exception( 'Custom TAR index loader: invalid file format' )
+
+    def getFileInfo( self, path, listDir = False ):
+        # go down file hierarchy tree along the given path
+        p = self.fileIndex
+        for name in os.path.normpath( path ).split( os.sep ):
+            if not name:
+                continue
+            if not name in p:
+                return
+            p = p[name]
+
+        def repackDeserializedNamedTuple( p ):
+            if isinstance( p, list ) and len( p ) == len( FileInfo._fields ):
+                return FileInfo( *p )
+            elif isinstance( p, dict ) and len( p ) == len( FileInfo._fields ) and \
+                 'uid' in p and isinstance( p['uid'], int ):
+                # a normal directory dict must only have dict or FileInfo values, so if the value to the 'uid'
+                # key is an actual int, then it is sure it is a deserialized FileInfo object and not a file named 'uid'
+                print( "P ===", p )
+                print( "FileInfo ===", FileInfo( **p ) )
+                return FileInfo( **p )
+            return p
+
+        p = repackDeserializedNamedTuple( p )
+
+        # if the directory contents are not to be printed and it is a directory, return the "file" info of "."
+        if not listDir and isinstance( p, dict ):
+            if '.' in p:
+                p = p['.']
+            else:
+                return FileInfo(
+                    offset   = 0, # not necessary for directory anyways
+                    size     = 1, # might be misleading / non-conform
+                    mtime    = 0,
+                    mode     = 0o555 | stat.S_IFDIR,
+                    type     = tarfile.DIRTYPE,
+                    linkname = "",
+                    uid      = 0,
+                    gid      = 0,
+                    istar    = False
+                )
+
+        return repackDeserializedNamedTuple( p )
+
+    def isDir( self, path ):
+        return True if isinstance( self.getFileInfo( path, listDir = True ), dict ) else False
+
+    def exists( self, path ):
+        path = os.path.normpath( path )
+        return self.isDir( path ) or isinstance( self.getFileInfo( path ), FileInfo )
+
+    def setFileInfo( self, path, fileInfo ):
+        """
+        path: the full path to the file with leading slash (/) for which to set the file info
+        """
+        assert( isinstance( fileInfo, FileInfo ) )
+
+        pathHierarchy = os.path.normpath( path ).split( os.sep )
+        if len( pathHierarchy ) == 0:
+            return
+
+        # go down file hierarchy tree along the given path
+        p = self.fileIndex
+        for name in pathHierarchy[:-1]:
+            if not name:
+                continue
+            assert( isinstance( p, dict ) )
+            p = p.setdefault( name, {} )
+
+        # create a new key in the dictionary of the parent folder
+        p.update( { pathHierarchy[-1] : fileInfo } )
+
+    def setDirInfo( self, path, dirInfo, dirContents = {} ):
+        """
+        path: the full path to the file with leading slash (/) for which to set the folder info
+        """
+        assert( isinstance( dirInfo, FileInfo ) )
+        assert( isinstance( dirContents, dict ) )
+
+        pathHierarchy = os.path.normpath( path ).strip( os.sep ).split( os.sep )
+        if len( pathHierarchy ) == 0:
+            return
+
+        # go down file hierarchy tree along the given path
+        p = self.fileIndex
+        for name in pathHierarchy[:-1]:
+            if not name:
+                continue
+            assert( isinstance( p, dict ) )
+            p = p.setdefault( name, {} )
+
+        # create a new key in the dictionary of the parent folder
+        p.update( { pathHierarchy[-1] : dirContents } )
+        p[pathHierarchy[-1]].update( { '.' : dirInfo } )
+
+    def createIndex( self, fileObject ):
+        if printDebug >= 1:
+            print( "Creating offset dictionary for", "<file object>" if self.tarFileName is None else self.tarFileName, "..." )
+        t0 = timer()
+
+        self.fileIndex = {}
+        try:
+            loadedTarFile = tarfile.open( fileobj = fileObject, mode = 'r:' )
+        except tarfile.ReadError as exception:
+            print( "Archive can't be opened! This might happen for compressed TAR archives, which currently is not supported." )
+            raise exception
+
+        for tarInfo in loadedTarFile:
+            mode = tarInfo.mode
+            if tarInfo.isdir() : mode |= stat.S_IFDIR
+            if tarInfo.isfile(): mode |= stat.S_IFREG
+            if tarInfo.issym() : mode |= stat.S_IFLNK
+            if tarInfo.ischr() : mode |= stat.S_IFCHR
+            if tarInfo.isfifo(): mode |= stat.S_IFIFO
+            fileInfo = FileInfo(
+                offset   = tarInfo.offset_data,
+                size     = tarInfo.size       ,
+                mtime    = tarInfo.mtime      ,
+                mode     = mode               ,
+                type     = tarInfo.type       ,
+                linkname = tarInfo.linkname   ,
+                uid      = tarInfo.uid        ,
+                gid      = tarInfo.gid        ,
+                istar    = False
+            )
+
+            # open contained tars for recursive mounting
+            indexedTar = None
+            if self.mountRecursively and tarInfo.isfile() and tarInfo.name.endswith( ".tar" ):
+                oldPos = fileObject.tell()
+                if oldPos != tarInfo.offset_data:
+                    fileObject.seek( tarInfo.offset_data )
+                indexedTar = IndexedTar( tarInfo.name, fileObject = fileObject, writeIndex = False )
+                fileObject.seek( fileObject.tell() ) # might be especially necessary if the .tar is not actually a tar!
+
+            # Add a leading '/' as a convention where '/' represents the TAR root folder
+            # Partly, done because fusepy specifies paths in a mounted directory like this
+            path = os.path.normpath( "/" + tarInfo.name )
+
+            # test whether the TAR file could be loaded and if so "mount" it recursively
+            if indexedTar is not None and indexedTar.indexIsLoaded():
+                # actually apply the recursive tar mounting
+                extractedName = re.sub( r"\.tar$", "", path )
+                if not self.exists( extractedName ):
+                    path = extractedName
+
+                mountMode = ( fileInfo.mode & 0o777 ) | stat.S_IFDIR
+                if mountMode & stat.S_IRUSR != 0: mountMode |= stat.S_IXUSR
+                if mountMode & stat.S_IRGRP != 0: mountMode |= stat.S_IXGRP
+                if mountMode & stat.S_IROTH != 0: mountMode |= stat.S_IXOTH
+                fileInfo = fileInfo._replace( mode = mountMode, istar = True )
+
+                if self.exists( path ):
+                    print( "[Warning]", path, "already exists in database and will be overwritten!" )
+
+                # merge fileIndex from recursively loaded TAR into our Indexes
+                self.setDirInfo( path, fileInfo, indexedTar.fileIndex )
+
+            elif path != '/':
+                # just a warning and check for the path already existing
+                if self.exists( path ):
+                    fileInfo = self.getFileInfo( path, listDir = False )
+                    if fileInfo.istar:
+                        # move recursively mounted TAR directory to original .tar name if there is a name-clash,
+                        # e.g., when foo/ also exists in the TAR but foo.tar would be mounted to foo/.
+                        # In this case, move that mount to foo.tar/
+                        self.setFileInfo( path + ".tar", fileInfo, self.getFileInfo( path, listDir = True ) )
+                    else:
+                        print( "[Warning]", path, "already exists in database and will be overwritten!" )
+
+                # simply store the file or directory information from current TAR item
+                if tarInfo.isdir():
+                    self.setDirInfo( path, fileInfo, {} )
+                else:
+                    self.setFileInfo( path, fileInfo )
+
+        t1 = timer()
+        if printDebug >= 1:
+            print( "Creating offset dictionary for", "<file object>" if self.tarFileName is None else self.tarFileName, "took {:.2f}s".format( t1 - t0 ) )
+
+    def serializationBackendFromFileName( self, fileName ):
+        splitName = fileName.split( '.' )
+
+        if len( splitName ) > 2 and '.'.join( splitName[-2:] ) in self.supportedIndexExtensions():
+            return '.'.join( splitName[-2:] )
+        elif splitName[-1] in self.supportedIndexExtensions():
+            return splitName[-1]
+        return None
+
+    def indexIsLoaded( self ):
+        return True if self.fileIndex else False
+
+    def writeIndex( self, outFileName ):
+        """
+        outFileName: full file name with backend extension. Depending on the extension the serialization is chosen.
+        """
+
+        serializationBackend = self.serializationBackendFromFileName( outFileName )
+
+        if printDebug >= 1:
+            print( "Writing out TAR index using", serializationBackend, "to", outFileName, "..." )
+        t0 = timer()
+
+        fileMode = 'wt' if 'json' in serializationBackend else 'wb'
+
+        if serializationBackend.endswith( '.lz4' ):
+            import lz4.frame
+            wrapperOpen = lambda x : lz4.frame.open( x, fileMode )
+        elif serializationBackend.endswith( '.gz' ):
+            import gzip
+            wrapperOpen = lambda x : gzip.open( x, fileMode )
+        else:
+            wrapperOpen = lambda x : open( x, fileMode )
+        serializationBackend = serializationBackend.split( '.' )[0]
+
+        # libraries tested but not working:
+        #  - marshal: can't serialize namedtuples
+        #  - hickle: for some reason, creates files almost 64x larger as pickle!? And also takes similarly longer
+        #  - yaml: almost a 10 times slower and more memory usage and deserializes everything including ints to string
+
+        with wrapperOpen( outFileName ) as outFile:
+            if serializationBackend == 'pickle2':
+                import pickle
+                pickle.dump( self.fileIndex, outFile )
+                pickle.dump( self.fileIndex, outFile, protocol = 2 )
+
+            # default serialization because it has the fewest dependencies and because it was legacy default
+            elif serializationBackend == 'pickle3' or \
+                 serializationBackend == 'pickle' or \
+                 serializationBackend is None:
+                import pickle
+                pickle.dump( self.fileIndex, outFile )
+                pickle.dump( self.fileIndex, outFile, protocol = 3 ) # 3 is default protocol
+
+            elif serializationBackend == 'simplejson':
+                import simplejson
+                simplejson.dump( self.fileIndex, outFile, namedtuple_as_object = True )
+
+            elif serializationBackend == 'custom':
+                IndexedTar.dump( self.fileIndex, outFile )
+
+            elif serializationBackend in [ 'msgpack', 'cbor', 'rapidjson', 'ujson' ]:
+                import importlib
+                module = importlib.import_module( serializationBackend )
+                getattr( module, 'dump' )( self.fileIndex, outFile )
+
+            else:
+                print( "Tried to save index with unsupported extension backend:", serializationBackend, "!" )
+
+        t1 = timer()
+        if printDebug >= 1:
+            print( "Writing out TAR index to", outFileName, "took {:.2f}s".format( t1 - t0 ),
+                   "and is sized", os.stat( outFileName ).st_size, "B" )
+
+    def loadIndex( self, indexFileName ):
+        if printDebug >= 1:
+            print( "Loading offset dictionary from", indexFileName, "..." )
+        t0 = timer()
+
+        serializationBackend = self.serializationBackendFromFileName( indexFileName )
+
+        fileMode = 'rt' if 'json' in serializationBackend else 'rb'
+
+        if serializationBackend.endswith( '.lz4' ):
+            import lz4.frame
+            wrapperOpen = lambda x : lz4.frame.open( x, fileMode )
+        elif serializationBackend.endswith( '.gz' ):
+            import gzip
+            wrapperOpen = lambda x : gzip.open( x, fileMode )
+        else:
+            wrapperOpen = lambda x : open( x, fileMode )
+        serializationBackend = serializationBackend.split( '.' )[0]
+
+        with wrapperOpen( indexFileName ) as indexFile:
+            if serializationBackend == 'pickle2' or \
+               serializationBackend == 'pickle3' or \
+               serializationBackend == 'pickle':
+                import pickle
+                self.fileIndex = pickle.load( indexFile )
+
+            elif serializationBackend == 'custom':
+                self.fileIndex = IndexedTar.load( indexFile )
+
+            elif serializationBackend == 'msgpack':
+                import msgpack
+                self.fileIndex = msgpack.load( indexFile, raw = False )
+
+            elif serializationBackend == 'simplejson':
+                import simplejson
+                self.fileIndex = simplejson.load( indexFile, namedtuple_as_object = True )
+
+            elif serializationBackend in [ 'cbor', 'rapidjson', 'ujson' ]:
+                import importlib
+                module = importlib.import_module( serializationBackend )
+                self.fileIndex = getattr( module, 'load' )( indexFile )
+
+            else:
+                print( "Tried to load index path with unsupported serializationBackend:", serializationBackend, "!" )
+                return
+
+        if printDebug >= 2:
+            def countDictEntries( d ):
+                n = 0
+                for key, value in d.items():
+                    n += countDictEntries( value ) if type( value ) is dict else 1
+                return n
+            print( "Files:", countDictEntries( self.fileIndex ) )
+
+        t1 = timer()
+        if printDebug >= 1:
+            print( "Loading offset dictionary from", indexFileName, "took {:.2f}s".format( t1 - t0 ) )
+
+
+class TarMount( fuse.Operations ):
+    """
+    This class implements the fusepy interface in order to create a mounted file system view
+    to a TAR archive.
+    This class can and is relatively thin as it only has to create and manage an IndexedTar
+    object and query it for directory or file contents.
+    It also adds a layer over the file permissions as all files must be read-only even
+    if the TAR reader reports the file as originally writable because no TAR write support
+    is planned.
+    """
+
+    def __init__( self, pathToMount, clearIndexCache = False, recursive = False, serializationBackend = None ):
+        self.tarFileName = pathToMount
+        self.tarFile = open( self.tarFileName, 'rb' )
+        self.indexedTar = IndexedTar( self.tarFileName, writeIndex = True,
+                                      clearIndexCache = clearIndexCache, recursive = recursive,
+                                      serializationBackend = serializationBackend )
+
+        # make the mount point read only and executable if readable, i.e., allow directory listing
+        tarStats = os.stat( self.tarFileName )
+        # clear higher bits like S_IFREG and set the directory bit instead
+        mountMode = ( tarStats.st_mode & 0o777 ) | stat.S_IFDIR
+        if mountMode & stat.S_IRUSR != 0: mountMode |= stat.S_IXUSR
+        if mountMode & stat.S_IRGRP != 0: mountMode |= stat.S_IXGRP
+        if mountMode & stat.S_IROTH != 0: mountMode |= stat.S_IXOTH
+        self.indexedTar.fileIndex[ '.' ] = FileInfo(
+            offset   = 0                ,
+            size     = tarStats.st_size ,
+            mtime    = tarStats.st_mtime,
+            mode     = mountMode        ,
+            type     = tarfile.DIRTYPE  ,
+            linkname = ""               ,
+            uid      = tarStats.st_uid  ,
+            gid      = tarStats.st_gid  ,
+            istar    = True
+        )
+
+        if printDebug >= 3:
+            print( "Loaded File Index:", self.indexedTar.fileIndex )
+
+    @overrides( fuse.Operations )
+    def getattr( self, path, fh = None ):
+        if printDebug >= 2:
+            print( "[getattr( path =", path, ", fh =", fh, ")] Enter" )
+
+        fileInfo = self.indexedTar.getFileInfo( path, listDir = False )
+        if not isinstance( fileInfo, FileInfo ):
+            if printDebug >= 2:
+                print( "Could not find path:", path )
+            raise fuse.FuseOSError( fuse.errno.EROFS )
+
+        # dictionary keys: https://pubs.opengroup.org/onlinepubs/007904875/basedefs/sys/stat.h.html
+        statDict = dict( ( "st_" + key, getattr( fileInfo, key ) ) for key in ( 'size', 'mtime', 'mode', 'uid', 'gid' ) )
+        # signal that everything was mounted read-only
+        statDict['st_mode'] &= ~( stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH )
+        statDict['st_mtime'] = int( statDict['st_mtime'] )
+        statDict['st_nlink'] = 2
+
+        if printDebug >= 2:
+            print( "[getattr( path =", path, ", fh =", fh, ")] return:", statDict )
+
+        return statDict
+
+    @overrides( fuse.Operations )
+    def readdir( self, path, fh ):
+        if printDebug >= 2:
+            print( "[readdir( path =", path, ", fh =", fh, ")] return:",
+                   self.indexedTar.getFileInfo( path, listDir = True ).keys() )
+
+        # we only need to return these special directories. FUSE automatically expands these and will not ask
+        # for paths like /../foo/./../bar, so we don't need to worry about cleaning such paths
+        yield '.'
+        yield '..'
+
+        for key in self.indexedTar.getFileInfo( path, listDir = True ).keys():
+            yield key
+
+    @overrides( fuse.Operations )
+    def readlink( self, path ):
+        if printDebug >= 2:
+            print( "[readlink( path =", path, ")]" )
+
+        fileInfo = self.indexedTar.getFileInfo( path )
+        if not isinstance( fileInfo, FileInfo ):
+            raise fuse.FuseOSError( fuse.errno.EROFS )
+
+        pathname = fileInfo.linkname
+        if pathname.startswith( "/" ):
+            return os.path.relpath( pathname, self.root )
+        else:
+            return pathname
+
+    @overrides( fuse.Operations )
+    def read( self, path, length, offset, fh ):
+        if printDebug >= 2:
+            print( "[read( path =", path, ", length =", length, ", offset =", offset, ",fh =", fh, ")] path:", path )
+
+        fileInfo = self.indexedTar.getFileInfo( path )
+        if not isinstance( fileInfo, FileInfo ):
+            raise fuse.FuseOSError( fuse.errno.EROFS )
+
+        self.tarFile.seek( fileInfo.offset + offset, os.SEEK_SET )
+        return self.tarFile.read( length )
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        formatter_class = argparse.ArgumentDefaultsHelpFormatter,
+        description = '''\
+        If no mount path is specified, then the tar will be mounted to a folder of the same name but without a file extension.
+        TAR files contained inside the tar and even TARs in TARs in TARs will be mounted recursively at folders of the same name barred the file extension '.tar'.
+
+        In order to reduce the mounting time, the created index for random access to files inside the tar will be saved to <path to tar>.index.<backend>[.<compression]. If it can't be saved there, it will be saved in ~/.ratarmount/<path to tar: '/' -> '_'>.index.<backend>[.<compression].
+        ''' )
+
+    parser.add_argument( '-f', '--foreground', action='store_true', default = False,
+                         help = 'keeps the python program in foreground so it can print debug output when the mounted path is accessed.' )
+
+    parser.add_argument( '-d', '--debug', type = int, default = 1,
+                         help = 'sets the debugging level. Higher means more output. Currently 3 is the highest' )
+
+    parser.add_argument( '-c', '--recreate-index', action='store_true', default = False,
+                         help = 'if specified, pre-existing .index files will be deleted and newly created' )
+
+    parser.add_argument( '-r', '--recursive', action='store_true', default = False,
+                         help = 'mount TAR archives inside the mounted TAR recursively. Note that this only has an effect when creating an index. If an index already exists, then this option will be effectively ignored. Recreate the index if you want change the recursive mounting policy anyways.' )
+
+    parser.add_argument( '-s', '--serialization-backend', type = str, default = 'custom',
+                         help = 'specify which library to use for writing out the TAR index. Supported keywords: (' +
+                                ','.join( IndexedTar.availableSerializationBackends ) + ')[.(' +
+                                ','.join( IndexedTar.availableCompressions ).strip( ',' ) + ')]' )
+
+    parser.add_argument( 'tarfilepath', metavar = 'tar-file-path',
+                         type = argparse.FileType( 'r' ), nargs = 1,
+                         help = 'the path to the TAR archive to be mounted' )
+    parser.add_argument( 'mountpath', metavar = 'mount-path', nargs = '?',
+                         help = 'the path to a folder to mount the TAR contents into' )
+
+    args = parser.parse_args()
+
+    tarToMount = os.path.abspath( args.tarfilepath[0].name )
+    try:
+        tarfile.open( tarToMount, mode = 'r:' )
+    except tarfile.ReadError:
+        print( "Archive", tarToMount, "can't be opened!",
+               "This might happen for compressed TAR archives, which currently is not supported." )
+        exit( 1 )
+
+    mountPath = args.mountpath
+    if mountPath is None:
+        mountPath = os.path.splitext( tarToMount )[0]
+
+    mountPathWasCreated = False
+    if not os.path.exists( mountPath ):
+        os.mkdir( mountPath )
+
+    printDebug = args.debug
+
+    fuse.FUSE( operations = TarMount(
+                   pathToMount = tarToMount,
+                   clearIndexCache = args.recreate_index,
+                   recursive = args.recursive,
+                   serializationBackend = args.serialization_backend  ),
+               mountpoint = mountPath,
+               foreground = args.foreground )
+
+    if mountPathWasCreated and args.foreground:
+        os.rmdir( mountPath )

python/ucf/__init__.py

@@ -0,0 +1 @@
+from .ucf import *

python/ucf/ucf.py

@@ -0,0 +1,622 @@
+import sys
+import io
+import struct 
+import warnings
+import numpy as np
+import tarfile
+import time
+from datetime import datetime
+
+def __warning_format(message, category, filename, lineno, file=None, line=None):
+        return '%s:%s: %s:%s\n' % (filename, lineno, category.__name__, message)
+warnings.formatwarning = __warning_format
+
+#############################
+# Low-level class interface #
+#############################
+class UCF:
+    """UCF low-level access class"""
+    def __init__(self,file=None,verbosity=False,debug=False):
+        self.__initializeConstants()
+        self.__resetPublicProperties()
+        self.__resetPrivateProperties()
+        self.__resetCurrentStep()
+        self.__resetCurrentSet()
+        if file is not None:
+            self.open(file)
+        self.Debug = debug
+        self.Verbosity = verbosity
+        
+    def open(self,file):
+        """Opens an input stream for reading access. The variable 'file' can be of the following types:
+           str                    opens a file on disk whose path is specified by 'file'
+           tarfile.ExFileObject   read from a file inside a tar archive (use tarfile.extractfile to generate an instance for 'file')
+           bytes                  data which is already located in memory as a bytes or bytearray object (can be used for streams)
+           """
+        # Check what class 'file' belongs to and treat it accordingly
+        if isinstance(file,str):
+            self.File = file
+            self.__external = False
+            self.__stream   = False
+            self.__fileID   = open(self.File,'rb')
+            self.__inputAvailable = True
+        elif isinstance(file,tarfile.ExFileObject):
+            self.File = file.name
+            self.__external = True
+            self.__stream   = False
+            self.__fileID   = file
+            self.__inputAvailable = True
+        elif isinstance(file,bytes) or isinstance(file,bytearray):
+            self.File = 'byte-stream'
+            self.__external = True
+            self.__stream   = True
+            self.__fileID   = io.BytesIO(file)
+            self.__inputAvailable = True
+
+        # Determine file size
+        self.__fileID.seek(0,2)
+        self.FileSize = self.__fileID.tell()
+        self.__fileID.seek(self.__fileBeg,0)
+
+        # Read the header of the file
+        self.__readHeaderFile()
+
+        # Scan through file to get the basic structure (steps/sets)
+        self.__timeStep      = np.zeros(self.__scanBuffSize,dtype=np.float64)
+        self.__posStep       = np.zeros(self.__scanBuffSize,dtype=np.int32)
+        self.__numSetPerStep = np.zeros(self.__scanBuffSize,dtype=np.int32)
+        istep = 0;
+        while self.__fileID.tell()<self.FileSize:
+            self.__readHeaderStep();
+            self.__timeStep[istep]      = self.__currentStepTime
+            self.__posStep[istep]       = self.__fileID.tell()
+            self.__numSetPerStep[istep] = self.__currentStepNumSet
+            istep = istep+1
+            if self.__currentStepSize==-1:
+              break
+            else:
+              self.__fileID.seek(self.__currentStepSize,1)
+        nstep = istep
+
+        # Truncate buffered arrays
+        if nstep>self.__scanBuffSize:
+            warnings.warn('Buffer overflow detected: increase scanBuffSize.')
+        self.__timeStep      = self.__timeStep[0:nstep]
+        self.__posStep       = self.__posStep[0:nstep]
+        self.__numSetPerStep = self.__numSetPerStep[0:nstep]
+
+        # Set some internal variables
+        self.NumDataset  = np.max(self.__numSetPerStep)
+        self.NumTimestep = nstep;
+        self.__isFileHeaderWritten = True;
+        self.__isStepHeaderWritten = True;
+
+    def close(self):
+        """Closes input file object"""
+        if not isinstance(self.__fileID,tarfile.ExFileObject):
+            self.__fileID.close()
+        self.__init__
+  
+    def addFileHeaderToBuffer(self,rank=0,rankijk=(0,0,0),ftype=1999):
+        """Initialize a buffer to generate a new UCF file."""
+        self.__bufNumSteps  = 0
+        self.__bufStep      = []
+        self.__bufParams    = []
+        self.__bufData      = []
+        self.__bufRank      = rank
+        self.__bufRankijk   = rankijk
+        self.__bufFileType  = ftype
+        self.__bufAvailable = True
+
+    def addStepToBuffer(self,step=1,time=0.0):
+        """Add a new step to buffer."""
+        if not self.__bufAvailable:
+            raise BufferError('Buffer has not been initialized.')
+        if step>self.__bufNumSteps:
+            self.__bufStep.extend([None] for ii in range(self.__bufNumSteps,step))
+            self.__bufParams.extend([] for ii in range(self.__bufNumSteps,step))
+            self.__bufData.extend([] for ii in range(self.__bufNumSteps,step))
+            self.__bufNumSteps = step
+        self.__bufStep[step-1] = time
+
+    def addDatasetToBuffer(self,data,params=None,step=1,dset=1):
+        """Add a new dataset to specified step of buffer."""
+        if not self.__bufAvailable:
+            raise BufferError('Buffer has not been initialized.')
+        if step>self.__bufNumSteps:
+            raise ValueError('Requested step does not exist.')
+        if not hasattr(data,'dtype'):
+            raise TypeError('Cannot determine datatype of provided data')
+        if not hasattr(data,'nbytes'):
+            raise TypeError('Cannot determine number of bytes of provided data')
+        if not hasattr(data,'tobytes'):
+            raise TypeError('Cannot convert provided data to bytes')
+        if not hasattr(data,'shape'):
+            raise TypeError('Cannot determine shape of provided data')
+        if params is not None and not all(np.issubdtype(type(ii),np.integer) for ii in params):
+            raise TypeError('Parameters must be provided as integer')
+
+        nset = len(self.__bufData[step-1])
+        if dset>nset:
+            self.__bufParams[step-1].extend(None for ii in range(nset,dset))
+            self.__bufData[step-1].extend(None for ii in range(nset,dset))
+        self.__bufParams[step-1][dset-1] = params
+        self.__bufData[step-1][dset-1] = data
+
+    def copyFileHeaderToBuffer(self):
+        if not self.__inputAvailable:
+            raise IOError('No input file available')
+        self.addFileHeaderToBuffer(rank=self.IORank[0],rankijk=self.IORank[1:],ftype=self.__typeID)
+
+    def copyStepToBuffer(self,step_in,step_out=1,recursive=False,singlePrecision=False):
+        """Copy a step from an input file to output buffer. If recursive copying is activated, all datasets
+           within the step will be copied, otherwise only the step header is copied without datasets.
+           If datasets are copied, the precision can be reduced using the 'singlePrecision' flag."""
+        if not self.__inputAvailable:
+            raise IOError('No input file available')
+        if not self.__bufAvailable:
+            raise BufferError('Buffer has not been initialized.')
+        self.addStepToBuffer(step=step_out,time=self.__timeStep[step_in-1])
+        if recursive:
+            for dset in range(0,self.__numSetPerStep[step_in-1]):
+                self.copyDatasetToBuffer(step_in,dset+1,step_out=step_out,dset_out=dset+1,singlePrecision=singlePrecision)
+
+    def copyDatasetToBuffer(self,step_in,dset_in,step_out=1,dset_out=1,singlePrecision=False):
+        """Copy a dataset from an input file to output buffer at specified step. The precision of the
+           dataset can be reduced using the 'singlePrecision' flag."""
+        if not self.__inputAvailable:
+            raise IOError('No input file available')
+        if not self.__bufAvailable:
+            raise BufferError('Buffer has not been initialized.')
+        (data,params) = self.readSet(step_in,dset_in)
+        if singlePrecision:
+            if data.dtype==np.dtype('float64'):
+                data = np.float32(data)
+            elif data.dtype==np.dtype('int64'):
+                data = np.int32(data)
+        self.addDatasetToBuffer(data,params=params,step=step_out,dset=dset_out)
+
+    def flushBuffer(self):
+        """Returns the buffer as a bytes object, which can be written to a file using a file object."""
+        # Sanity check and size gathering
+        sizeStep = []
+        sizeSet  = [[]]
+        for step in range(0,self.__bufNumSteps):
+            nset = len(self.__bufData[step])
+            tmpSizeStep = 0
+            if nset==0:
+                warnings.warn('Step #{} in buffer does not contain any dataset.'.format(step+1),RuntimeWarning)
+            for dset in range(0,nset):
+                if self.__bufData[step][dset] is None:
+                    raise ValueError('Step #{}, dataset #{} does not contain any data.'.format(step+1,dset+1))
+                if self.__bufParams[step][dset] is None:
+                    warnings.warn('No parameters were provided for step #{}, dataset #{}.'.format(step+1,dset+1),RuntimeWarning)
+                    nparam==0
+                else:
+                    nparam = len(self.__bufParams[step][dset])
+                sizeSet[step].append(self.__bufData[step][dset].nbytes)
+                tmpSizeStep += (self.__nHeaderSet+nparam)*self.__nByteHeaderSet
+                tmpSizeStep += self.__bufData[step][dset].nbytes 
+            sizeStep.append(tmpSizeStep)
+
+        # Create output buffer
+        obuff = b''
+
+        # Build file header
+        magicFile   = self.__magicFile
+        fileVersion = 2
+        unixTime    = int(time.time())
+        fileType    = self.__bufFileType
+        rank        = self.__bufRank
+        (iproc,jproc,kproc) = self.__bufRankijk
+        if self.Debug:
+            print('Write the following file header at {} bytes'.format(len(obuff)),file=sys.stderr)
+            print((magicFile,fileVersion,unixTime,fileType,rank,iproc,jproc,kproc),file=sys.stderr)
+        obuff += struct.pack('qqqqqqqq',magicFile,fileVersion,unixTime,fileType,rank,iproc,jproc,kproc)
+        
+        # Build step header
+        for step in range(0,self.__bufNumSteps):
+            if self.Verbosity:
+                print('Adding step #{} to output buffer'.format(step+1),file=sys.stderr)
+            magicStep = self.__magicStep
+            stepBytes = sizeStep[step]
+            stepTime  = self.__bufStep[step]
+            nset      = len(self.__bufData[step])
+            if self.Debug:
+                print('Write the following step header at {} bytes'.format(len(obuff)),file=sys.stderr)
+                print((magicStep,stepBytes,stepTime,nset),file=sys.stderr)
+            obuff += struct.pack('qqdq',magicStep,stepBytes,stepTime,nset)
+            # Build dataset headers + attach data
+            for dset in range(0,nset):
+                if self.Verbosity:
+                   print('  dataset #{}'.format(dset+1),file=sys.stderr)
+                magicSet = self.__magicSet
+                setSize  = sizeSet[step][dset]
+                nptype = self.__bufData[step][dset].dtype
+                if nptype==np.dtype('int32'):
+                    dtEncoded = 11
+                elif nptype==np.dtype('int64'):
+                    dtEncoded = 12
+                elif nptype==np.dtype('float32'):
+                    dtEncoded = 21
+                elif nptype==np.dtype('float64'):
+                    dtEncoded = 22
+                else:
+                    raise TypeError('Data at step #{}, dataset #{} has an invalid datatype.'.format(step+1,dset+1))
+                if self.__bufParams[step][dset] is None:
+                    nparam = 0
+                else:
+                    nparam = len(self.__bufParams[step][dset])
+                if self.Debug:
+                    print('Write the following set header at {} bytes'.format(len(obuff)),file=sys.stderr)
+                    print((magicSet,setSize,dtEncoded,nparam),file=sys.stderr)
+                    print('with parameters:',file=sys.stderr)
+                    print(self.__bufParams[step][dset],file=sys.stderr)
+
+                obuff += struct.pack('qqqq',magicSet,setSize,dtEncoded,nparam)
+                if nparam!=0:
+                    obuff += struct.pack(nparam*'q',*self.__bufParams[step][dset])
+                obuff += self.__bufData[step][dset].tobytes('F')
+        # Return bytes
+        return obuff
+
+    def readSet(self,step=1,dset=1,memmap=False):
+        """Read a dataset from input file. If 'memmap' is activated, the file will only be read partially on demand."""
+        if not self.__inputAvailable:
+            raise IOError('No input file available')
+        self.__fileID.seek(self.__findSet(step,dset),0)
+        self.__readHeaderSet();
+        params = self.__currentSetParams
+        if memmap:
+          if self.__external:
+            raise TypeError('Cannont memory map from tar-archive (yet)')
+          else:
+            data = np.memmap(self.__fileID,dtype=self.__currentSetDatatype,offset=self.__fileID.tell(),mode='c')
+        else:
+          if self.__external:
+            data = np.frombuffer(self.__fileID.read(self.__currentSetSize),dtype=self.__currentSetDatatype) 
+          else:
+            data = np.fromfile(self.__fileID,dtype=self.__currentSetDatatype,count=self.__currentSetNumElements)
+        return (data,params)
+    
+    def __readHeaderFile(self):
+        self.__fileID.seek(self.__fileBeg,0);
+        # Determine endianess
+        mfmt = "<>"
+        buff = self.__fileID.read(8)
+        for fmt in mfmt:
+            currentMagic = struct.unpack("%sq"%fmt,buff)[0]
+            if currentMagic==self.__magicFile:
+                break
+        if currentMagic!=self.__magicFile:
+            raise ValueError('Magic mismatch: invalid file header. {}'.format(currentMagic))
+        self.Endian = fmt
+        # Read header
+        self.__fileID.seek(self.__fileBeg,0);
+        buff = self.__fileID.read(self.__nHeaderFile*8)
+        header = struct.unpack("%s%dq"%(self.Endian,8),buff)
+        if self.Debug:
+            print('Read the following file header at 0 bytes',file=sys.stderr)
+            print(header,file=sys.stderr)
+        # Parse version
+        self.__versionMajor = np.floor(header[1]/self.__factorMajor)
+        self.__versionMinor = np.floor(np.mod(header[1],self.__factorMajor)/self.__factorMinor)
+        self.__versionPatch = np.floor(np.mod(header[1],self.__factorMinor)/self.__factorPatch)
+        self.CodeVersion  = "%d.%d.%d" %(self.__versionMajor,self.__versionMinor,self.__versionPatch)
+        self.UCFVersion   = np.mod(header[1],self.__factorPatch);
+        # Parse time stamp (UTC)
+        self.__creationTimeUnix = header[2];
+        self.CreationTime = datetime.utcfromtimestamp(self.__creationTimeUnix).strftime('%Y-%m-%d %H:%M:%S')
+        #Parse file type
+        self.__typeID = header[3];
+        typeDict = {
+               0: "grid",
+              10: "processor grid",
+            1000: "fluid snapshot",
+            1010: "scalar snapshot",
+            1999: "matlab field data",
+            2000: "particle snapshot",
+            2001: "particle append",
+            2011: "particle lagrange",
+            2021: "particle balancing",
+            2999: "matlab particle data",
+            3000: "statistics fluid",
+            3010: "statistics fluid pure",
+            3020: "statistics scalar"
+            }
+        self.Type = typeDict.get(self.__typeID,"unkmown")
+        # Parse file class
+        classDict = {
+            1: "field",
+            2: "particle",
+            3: "statistics"
+            }
+        self.Class = classDict.get(np.floor(self.__typeID/self.__factorTypeIDClass),"unknown")
+        # Parse IO rank
+        self.IORank = header[4:8]
+    
+    def __readHeaderStep(self):
+        # Read and parse
+        self.__currentStepPosHeader = self.__fileID.tell()
+        buff = self.__fileID.read(self.__nHeaderStep*8)
+        header = struct.unpack("%s%dq"%(self.Endian,self.__nHeaderStep),buff)
+        self.__currentStepPosData = self.__fileID.tell()
+        currentMagic = header[0]
+        self.__currentStepSize   = header[1]
+        self.__currentStepTime   = struct.unpack("%sd"%self.Endian,buff[16:24])[0]
+        self.__currentStepNumSet = header[3]
+        if self.Debug:
+            print("Read the following step header at %d bytes" % self.__currentStepPosHeader,file=sys.stderr)
+            print("%d,%d,%f,%d" % (currentMagic,self.__currentStepSize,self.__currentStepTime,self.__currentStepNumSet),file=sys.stderr) 
+        # Check if magic is correct
+        if currentMagic!=self.__magicStep:
+            raise ValueError("Magic mismatch: invalid step header. %d" & currentMagic);
+    
+    def __readHeaderSet(self):
+      # Read and parse
+      self.__currentSetPosHeader = self.__fileID.tell()
+      buff = self.__fileID.read(self.__nHeaderSet*8)
+      header = struct.unpack("%s%dq"%(self.Endian,self.__nHeaderSet),buff)
+      self.__currentSetPosData = self.__fileID.tell()
+      currentMagic = header[0]
+      self.__currentSetSize = header[1]
+      self.__currentSetDatatypeNumeric = header[2]
+      dtSizeDict = {
+        11: 4,
+        12: 8,
+        21: 4,
+        22: 8
+        }
+      dtNameDict = {
+        11: "%si4" % self.Endian,
+        12: "%si8" % self.Endian,
+        21: "%sf4" % self.Endian,
+        22: "%sf8" % self.Endian
+      }
+      self.__currentSetSizeof      = dtSizeDict[self.__currentSetDatatypeNumeric]
+      self.__currentSetDatatype    = dtNameDict[self.__currentSetDatatypeNumeric]
+      self.__currentSetNumParams   = header[3]
+      self.__currentSetNumElements = np.around(self.__currentSetSize/self.__currentSetSizeof).astype(np.int32)
+      if self.Debug:
+          print("Read the following set header at %d bytes" % self.__currentSetPosHeader,file=sys.stderr)
+          print(header,file=sys.stderr)
+      # Check if magic is correct
+      if currentMagic!=self.__magicSet:
+          raise ValueError("Magic mismatch: invalid dataset header. %d" % currentMagic)
+      # Read variable number of parameters
+      buff = self.__fileID.read(self.__currentSetNumParams*8)
+      self.__currentSetParams = struct.unpack("%s%dq"%(self.Endian,self.__currentSetNumParams),buff)
+      if self.Debug:
+          print('with parameters:',file=sys.stderr)
+          print(self.__currentSetParams,file=sys.stderr)
+    
+    def __findSet(self,tstep,dset):
+        # Check input
+        if tstep>self.NumTimestep:
+            raise ValueError("Out of bounds: timestep. %d, %d" %(tstep,self.NumTimestep))
+        if dset>self.__numSetPerStep[tstep-1]:
+            raise ValueError("Out of bounds: dataset. %d, %d" % (dset,self.NumDataset))
+        # Navigate to correct set
+        self.__fileID.seek(self.__posStep[tstep-1],0)
+        for iset in range(0,dset-1):
+            self.__readHeaderSet()
+            self.__fileID.seek(self.__currentSetSize,1)
+        posHeader = self.__fileID.tell()
+        if self.Debug:
+            print("Found step #%d, set #%d at position %d" % (tstep,dset,posHeader),file=sys.stderr)
+        return posHeader
+    
+    def __initializeConstants(self):
+        self.__magicFile                 = 81985529216486895;
+        self.__magicStep                 = 11944304052957;
+        self.__magicSet                  = 240217520921210;
+        self.__nHeaderFile               = 8;
+        self.__nHeaderStep               = 4;
+        self.__nHeaderSet                = 4;
+        self.__nByteHeaderFile           = 8;
+        self.__nByteHeaderStep           = 8;
+        self.__nByteHeaderSet            = 8;
+        self.__nSetParamsField           = 10;
+        self.__nSetParamsParticle        = 16;
+        self.__factorMajor               = 1000000000;
+        self.__factorMinor               = 1000000;
+        self.__factorPatch               = 1000;
+        self.__factorTypeIDClass         = 1000;
+        self.__factorTypeIDKind          = 10;
+        self.__typeIDmatlabField         = 1999;
+        self.__typeIDmatlabParticle      = 2999;
+        self.__scanBuffSize              = 4096;
+    def __resetPublicProperties(self):
+        self.File         = '' # file name
+        self.Type         = '' # file type
+        self.Class        = '' # file class
+        self.Endian       = '' # endianess
+        self.CodeVersion  = '' # version of the simulation code
+        self.UCFVersion   = '' # version of the data format ("unified container format")
+        self.NumDataset   = 0  # maximum number of datasets in this file (over all time steps)
+        self.NumTimestep  = 0  # number of time steps in this file
+        self.FileSize     = 0  # file size
+        self.CreationTime = 0  # time of creation
+        self.IOMode       = '' # file opened in read-only or read-write mode?
+        self.IORank       = 0  # rank of processor + col,row,pln
+        self.Verbosity    = 0  # verbose output?
+        self.Debug        = 0  # debug information?
+    def __resetPrivateProperties(self):
+        self.__fileID                    = None
+        self.__fileBeg                   = 0
+        self.__typeID                    = 0
+        self.__creationTimeUnix          = ''
+        self.__versionMajor              = 0
+        self.__versionMinor              = 0
+        self.__versionPatch              = 0
+        self.__versionFile               = 0
+        self.__posStep                   = 0
+        self.__numSetPerStep             = 0
+        self.__timeStep                  = 0
+        self.__inputAvailable            = False
+        self.__stream                    = False
+        self.__external                  = False
+        self.__bufAvailable              = False
+    def __resetCurrentStep(self):
+        self.__currentStep               = 0
+        self.__currentStepPosHeader      = 0
+        self.__currentStepPosData        = 0
+        self.__currentStepSize           = 0
+        self.__currentStepTime           = 0
+        self.__currentStepNumSet         = 0
+    def __resetCurrentSet(self):
+        self.__currentSet                = 0
+        self.__currentSetPosHeader       = 0
+        self.__currentSetPosData         = 0
+        self.__currentSetSize            = 0
+        self.__currentSetDatatype        = 0
+        self.__currentSetDatatypeNumeric = 0
+        self.__currentSetSizeof          = 0
+        self.__currentSetNumParams       = 0
+        self.__currentSetParams          = 0
+        self.__currentSetNumElements     = 0
+
+#################################
+# High-level function interface #
+#################################
+def readGrid(file,verbosity=False,debug=False):
+    obj = UCF(file=file,verbosity=verbosity,debug=debug)
+    output = []
+    for iset in range(0,obj.NumDataset):
+        (data,params) = obj.readSet(step=1,dset=iset+1)
+        nx = params[0]
+        ny = params[1]
+        nz = params[2]
+        output.append(data[0:nx])
+        output.append(data[nx:nx+ny])
+        output.append(data[nx+ny:nx+ny+nz])
+    #if obj.UCFVersion<2:
+    if obj.NumDataset<5:
+        output.extend(output[-3:])
+    obj.close()
+    return output
+
+def readProcgrid(file,verbosity=False,debug=False):
+    obj = UCF(file=file,verbosity=verbosity,debug=debug)
+    output = []
+    for iset in range(0,obj.NumDataset):
+        (data,params) = obj.readSet(step=1,dset=iset+1)
+        nxp = params[0]
+        nyp = params[1]
+        nzp = params[2]
+        output.append(data[0:nxp])                             # ibeg
+        output.append(data[nxp:2*nxp])                         # iend
+        output.append(data[2*nxp:2*nxp+nyp])                   # jbeg
+        output.append(data[2*nxp+nyp:2*nxp+2*nyp])             # jend
+        output.append(data[2*nxp+2*nyp:2*nxp+2*nyp+nzp])       # kbeg
+        output.append(data[2*nxp+2*nyp+nzp:2*nxp+2*nyp*2*nzp]) # kend
+    #if obj.UCFVersion<2:
+    if obj.NumDataset<5:
+        output.extend(output[-6:])
+    obj.close()
+    return output
+
+def readFieldChunk(file,step=1,dset=-1,verbosity=False,debug=False):
+    obj = UCF(file=file,verbosity=verbosity,debug=debug)
+    if not isinstance(dset,list):
+        if dset==-1:
+            dset = range(1,obj.NumDataset+1) # fix that maybe later (this is maximum over all timesteps)
+        else:
+            dset = [dset]
+    output = []
+    for ii in dset:
+        tmp = dict()
+        (data,params) = obj.readSet(step=step,dset=ii)
+        tmp['ighost'] = params[0]
+        tmp['ibeg']   = params[1]
+        tmp['jbeg']   = params[2]
+        tmp['kbeg']   = params[3]
+        tmp['nxl']    = params[4]
+        tmp['nyl']    = params[5]
+        tmp['nzl']    = params[6]
+        tmp['nx']     = params[7]
+        tmp['ny']     = params[8]
+        tmp['nz']     = params[9]
+        tmp['data']   = data.reshape((tmp['nxl']+2*tmp['ighost'],
+                                      tmp['nyl']+2*tmp['ighost'],
+                                      tmp['nzl']+2*tmp['ighost']),
+                                     order='F')
+        tmp['rank']   = obj.IORank[0]
+        tmp['rankijk']= obj.IORank[1:]
+        output.append(tmp)
+    obj.close()
+    return output
+
+def readParticles(file,step=-1,verbosity=False,debug=False):
+    # Check what kind of file was passed: standalone, tar, bytes
+    # TBD: tar is not supported yet
+    obj = UCF(file=file,verbosity=verbosity,debug=debug)
+    if not isinstance(step,list):
+        if step==-1:
+            step = range(1,obj.NumTimestep+1)
+        else:
+            step = [step]
+    # The output will be the following: 
+    # 1) numpy array with dimension (ncol,np,ntime) 
+    # 2) dictionary which specifies the columns
+    # We read the data step by step in a list, which is then converted to a 3D array
+    pp = []
+    for ii in step:
+      (data,params) = obj.readSet(step=ii,dset=1)
+      npart       = params[0]
+      ncol        = params[1]
+      ncol_rank   = params[2]
+      ncol_hybrid = params[3]
+      ncol_dem    = params[4]
+      ncol_scalar = params[5]
+      nscal       = ncol_scalar//2
+      pp.append(data.reshape((ncol,npart),order='F'))
+    # Close UCF obeject
+    obj.close()
+    # Convert list of 2D arrays to 3D array
+    pp = np.stack(pp,axis=2)
+    # Create the dictionary
+    col = colmap_from_flags(ncol_rank,ncol_hybrid,ncol_dem,nscal)
+    # Return result
+    return (pp,col)
+    
+def colmap_from_flags(irank,ihybrid,idem,iscal):
+    '''Creates a dictionary which specifies the columns of a particle array.'''
+    col = {}
+    ioffset = 0
+    if irank>0:
+      col['rank'] = ioffset; ioffset+=1
+    if ihybrid>0:
+      col['id'] = ioffset; ioffset+=1
+      col['x']  = ioffset; ioffset+=1
+      col['y']  = ioffset; ioffset+=1
+      col['z']  = ioffset; ioffset+=1
+      col['r']  = ioffset; ioffset+=1
+      col['rho']= ioffset; ioffset+=1
+      col['ax'] = ioffset; ioffset+=1
+      col['ay'] = ioffset; ioffset+=1
+      col['az'] = ioffset; ioffset+=1
+      col['u']  = ioffset; ioffset+=1
+      col['v']  = ioffset; ioffset+=1
+      col['w']  = ioffset; ioffset+=1
+      col['ox'] = ioffset; ioffset+=1
+      col['oy'] = ioffset; ioffset+=1
+      col['oz'] = ioffset; ioffset+=1
+      col['fx'] = ioffset; ioffset+=1
+      col['fy'] = ioffset; ioffset+=1
+      col['fz'] = ioffset; ioffset+=1
+      col['tx'] = ioffset; ioffset+=1
+      col['ty'] = ioffset; ioffset+=1
+      col['tz'] = ioffset; ioffset+=1
+    if idem>0:
+      col['fxc'] = ioffset; ioffset+=1
+      col['fyc'] = ioffset; ioffset+=1
+      col['fzc'] = ioffset; ioffset+=1
+      col['txc'] = ioffset; ioffset+=1
+      col['tyc'] = ioffset; ioffset+=1
+      col['tzc'] = ioffset; ioffset+=1
+    if iscal>0:
+      for ii in range(0,iscal):
+        col['s'+str(ii)] = ioffset; ioffset+=1
+        col['q'+str(ii)] = ioffset; ioffset+=1
+    return col

python/ucftar_downsampler

@@ -0,0 +1,185 @@
+#!/usr/bin/env python3
+import sys
+import io
+import tarfile
+import argparse
+import numpy as np
+import ucf
+
+parser = argparse.ArgumentParser(description='Reads an ucf.tar archive, downsamples it and saves it to a new ucf.tar archive. Can be used as a pipe.')
+parser.add_argument("-i", "--infile", metavar='file',nargs='?', default=None, help="name of the input file [default: stdin]", action="store")
+parser.add_argument("-o", "--outfile", metavar='file',nargs='?', default=None, help="name of the output file [default: stdout]", action="store")
+parser.add_argument("-n", "--nskip", metavar='N',nargs='?', type=int, default=2, help="keep every Nth grid point [default: 2]", action="store")
+parser.add_argument("-sp", "--single-precision", help="output data in single-precision? [default: False]", action="store_true")
+args = parser.parse_args()
+
+nskip    = args.nskip
+file_in  = args.infile
+file_out = args.outfile
+saveSinglePrecision = args.single_precision
+
+if file_in is None:
+  istream = tarfile.open(fileobj=sys.stdin.buffer,mode='r|',bufsize=512*1024**2,ignore_zeros=True)
+else:
+  filehandle_in = open(file_in,'rb')
+  istream = tarfile.open(fileobj=filehandle_in,mode='r')
+
+if file_out is None:
+  ostream = tarfile.open(fileobj=sys.stdout.buffer,mode='w|',bufsize=512*1024**2,pax_headers=tarfile.USTAR_FORMAT)
+else:
+  filehandle_out = open(file_out,'wb')
+  ostream = tarfile.open(fileobj=filehandle_out,mode='w',pax_headers=tarfile.USTAR_FORMAT)
+
+while True:
+    iinfo = istream.next()
+    if iinfo is None:
+        break
+    print(iinfo.name,file=sys.stderr)
+
+    ucfbytes_in = istream.extractfile(iinfo).read()
+    ucfbytes_out = b''
+
+    if iinfo.name=='parameters.asc':
+      ucfbytes_out += ucfbytes_in
+
+    if iinfo.name=='particles.bin':
+      ucfbytes_out += ucfbytes_in
+
+    if iinfo.name=='grid.bin':
+      ucfhandle = ucf.UCF(file=ucfbytes_in,verbosity=False)
+      ucfhandle.copyFileHeaderToBuffer()
+      ucfhandle.copyStepToBuffer(1,step_out=1,recursive=False)
+      for iset in range(0,ucfhandle.NumDataset):
+        (data,params) = ucfhandle.readSet(step=1,dset=iset+1)
+        params = list(params)
+        nx = params[0]
+        ny = params[1]
+        nz = params[2]
+        x = data[0:nx:nskip]
+        y = data[nx:nx+ny:nskip]
+        z = data[nx+ny:nx+ny+nz:nskip]
+        params[0] = len(x)
+        params[1] = len(y)
+        params[2] = len(z)
+        data = np.concatenate((x,y,z))
+        ucfhandle.addDatasetToBuffer(data,params=params,step=1,dset=iset+1)
+      ucfbytes_out += ucfhandle.flushBuffer()
+      ucfhandle.close()
+
+    if iinfo.name=='proc.bin':
+      ucfhandle = ucf.UCF(file=ucfbytes_in,verbosity=False)
+      ucfhandle.copyFileHeaderToBuffer()
+      ucfhandle.copyStepToBuffer(1,step_out=1,recursive=False)
+      for iset in range(0,ucfhandle.NumDataset):
+        (data,params) = ucfhandle.readSet(step=1,dset=iset+1)
+        nxp = params[0]
+        nyp = params[1]
+        nzp = params[2]
+        ibeg = np.copy(data[0:nxp]                            )
+        iend = np.copy(data[nxp:2*nxp]                        )
+        jbeg = np.copy(data[2*nxp:2*nxp+nyp]                  )
+        jend = np.copy(data[2*nxp+nyp:2*nxp+2*nyp]            )
+        kbeg = np.copy(data[2*nxp+2*nyp:2*nxp+2*nyp+nzp]      )
+        kend = np.copy(data[2*nxp+2*nyp+nzp:2*nxp+2*nyp*2*nzp])
+        for ixp in range(0,nxp):
+          ibeg[ixp] = (ibeg[ixp]-1)//nskip+1
+          iend[ixp] = (iend[ixp]-1)//nskip+1
+        for iyp in range(0,nyp):
+          jbeg[iyp] = (jbeg[iyp]-1)//nskip+1
+          jend[iyp] = (jend[iyp]-1)//nskip+1 
+        for izp in range(0,nzp):
+          kbeg[izp] = (kbeg[izp]-1)//nskip+1
+          kend[izp] = (kend[izp]-1)//nskip+1
+        data = np.concatenate((ibeg,iend,jbeg,jend,kbeg,kend))
+        ucfhandle.addDatasetToBuffer(data,params=params,step=1,dset=iset+1)
+      ucfbytes_out += ucfhandle.flushBuffer()
+      ucfhandle.close()
+
+    if 'uvwp.' in iinfo.name:
+      ucfhandle = ucf.UCF(file=ucfbytes_in,verbosity=False)
+      ucfhandle.copyFileHeaderToBuffer()
+      ucfhandle.copyStepToBuffer(1,step_out=1,recursive=False)
+      for iset in range(0,4):
+        (data,params_in) = ucfhandle.readSet(step=1,dset=iset+1)
+        ighost  = params_in[0]
+        (ibeg,jbeg,kbeg) = params_in[1:4]
+        (nxl,nyl,nzl)    = params_in[4:7]
+        (nxg,nyg,nzg)    = params_in[7:10]
+        data    = data.reshape((nxl+2*ighost,nyl+2*ighost,nzl+2*ighost),order='F')
+
+        if nskip>1:
+          islice = [ii-ibeg+ighost for ii in range(ibeg,ibeg+nxl) if (ii-1)%nskip==0] 
+          jslice = [ii-jbeg+ighost for ii in range(jbeg,jbeg+nyl) if (ii-1)%nskip==0] 
+          kslice = [ii-kbeg+ighost for ii in range(kbeg,kbeg+nzl) if (ii-1)%nskip==0]
+
+          data = data[np.ix_(islice,jslice,kslice)]
+          ibeg = (islice[0]+ibeg-ighost)//nskip+1 
+          jbeg = (jslice[0]+jbeg-ighost)//nskip+1 
+          kbeg = (kslice[0]+kbeg-ighost)//nskip+1 
+          (nxl,nyl,nzl) = data.shape
+          nxg = (nxg-1)//nskip+1
+          nyg = (nyg-1)//nskip+1
+          nzg = (nzg-1)//nskip+1
+
+        params_out = list(params_in)
+        if nskip>1:
+          params_out[0]    = 0
+          params_out[1:4]  = (ibeg,jbeg,kbeg)
+          params_out[4:7]  = (nxl,nyl,nzl)
+          params_out[7:10] = (nxg,nyg,nzg)
+
+        if saveSinglePrecision:
+          data = data.astype(np.float32,casting='same_kind')
+        ucfhandle.addDatasetToBuffer(data,params=params_out,step=1,dset=iset+1)
+      ucfbytes_out += ucfhandle.flushBuffer()
+      ucfhandle.close()
+    
+    if 'scal.' in iinfo.name:
+      ucfhandle = ucf.UCF(file=ucfbytes_in,verbosity=False)
+      ucfhandle.copyFileHeaderToBuffer()
+      ucfhandle.copyStepToBuffer(1,step_out=1,recursive=False)
+      for iset in range(0,ucfhandle.NumDataset):
+        (data,params_in) = ucfhandle.readSet(step=1,dset=iset+1)
+        ighost  = params_in[0]
+        (ibeg,jbeg,kbeg) = params_in[1:4]
+        (nxl,nyl,nzl)    = params_in[4:7]
+        (nxg,nyg,nzg)    = params_in[7:10]
+        data    = data.reshape((nxl+2*ighost,nyl+2*ighost,nzl+2*ighost),order='F')
+ 
+        if nskip>1:
+          islice = [ii-ibeg+ighost for ii in range(ibeg,ibeg+nxl) if (ii-1)%nskip==0] 
+          jslice = [ii-jbeg+ighost for ii in range(jbeg,jbeg+nyl) if (ii-1)%nskip==0] 
+          kslice = [ii-kbeg+ighost for ii in range(kbeg,kbeg+nzl) if (ii-1)%nskip==0]
+
+          data = data[np.ix_(islice,jslice,kslice)]
+          ibeg = (islice[0]+ibeg-ighost)//nskip+1 
+          jbeg = (jslice[0]+jbeg-ighost)//nskip+1 
+          kbeg = (kslice[0]+kbeg-ighost)//nskip+1 
+          (nxl,nyl,nzl) = data.shape
+          nxg = (nxg-1)//nskip+1
+          nyg = (nyg-1)//nskip+1
+          nzg = (nzg-1)//nskip+1
+
+        params_out = list(params_in)
+        if nskip>1:
+          params_out[0]    = 0
+          params_out[1:4]  = (ibeg,jbeg,kbeg)
+          params_out[4:7]  = (nxl,nyl,nzl)
+          params_out[7:10] = (nxg,nyg,nzg)
+        
+        if saveSinglePrecision:
+          data = data.astype(np.float32,casting='same_kind')
+        ucfhandle.addDatasetToBuffer(data,params=params_out,step=1,dset=iset+1)
+      ucfbytes_out += ucfhandle.flushBuffer()
+      ucfhandle.close()
+
+    oinfo = tarfile.TarInfo(name=iinfo.name)
+    oinfo.size = len(ucfbytes_out)
+    ostream.addfile(oinfo,fileobj=io.BytesIO(ucfbytes_out))
+
+istream.close()
+ostream.close()
+if file_in is not None:
+  filehandle_in.close()
+if file_out is not None:
+  filehandle_out.close()