From 0284e8f95422cac32d1589f256420a503b3fc9c6 Mon Sep 17 00:00:00 2001 From: Michael Krayer Date: Fri, 22 Jan 2021 14:10:00 +0100 Subject: [PATCH] hawk update --- python/ucf/tools/ucf_file_diff | 64 ++++++++++++++++++ python/ucf/tools/ucf_file_diff_single_multi | 73 +++++++++++++++++++++ python/ucf/tools/ucf_file_integrity | 50 ++++++++++++++ python/ucf/ucf.py | 6 +- 4 files changed, 190 insertions(+), 3 deletions(-) create mode 100755 python/ucf/tools/ucf_file_diff create mode 100755 python/ucf/tools/ucf_file_diff_single_multi create mode 100755 python/ucf/tools/ucf_file_integrity diff --git a/python/ucf/tools/ucf_file_diff b/python/ucf/tools/ucf_file_diff new file mode 100755 index 0000000..341daa6 --- /dev/null +++ b/python/ucf/tools/ucf_file_diff @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +import ucf +import argparse +import os +import sys +import traceback +import numpy as np + +parser = argparse.ArgumentParser(description='Tests basic integrity of a UCF file') +parser.add_argument('file1', metavar='file1',help='input file #1') +parser.add_argument('file2', metavar='file2',help='input file #2') +parser.add_argument("--debug", help="enable debug output? [default: False]", action="store_true") +parser.add_argument("--stop", help="stop as soon as an error has been detected? [default: False]", action="store_true") +parser.add_argument("--tolerance", metavar='tol', help="floating point tolerance for comparison [default: 1e-12]", type=float) +parser.add_argument("-v","--verbose", help="enable verbose output? [default: False]", action="store_true") +args = parser.parse_args() + +file_in1 = args.file1 +file_in2 = args.file2 +flag_debug = args.debug +flag_verbose = args.verbose +fptol = args.tolerance +flag_stop = args.stop + +if fptol is None: + fptol = 1e-12 + +if os.path.isfile(file_in1)==False: + print('File not found:',file_in1) + sys.exit(1) +if os.path.isfile(file_in2)==False: + print('File not found:',file_in2) + sys.exit(1) + +obj1 = ucf.UCF(file=file_in1,verbosity=flag_verbose,debug=flag_debug) +obj2 = ucf.UCF(file=file_in2,verbosity=flag_verbose,debug=flag_debug) +if obj1.NumTimestep!=obj2.NumTimestep: + print("Number of time steps differ: {:d}, {:d}".format(obj1.NumTimestep,obj2.NumTimestep)) + sys.exit(2) + +errfound=False +for istep in range(0,obj1.NumTimestep): + for iset in range(0,obj1._UCF__numSetPerStep[istep]): + (data1,param1) = obj1.readSet(step=istep+1,dset=iset+1) + (data2,param2) = obj2.readSet(step=istep+1,dset=iset+1) + if not param1==param2: + print('Step #{:d}, set #{:d}: parameter mismatch. {}, {}'.format(istep,iset,param1,param2)) + if flag_stop: + sys.exit(3) + else: + errfound=True + if not np.allclose(data1,data2,rtol=fptol): + data_diff = np.abs(data1-data2) + idx = np.argmax(data_diff) + vald = data_diff.flatten()[idx] + val1 = data1.flatten()[idx] + val2 = data2.flatten()[idx] + print('Step #{:d}, set #{:d}: difference in FP data above tolerance. idx={}, diff={}, val1={}, val2={}'.format(istep,iset,idx,vald,val1,val2)) + if flag_stop: + sys.exit(4) + else: + errfound=True +if errfound: + sys.exit(5) diff --git a/python/ucf/tools/ucf_file_diff_single_multi b/python/ucf/tools/ucf_file_diff_single_multi new file mode 100755 index 0000000..4c39d3c --- /dev/null +++ b/python/ucf/tools/ucf_file_diff_single_multi @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +import ucf +import argparse +import os +import sys +import traceback +import numpy as np + +parser = argparse.ArgumentParser(description='Verifies that content of single UCF file is equivalent to multiple UCF files') +parser.add_argument('file1', metavar='file_single',help='input file #1 (single file)') +parser.add_argument('file2', metavar='filebase_multi',help='input file #2 (multiple files)') +parser.add_argument("--debug", help="enable debug output? [default: False]", action="store_true") +parser.add_argument("--stop", help="stop as soon as an error has been detected? [default: False]", action="store_true") +parser.add_argument("--tolerance", metavar='tol', help="floating point tolerance for comparison [default: 1e-12]", type=float) +parser.add_argument("-v","--verbose", help="enable verbose output? [default: False]", action="store_true") +args = parser.parse_args() + +file_single = args.file1 +fiba_multi = args.file2 +flag_debug = args.debug +flag_verbose = args.verbose +fptol = args.tolerance +flag_stop = args.stop + +if fptol is None: + fptol = 1e-12 + +if os.path.isfile(file_single)==False: + print('File not found:',os.path.isfile(file_single)) + sys.exit(1) + +objs = ucf.UCF(file=file_single,verbosity=flag_verbose,debug=flag_debug) +#if obj1.NumTimestep!=obj2.NumTimestep: + +errfound=False +for istep in range(0,objs.NumTimestep): + print('Comparing rank {:5d}'.format(istep)) + file_multi = '{}{:05d}'.format(fiba_multi,istep) + if os.path.isfile(file_multi)==False: + print('File not found:',os.path.isfile(file_multi)) + if flag_stop: + sys.exit(1) + else: + errfound=True + objm = ucf.UCF(file=file_multi,verbosity=flag_verbose,debug=flag_debug) + if objs._UCF__numSetPerStep[istep]!=objm._UCF__numSetPerStep[0]: + print("Number of datasets differs: {:d}, {:d}".format(objs._UCF__numSetPerStep[istep],objm._UCF__numSetPerStep[0])) + if flag_stop: + sys.exit(2) + else: + errfound=True + for iset in range(0,objs._UCF__numSetPerStep[istep]): + (data1,param1) = objs.readSet(step=istep+1,dset=iset+1) + (data2,param2) = objm.readSet(step=1,dset=iset+1) + if not param1==param2: + print('Step #{:d}, set #{:d}: parameter mismatch. {}, {}'.format(istep,iset,param1,param2)) + if flag_stop: + sys.exit(3) + else: + errfound=True + if not np.allclose(data1,data2,rtol=fptol): + data_diff = np.abs(data1-data2) + idx = np.argmax(data_diff) + vald = data_diff.flatten()[idx] + val1 = data1.flatten()[idx] + val2 = data2.flatten()[idx] + print('Step #{:d}, set #{:d}: difference in FP data above tolerance. idx={}, diff={}, val1={}, val2={}'.format(istep,iset,idx,vald,val1,val2)) + if flag_stop: + sys.exit(4) + else: + errfound=True +if errfound: + sys.exit(5) diff --git a/python/ucf/tools/ucf_file_integrity b/python/ucf/tools/ucf_file_integrity new file mode 100755 index 0000000..d97c929 --- /dev/null +++ b/python/ucf/tools/ucf_file_integrity @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +import ucf +import argparse +import os +import sys +import traceback + +parser = argparse.ArgumentParser(description='Tests basic integrity of a UCF file') +parser.add_argument('infile', metavar='file', nargs='+',help='input file') +parser.add_argument("--debug", help="enable debug output? [default: False]", action="store_true") +parser.add_argument("--print-error", help="print possible error messages? [default: False]", action="store_true") +parser.add_argument("--stop", help="stop as soon as a corrupted file has been found? [default: False]", action="store_true") +parser.add_argument("-v","--verbose", help="enable verbose output? [default: False]", action="store_true") +args = parser.parse_args() + +file_in = args.infile +flag_debug = args.debug +flag_verbose = args.verbose +flag_error = args.print_error +flag_stop = args.stop + +numerror=0 +for file in file_in: + print(file+": ",end="") + if os.path.isfile(file)==False: + print('not found.') + numerror+=1 + continue + try: + obj = ucf.UCF(file=file,verbosity=flag_verbose,debug=flag_debug) + for it in range(0,obj.NumTimestep): + for iset in range(0,obj._UCF__numSetPerStep[it]): + obj._UCF__findSet(it+1,iset+1) + obj._UCF__readHeaderSet() + obj.close() + print('successful ({:d} time steps, {:d} data sets)'.format(obj.NumTimestep,obj.NumDataset)) + except Exception: + print('corrupt') + if flag_error: + print(traceback.print_exc()) + if flag_stop: + sys.exit(1) + numerror+=1 + +if numerror==0: + print('Test completed with no errors.') + sys.exit(0) +else: + print("Test completed with {:d} error(s).".format(numerror)) + sys.exit(1) diff --git a/python/ucf/ucf.py b/python/ucf/ucf.py index 70eeeac..d94c13c 100644 --- a/python/ucf/ucf.py +++ b/python/ucf/ucf.py @@ -63,7 +63,7 @@ class UCF: # Scan through file to get the basic structure (steps/sets) self.__timeStep = np.zeros(self.__scanBuffSize,dtype=np.float64) - self.__posStep = np.zeros(self.__scanBuffSize,dtype=np.int32) + self.__posStep = np.zeros(self.__scanBuffSize,dtype=np.int64) self.__numSetPerStep = np.zeros(self.__scanBuffSize,dtype=np.int32) istep = 0; while self.__fileID.tell()