mwtkrayer
/
ucftools
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

ucftar downsampler from lsdf

This commit is contained in:
Michael Krayer 2021-06-07 18:46:09 +02:00
parent 414469de1e
commit 34cdb37b7c
1 changed files with 25 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
python/programs/ucftar_downsampler
View File

@ -7,22 +7,32 @@ import numpy as np
import ucf import ucf
parser = argparse.ArgumentParser(description='Reads an ucf.tar archive, downsamples it and saves it to a new ucf.tar archive. Can be used as a pipe.') parser = argparse.ArgumentParser(description='Reads an ucf.tar archive, downsamples it and saves it to a new ucf.tar archive. Can be used as a pipe.')
parser.add_argument("-c", "--chunks", metavar='file',nargs='?', type=int, default=None, help="number of chunks [default: not chunked]", action="store")
parser.add_argument("-i", "--infile", metavar='file',nargs='?', default=None, help="name of the input file [default: stdin]", action="store") parser.add_argument("-i", "--infile", metavar='file',nargs='?', default=None, help="name of the input file [default: stdin]", action="store")
parser.add_argument("-o", "--outfile", metavar='file',nargs='?', default=None, help="name of the output file [default: stdout]", action="store") parser.add_argument("-o", "--outfile", metavar='file',nargs='?', default=None, help="name of the output file [default: stdout]", action="store")
parser.add_argument("-n", "--nskip", metavar='N',nargs='?', type=int, default=2, help="keep every Nth grid point [default: 2]", action="store") parser.add_argument("-n", "--nskip", metavar='N',nargs='?', type=int, default=2, help="keep every Nth grid point [default: 2]", action="store")
parser.add_argument("-sp", "--single-precision", help="output data in single-precision? [default: False]", action="store_true") parser.add_argument("-sp", "--single-precision", help="output data in single-precision? [default: False]", action="store_true")
args = parser.parse_args() args = parser.parse_args()
nchunk = args.chunks
nskip = args.nskip nskip = args.nskip
file_in = args.infile file_in = args.infile
file_out = args.outfile file_out = args.outfile
saveSinglePrecision = args.single_precision saveSinglePrecision = args.single_precision
if file_in is None: if nchunk is None:
istream = tarfile.open(fileobj=sys.stdin.buffer,mode='r|',bufsize=512*1024**2,ignore_zeros=True) if file_in is None:
istream = tarfile.open(fileobj=sys.stdin.buffer,mode='r|',bufsize=512*1024**2,ignore_zeros=True)
else:
filehandle_in = open(file_in,'rb')
istream = tarfile.open(fileobj=filehandle_in,mode='r')
else: else:
filehandle_in = open(file_in,'rb') if file_in is None:
istream = tarfile.open(fileobj=filehandle_in,mode='r') raise ValueError('Chunked mode only works with input file, not stream')
else:
ichunk = 0
filehandle_in = open(file_in+'.{:d}'.format(ichunk),'rb')
istream = tarfile.open(fileobj=filehandle_in,mode='r')
if file_out is None: if file_out is None:
ostream = tarfile.open(fileobj=sys.stdout.buffer,mode='w|',bufsize=512*1024**2,pax_headers=tarfile.USTAR_FORMAT) ostream = tarfile.open(fileobj=sys.stdout.buffer,mode='w|',bufsize=512*1024**2,pax_headers=tarfile.USTAR_FORMAT)
@ -33,6 +43,17 @@ else:
while True: while True:
iinfo = istream.next() iinfo = istream.next()
if iinfo is None: if iinfo is None:
if nchunk is not None:
if ichunk+1<nchunk:
istream.close()
filehandle_in.close()
ichunk = ichunk+1
filehandle_in = open(file_in+'.{:d}'.format(ichunk),'rb')
istream = tarfile.open(fileobj=filehandle_in,mode='r')
iinfo = istream.next()
else:
break
else:
break break
print(iinfo.name,file=sys.stderr) print(iinfo.name,file=sys.stderr)