From d8a650884a8c8fbc7da241ca7ef75377c76a7ffd Mon Sep 17 00:00:00 2001 From: "Michael Stumpf (ifhcluster)" Date: Tue, 26 Mar 2019 12:25:08 +0000 Subject: [PATCH] unified ustar/multifile interface --- matlab/@ucfmulti/ucfmulti.m | 163 ++++++++++++++++++++++++++++++++ matlab/read_field_chunk_ucf.m | 11 ++- matlab/read_grid_ucf.m | 27 ++++-- matlab/read_parameters_ucf.m | 17 ++-- matlab/read_particles_ucf.m | 12 ++- matlab/read_procgrid_ucf.m | 16 +++- matlab/read_scal_chunk_ucf.m | 11 ++- matlab/read_scal_complete_ucf.m | 35 ++++--- matlab/read_uvwp_chunk_legacy.m | 2 +- matlab/read_uvwp_chunk_ucf.m | 11 ++- matlab/read_uvwp_complete_ucf.m | 36 ++++--- 11 files changed, 276 insertions(+), 65 deletions(-) create mode 100644 matlab/@ucfmulti/ucfmulti.m diff --git a/matlab/@ucfmulti/ucfmulti.m b/matlab/@ucfmulti/ucfmulti.m new file mode 100644 index 0000000..c1018ec --- /dev/null +++ b/matlab/@ucfmulti/ucfmulti.m @@ -0,0 +1,163 @@ +classdef ucfmulti < handle + % High-level class to treat a directory with UCF files similarly to UCFtar files + properties (Access = public) + Directory % directory name + Seq % sequence number + IOMode % file opened in read-only or read-write mode? + NumberOfSubfiles % number of subfiles + end + properties (Access = private) + % File info + fileID + ioflag + subFile + subFileAlias + subFileSize + scanBuffSize = 2^17; % buffer size of scanner (max. number of files in tar) + end + %% ------------------------------------------------------------------------%% + %% CONSTRUCORS/DESTRUCTORS %% + %% ------------------------------------------------------------------------%% + methods(Access=public) + function obj = ucfmulti() + % obj = ucf() + % Default contructor + obj.resetPublicProperties(); + obj.resetPrivateProperties(); + end + function delete(obj) + % obj.delete() + % Default destructor + obj.close(); + end + end + %% ------------------------------------------------------------------------%% + %% INITIALIZATION METHODS %% + %% ------------------------------------------------------------------------%% + methods(Access=public) + function open(obj,directory,iseq) + % obj.open(file) + % Opens a file in read-only mode + tmp = what(directory); % get absoulte path + obj.Directory = tmp.path; + obj.Seq = iseq; + obj.IOMode = 'read'; + obj.ioflag = 'r'; + if ~exist(directory,'dir') + error('Unable to open directory: %s',obj.Directory); + end + obj.scanDirectory(); + end + end + %% ------------------------------------------------------------------------%% + %% PUBLIC METHODS %% + %% ------------------------------------------------------------------------%% + methods(Access=public) + function close(obj) + % obj.close() + % Closes a file + if obj.fileID<0 + return; + end + status = fclose(obj.fileID); + if status<0 + warning('Unable to close file (exit code: %d)',status); + return; + end + obj.resetPublicProperties(); + obj.resetPrivateProperties(); + obj.fileID = -1; + end + function [ptr] = pointer(obj,fname) + % [ptr] = obj.pointer(fname) + % Returns a 'pointer' to the requested file within the tar-ball + % which can be used to read the data without extracting. + % Input + % fname file name of subfile within tar-ball + % Output + % ptr pointer: [fid,first byte,number of bytes] + if obj.fileID>=0 + fclose(obj.fileID); + end + idx = obj.findSubfile(fname); + filepath = sprintf('%s/%s',obj.Directory,obj.subFile{idx}); + obj.fileID = fopen(filepath,obj.ioflag); + ptr = [obj.fileID,0,obj.subFileSize(idx)]; + end + function [fname,fsize] = list(obj) + % [fname,fsize] = obj.list() + % Returns a list of name/size of all subfiles within the tar-ball + % Output + % fname cell array with filenames + % fsize array with file sizes in bytes + fname = obj.subFileAlias; + fsize = obj.subFileSize; + end + function [flag] = isSubfile(obj,fname) + % [flag] = obj.isSubfile(fname) + % Checks if a subfile exists within tar-ball. + % Input + % fname name of subfile + flag = any(ismember(obj.subFileAlias,fname)); + end + end + %% ------------------------------------------------------------------------%% + %% PRIVATE METHODS %% + %% ------------------------------------------------------------------------%% + methods(Access=private) + function scanDirectory(obj) + % obj.scanArchive() + % Scans the directory for subfiles and stores meta-data in class variables. + obj.subFile = cell(obj.scanBuffSize,1); + obj.subFileSize = zeros(obj.scanBuffSize,1); + + tmp = dir(obj.Directory); + strseq = sprintf('_%04d.',obj.Seq); + ii = 0; + for itmp=1:numel(tmp) + if tmp(itmp).isdir || isempty(strfind(tmp(itmp).name,strseq)) + continue; + end + ii = ii+1; + obj.subFile{ii} = tmp(itmp).name; + obj.subFileSize(ii) = tmp(itmp).bytes; + obj.subFileAlias{ii} = strrep(tmp(itmp).name,strseq,'.'); + end + % Truncate preallocated arrays + obj.NumberOfSubfiles = ii; + obj.subFile = obj.subFile(1:ii); + obj.subFileAlias= obj.subFileAlias(1:ii); + obj.subFileSize = obj.subFileSize(1:ii); + if obj.NumberOfSubfiles>obj.scanBuffSize + warning('Number of subfiles exceeds scanBuffSize.'); + end + end + function [idx] = findSubfile(obj,fname) + % [idx] = obj.findSubfile(fname) + % Get index of requested subfile + % Input + % fname name of subfile + % Output + % idx index of subfile + isReqFile = ismember(obj.subFileAlias,fname); + switch sum(isReqFile) + case 0; error('File not found: %s',fname); + case 1; + otherwise; warning('More than one matching file found.'); + end + idx = find(isReqFile); + end + function resetPublicProperties(obj) + obj.Directory = []; + obj.Seq = []; + obj.IOMode = []; + obj.NumberOfSubfiles = []; + end + function resetPrivateProperties(obj) + obj.ioflag = []; + obj.subFile = []; + obj.subFileAlias= []; + obj.subFileSize = []; + end + end +end diff --git a/matlab/read_field_chunk_ucf.m b/matlab/read_field_chunk_ucf.m index 00dc264..5d128ba 100644 --- a/matlab/read_field_chunk_ucf.m +++ b/matlab/read_field_chunk_ucf.m @@ -24,7 +24,7 @@ function [data,ib,jb,kb,nxl,nyl,nzl,ighost] = read_field_chunk_ucf(file,field,va addParamValue(par,'ghost',1,@isnumeric); addParamValue(par,'verbosity',0,@isnumeric); addParamValue(par,'debug',0,@isnumeric); - addParamValue(par,'tarmode',0,@isnumeric); + addParamValue(par,'tarmode',0,@isnumeric); % deprecated addParamValue(par,'rank',0,@isnumeric); parse(par,varargin{:}); istep = par.Results.step; @@ -50,11 +50,14 @@ function [data,ib,jb,kb,nxl,nyl,nzl,ighost] = read_field_chunk_ucf(file,field,va % Open file obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug); - if par.Results.tarmode + switch class(file) + case 'char' + obj.open(file); + case {'ustar','ucfmulti'} subfile = sprintf('%s.%05d',fbase,par.Results.rank); obj.opentar(file.pointer(subfile)); - else - obj.open(file); + otherwise + error('Input file type not supported: %s',class(file)); end % Read raw data diff --git a/matlab/read_grid_ucf.m b/matlab/read_grid_ucf.m index 442f6bc..15f2232 100644 --- a/matlab/read_grid_ucf.m +++ b/matlab/read_grid_ucf.m @@ -1,4 +1,5 @@ -function [xu,yu,zu,xv,yv,zv,xw,yw,zw,xp,yp,zp] = read_grid_ucf(file,varargin) +function [xu,yu,zu,xv,yv,zv,xw,yw,zw,xp,yp,zp,... + nxu,nyu,nzu,nxv,nyv,nzv,nxw,nyw,nzw,nxp,nyp,nzp] = read_grid_ucf(file,varargin) % [xu,yu,zu,xv,yv,zv,xw,yw,zw,xp,yp,zp] = read_grid_ucf(file,varargin) % Reads staggered grid from ucf file. % Input @@ -39,12 +40,12 @@ function [xu,yu,zu,xv,yv,zv,xw,yw,zw,xp,yp,zp] = read_grid_ucf(file,varargin) for iset=1:nset [data,params] = obj.readSet(1,iset); params = cast(params,'double'); - nx = params(1); - ny = params(2); - nz = params(3); - x{iset} = data(1:nx); - y{iset} = data(nx+1:nx+ny); - z{iset} = data(nx+ny+1:nx+ny+nz); + nx{iset} = params(1); + ny{iset} = params(2); + nz{iset} = params(3); + x{iset} = data(1:nx{iset}); + y{iset} = data(nx{iset}+1:nx{iset}+ny{iset}); + z{iset} = data(nx{iset}+ny{iset}+1:nx{iset}+ny{iset}+nz{iset}); end % Close UCF file @@ -55,19 +56,31 @@ function [xu,yu,zu,xv,yv,zv,xw,yw,zw,xp,yp,zp] = read_grid_ucf(file,varargin) xu = x{iset}; yu = y{iset}; zu = z{iset}; + nxu = nx{iset}; + nyu = ny{iset}; + nzu = nz{iset}; iset = find(strcmp(sets,'v')); xv = x{iset}; yv = y{iset}; zv = z{iset}; + nxv = nx{iset}; + nyv = ny{iset}; + nzv = nz{iset}; iset = find(strcmp(sets,'w')); xw = x{iset}; yw = y{iset}; zw = z{iset}; + nxw = nx{iset}; + nyw = ny{iset}; + nzw = nz{iset}; iset = find(strcmp(sets,'p')); xp = x{iset}; yp = y{iset}; zp = z{iset}; + nxp = nx{iset}; + nyp = ny{iset}; + nzp = nz{iset}; end diff --git a/matlab/read_parameters_ucf.m b/matlab/read_parameters_ucf.m index 514f720..ef8c4af 100644 --- a/matlab/read_parameters_ucf.m +++ b/matlab/read_parameters_ucf.m @@ -12,16 +12,19 @@ function [params] = read_parameters_ucf(file,varargin) % Parse optional input arguments par = inputParser; addParamValue(par,'verbosity',0,@isnumeric); - addParamValue(par,'tarmode',0,@isnumeric); + addParamValue(par,'tarmode',0,@isnumeric); % deprecated parse(par,varargin{:}); % Open file obj = ini('verbosity',par.Results.verbosity); - if par.Results.tarmode - ptr = file.pointer('parameters.asc'); - obj.opentar(ptr); - else - obj.open(file); - end + switch class(file) + case 'char' + obj.open(file); + case {'ustar','ucfmulti'} + ptr = file.pointer('parameters.asc'); + obj.opentar(ptr); + otherwise + error('Input file type not supported: %s',class(file)); + end % Get parsed content params = obj.getContent(); % Close file diff --git a/matlab/read_particles_ucf.m b/matlab/read_particles_ucf.m index 24acc41..6cdc398 100644 --- a/matlab/read_particles_ucf.m +++ b/matlab/read_particles_ucf.m @@ -23,19 +23,23 @@ function [pp,col,stime] = read_particles_ucf(file,varargin) addParamValue(par,'format','array',@ischar); addParamValue(par,'verbosity',0,@isnumeric); addParamValue(par,'debug',0,@isnumeric); - addParamValue(par,'tarmode',0,@isnumeric); + addParamValue(par,'tarmode',0,@isnumeric); % deprecated (automatically checked now) parse(par,varargin{:}); istep = par.Results.step; mlformat = par.Results.format; % Open file obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug); - if par.Results.tarmode + switch class(file) + case 'char' + obj.open(file); + case {'ustar','ucfmulti'} ptr = file.pointer('particles.bin'); obj.opentar(ptr); - else - obj.open(file); + otherwise + error('Input file type not supported: %s',class(file)); end + if ~obj.validateType('particle') error('read error: no particle data.'); end diff --git a/matlab/read_procgrid_ucf.m b/matlab/read_procgrid_ucf.m index b7467c8..5765089 100644 --- a/matlab/read_procgrid_ucf.m +++ b/matlab/read_procgrid_ucf.m @@ -1,11 +1,13 @@ function [ibegu,iendu,jbegu,jendu,kbegu,kendu,... ibegv,iendv,jbegv,jendv,kbegv,kendv,... ibegw,iendw,jbegw,jendw,kbegw,kendw,... - ibegp,iendp,jbegp,jendp,kbegp,kendp] = read_procgrid_ucf(file,varargin) + ibegp,iendp,jbegp,jendp,kbegp,kendp,... + nxprocs,nyprocs,nzprocs] = read_procgrid_ucf(file,varargin) % [ibegu,iendu,jbegu,jendu,kbegu,kendu,... % ibegv,iendv,jbegv,jendv,kbegv,kendv,... % ibegw,iendw,jbegw,jendw,kbegw,kendw,... - % ibegp,iendp,jbegp,jendp,kbegp,kendp] = read_procgrid_ucf(file,varargin) + % ibegp,iendp,jbegp,jendp,kbegp,kendp,... + % nxprocs,nyprocs,nzprocs] = read_procgrid_ucf(file,varargin) % Reads processor grids. % Input % file file name (if tar-mode: ustar handle) @@ -16,6 +18,7 @@ function [ibegu,iendu,jbegu,jendu,kbegu,kendu,... % ibegv,iendv,jbegv,jendv,kbegv,kendv processor grid v % ibegw,iendw,jbegw,jendw,kbegw,kendw processor grid w % ibegp,iendp,jbegp,jendp,kbegp,kendp processor grid p + % nxprocs,nyprocs,nzprocs number of processors % Parse optional input arguments par = inputParser; @@ -30,11 +33,14 @@ function [ibegu,iendu,jbegu,jendu,kbegu,kendu,... % Open file obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug); - if par.Results.tarmode + switch class(file) + case 'char' + obj.open(file); + case {'ustar','ucfmulti'} ptr = file.pointer('proc.bin'); obj.opentar(ptr); - else - obj.open(file); + otherwise + error('Input file type not supported: %s',class(file)); end % Read raw data diff --git a/matlab/read_scal_chunk_ucf.m b/matlab/read_scal_chunk_ucf.m index d3320bd..1200ede 100644 --- a/matlab/read_scal_chunk_ucf.m +++ b/matlab/read_scal_chunk_ucf.m @@ -21,7 +21,7 @@ function [s,ib,jb,kb,nxl,nyl,nzl,ighost] = read_scal_chunk_ucf(file,varargin) addParamValue(par,'ghost',1,@isnumeric); addParamValue(par,'verbosity',0,@isnumeric); addParamValue(par,'debug',0,@isnumeric); - addParamValue(par,'tarmode',0,@isnumeric); + addParamValue(par,'tarmode',0,@isnumeric); % deprecated addParamValue(par,'rank',0,@isnumeric); parse(par,varargin{:}); istep = par.Results.step; @@ -29,11 +29,14 @@ function [s,ib,jb,kb,nxl,nyl,nzl,ighost] = read_scal_chunk_ucf(file,varargin) % Open file obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug); - if par.Results.tarmode + switch class(file) + case 'char' + obj.open(file); + case {'ustar','ucfmulti'} subfile = sprintf('scal.%05d',par.Results.rank); obj.opentar(file.pointer(subfile)); - else - obj.open(file); + otherwise + error('Input file type not supported: %s',class(file)); end if ~obj.validateType('field') diff --git a/matlab/read_scal_complete_ucf.m b/matlab/read_scal_complete_ucf.m index 6fa44c7..7c5abe5 100644 --- a/matlab/read_scal_complete_ucf.m +++ b/matlab/read_scal_complete_ucf.m @@ -15,19 +15,22 @@ function [s] = read_scal_complete_ucf(file,varargin) addParamValue(par,'step',1,@isnumeric); addParamValue(par,'verbosity',0,@isnumeric); addParamValue(par,'debug',0,@isnumeric); - addParamValue(par,'tarmode',0,@isnumeric); + addParamValue(par,'tarmode',0,@isnumeric); % deprecated parse(par,varargin{:}); istep = par.Results.step; % Open first file obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug); - if par.Results.tarmode - fname = sprintf('scal.%05d',0); - obj.opentar(file.pointer(fname)); - else + switch class(file) + case 'char' [fdir,fbase,fext] = fileparts(file); fname = sprintf('%s/%s.%05d',fdir,fbase,0); obj.open(fname); + case {'ustar','ucfmulti'} + fname = sprintf('scal.%05d',0); + obj.opentar(file.pointer(fname)); + otherwise + error('Input file type not supported: %s',class(file)); end if ~obj.validateType('field') @@ -67,20 +70,23 @@ function [s] = read_scal_complete_ucf(file,varargin) % Now loop consecutively through files ifile = 1; - if par.Results.tarmode - loopCondition = @(x) file.isSubfile(x); - fname = sprintf('scal.%05d',ifile); - else + switch class(file) + case 'char' loopCondition = @(x) exist(x,'file'); fname = sprintf('%s/%s.%05d',fdir,fbase,ifile); + case {'ustar','ucfmulti'} + loopCondition = @(x) file.isSubfile(x); + fname = sprintf('scal.%05d',ifile); end + while loopCondition(fname) % Open file obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug); - if par.Results.tarmode - obj.opentar(file.pointer(fname)); - else + switch class(file) + case 'char' obj.open(fname); + case {'ustar','ucfmulti'} + obj.opentar(file.pointer(fname)); end if ~obj.validateType('field') @@ -115,9 +121,10 @@ function [s] = read_scal_complete_ucf(file,varargin) % Move on to next file ifile = ifile+1; - if par.Results.tarmode + switch class(file) + case 'char' fname = sprintf('scal.%05d',ifile); - else + case {'ustar','ucfmulti'} fname = sprintf('%s/%s.%05d',fdir,fbase,ifile); end end diff --git a/matlab/read_uvwp_chunk_legacy.m b/matlab/read_uvwp_chunk_legacy.m index a90fbfa..3aaaa1b 100644 --- a/matlab/read_uvwp_chunk_legacy.m +++ b/matlab/read_uvwp_chunk_legacy.m @@ -49,7 +49,7 @@ function [u,ibu,jbu,kbu,nxul,nyul,nzul,... nxprocs,nyprocs,nzprocs] = read_procgrid_legacy(fproc); % Determine processor rank from filename - [idxbeg,idxend] = regexp(fuvwp,'[\d+]$'); + [idxbeg,idxend] = regexp(fuvwp,'[\d]+$'); if isempty(idxbeg) error('Invalid file: does not contain rank. %s',fuvwp); end diff --git a/matlab/read_uvwp_chunk_ucf.m b/matlab/read_uvwp_chunk_ucf.m index f9a3370..6651063 100644 --- a/matlab/read_uvwp_chunk_ucf.m +++ b/matlab/read_uvwp_chunk_ucf.m @@ -27,7 +27,7 @@ function [u,ibu,jbu,kbu,nxul,nyul,nzul,... addParamValue(par,'ghost',1,@isnumeric); addParamValue(par,'verbosity',0,@isnumeric); addParamValue(par,'debug',0,@isnumeric); - addParamValue(par,'tarmode',0,@isnumeric); + addParamValue(par,'tarmode',0,@isnumeric); % deprecated addParamValue(par,'rank',0,@isnumeric); parse(par,varargin{:}); istep = par.Results.step; @@ -39,11 +39,14 @@ function [u,ibu,jbu,kbu,nxul,nyul,nzul,... % Open file obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug); - if par.Results.tarmode + switch class(file) + case 'char' + obj.open(file); + case {'ustar','ucfmulti'} subfile = sprintf('uvwp.%05d',par.Results.rank); obj.opentar(file.pointer(subfile)); - else - obj.open(file); + otherwise + error('Input file type not supported: %s',class(file)); end % Read raw data diff --git a/matlab/read_uvwp_complete_ucf.m b/matlab/read_uvwp_complete_ucf.m index 970119d..1fab402 100644 --- a/matlab/read_uvwp_complete_ucf.m +++ b/matlab/read_uvwp_complete_ucf.m @@ -15,9 +15,9 @@ function [u,v,w,p] = read_uvwp_complete_ucf(file,varargin) addParamValue(par,'step',1,@isnumeric); addParamValue(par,'verbosity',0,@isnumeric); addParamValue(par,'debug',0,@isnumeric); - addParamValue(par,'tarmode',0,@isnumeric); + addParamValue(par,'tarmode',0,@isnumeric); % deprecated parse(par,varargin{:}); - istep = par.Results.timestep; + istep = par.Results.step; % Define sets to be read sets = {'u','v','w','p'}; @@ -25,13 +25,16 @@ function [u,v,w,p] = read_uvwp_complete_ucf(file,varargin) % Open first file obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug); - if par.Results.tarmode - fname = sprintf('uvwp.%05d',0); - obj.opentar(file.pointer(fname)); - else + switch class(file) + case 'char' [fdir,fbase,fext] = fileparts(file); fname = sprintf('%s/%s.%05d',fdir,fbase,0); obj.open(fname); + case {'ustar','ucfmulti'} + fname = sprintf('uvwp.%05d',0); + obj.opentar(file.pointer(fname)); + otherwise + error('Input file type not supported: %s',class(file)); end if ~obj.validateType('field') @@ -98,20 +101,22 @@ function [u,v,w,p] = read_uvwp_complete_ucf(file,varargin) % Now loop consecutively through files ifile = 1; - if par.Results.tarmode - loopCondition = @(x) file.isSubfile(x); - fname = sprintf('uvwp.%05d',ifile); - else + switch class(file) + case 'char' loopCondition = @(x) exist(x,'file'); fname = sprintf('%s/%s.%05d',fdir,fbase,ifile); + case {'ustar','ucfmulti'} + loopCondition = @(x) file.isSubfile(x); + fname = sprintf('uvwp.%05d',ifile); end while loopCondition(fname) % Open file obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug); - if par.Results.tarmode - obj.opentar(file.pointer(fname)); - else + switch class(file) + case 'char' obj.open(fname); + case {'ustar','ucfmulti'} + obj.opentar(file.pointer(fname)); end if ~obj.validateType('field') @@ -168,9 +173,10 @@ function [u,v,w,p] = read_uvwp_complete_ucf(file,varargin) % Move on to next file ifile = ifile+1; - if par.Results.tarmode + switch class(file) + case 'char' fname = sprintf('uvwp.%05d',ifile); - else + case {'ustar','ucfmulti'} fname = sprintf('%s/%s.%05d',fdir,fbase,ifile); end end