mwtkrayer
/
ucftools
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

unified ustar/multifile interface

This commit is contained in:
Michael Stumpf (ifhcluster) 2019-03-26 12:25:08 +00:00
parent ed8a3835f0
commit d8a650884a
11 changed files with 276 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

163
matlab/@ucfmulti/ucfmulti.m Normal file
View File

@ -0,0 +1,163 @@
classdef ucfmulti < handle
% High-level class to treat a directory with UCF files similarly to UCFtar files
properties (Access = public)
Directory % directory name
Seq % sequence number
IOMode % file opened in read-only or read-write mode?
NumberOfSubfiles % number of subfiles
end
properties (Access = private)
% File info
fileID
ioflag
subFile
subFileAlias
subFileSize
scanBuffSize = 2^17; % buffer size of scanner (max. number of files in tar)
end
%% ------------------------------------------------------------------------%%
%% CONSTRUCORS/DESTRUCTORS %%
%% ------------------------------------------------------------------------%%
methods(Access=public)
function obj = ucfmulti()
% obj = ucf()
% Default contructor
obj.resetPublicProperties();
obj.resetPrivateProperties();
end
function delete(obj)
% obj.delete()
% Default destructor
obj.close();
end
end
%% ------------------------------------------------------------------------%%
%% INITIALIZATION METHODS %%
%% ------------------------------------------------------------------------%%
methods(Access=public)
function open(obj,directory,iseq)
% obj.open(file)
% Opens a file in read-only mode
tmp = what(directory); % get absoulte path
obj.Directory = tmp.path;
obj.Seq = iseq;
obj.IOMode = 'read';
obj.ioflag = 'r';
if ~exist(directory,'dir')
error('Unable to open directory: %s',obj.Directory);
end
obj.scanDirectory();
end
end
%% ------------------------------------------------------------------------%%
%% PUBLIC METHODS %%
%% ------------------------------------------------------------------------%%
methods(Access=public)
function close(obj)
% obj.close()
% Closes a file
if obj.fileID<0
return;
end
status = fclose(obj.fileID);
if status<0
warning('Unable to close file (exit code: %d)',status);
return;
end
obj.resetPublicProperties();
obj.resetPrivateProperties();
obj.fileID = -1;
end
function [ptr] = pointer(obj,fname)
% [ptr] = obj.pointer(fname)
% Returns a 'pointer' to the requested file within the tar-ball
% which can be used to read the data without extracting.
% Input
% fname file name of subfile within tar-ball
% Output
% ptr pointer: [fid,first byte,number of bytes]
if obj.fileID>=0
fclose(obj.fileID);
end
idx = obj.findSubfile(fname);
filepath = sprintf('%s/%s',obj.Directory,obj.subFile{idx});
obj.fileID = fopen(filepath,obj.ioflag);
ptr = [obj.fileID,0,obj.subFileSize(idx)];
end
function [fname,fsize] = list(obj)
% [fname,fsize] = obj.list()
% Returns a list of name/size of all subfiles within the tar-ball
% Output
% fname cell array with filenames
% fsize array with file sizes in bytes
fname = obj.subFileAlias;
fsize = obj.subFileSize;
end
function [flag] = isSubfile(obj,fname)
% [flag] = obj.isSubfile(fname)
% Checks if a subfile exists within tar-ball.
% Input
% fname name of subfile
flag = any(ismember(obj.subFileAlias,fname));
end
end
%% ------------------------------------------------------------------------%%
%% PRIVATE METHODS %%
%% ------------------------------------------------------------------------%%
methods(Access=private)
function scanDirectory(obj)
% obj.scanArchive()
% Scans the directory for subfiles and stores meta-data in class variables.
obj.subFile = cell(obj.scanBuffSize,1);
obj.subFileSize = zeros(obj.scanBuffSize,1);
tmp = dir(obj.Directory);
strseq = sprintf('_%04d.',obj.Seq);
ii = 0;
for itmp=1:numel(tmp)
if tmp(itmp).isdir || isempty(strfind(tmp(itmp).name,strseq))
continue;
end
ii = ii+1;
obj.subFile{ii} = tmp(itmp).name;
obj.subFileSize(ii) = tmp(itmp).bytes;
obj.subFileAlias{ii} = strrep(tmp(itmp).name,strseq,'.');
end
% Truncate preallocated arrays
obj.NumberOfSubfiles = ii;
obj.subFile = obj.subFile(1:ii);
obj.subFileAlias= obj.subFileAlias(1:ii);
obj.subFileSize = obj.subFileSize(1:ii);
if obj.NumberOfSubfiles>obj.scanBuffSize
warning('Number of subfiles exceeds scanBuffSize.');
end
end
function [idx] = findSubfile(obj,fname)
% [idx] = obj.findSubfile(fname)
% Get index of requested subfile
% Input
% fname name of subfile
% Output
% idx index of subfile
isReqFile = ismember(obj.subFileAlias,fname);
switch sum(isReqFile)
case 0; error('File not found: %s',fname);
case 1;
otherwise; warning('More than one matching file found.');
end
idx = find(isReqFile);
end
function resetPublicProperties(obj)
obj.Directory = [];
obj.Seq = [];
obj.IOMode = [];
obj.NumberOfSubfiles = [];
end
function resetPrivateProperties(obj)
obj.ioflag = [];
obj.subFile = [];
obj.subFileAlias= [];
obj.subFileSize = [];
end
end
end

11
matlab/read_field_chunk_ucf.m
View File

@ -24,7 +24,7 @@ function [data,ib,jb,kb,nxl,nyl,nzl,ighost] = read_field_chunk_ucf(file,field,va
addParamValue(par,'ghost',1,@isnumeric);
addParamValue(par,'verbosity',0,@isnumeric);
addParamValue(par,'debug',0,@isnumeric);
addParamValue(par,'tarmode',0,@isnumeric);
addParamValue(par,'tarmode',0,@isnumeric); % deprecated
addParamValue(par,'rank',0,@isnumeric);
parse(par,varargin{:});
istep = par.Results.step;
@ -50,11 +50,14 @@ function [data,ib,jb,kb,nxl,nyl,nzl,ighost] = read_field_chunk_ucf(file,field,va
% Open file
obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug);
if par.Results.tarmode
switch class(file)
case 'char'
obj.open(file);
case {'ustar','ucfmulti'}
subfile = sprintf('%s.%05d',fbase,par.Results.rank);
obj.opentar(file.pointer(subfile));
else
obj.open(file);
otherwise
error('Input file type not supported: %s',class(file));
end
% Read raw data

27
matlab/read_grid_ucf.m
View File

@ -1,4 +1,5 @@
function [xu,yu,zu,xv,yv,zv,xw,yw,zw,xp,yp,zp] = read_grid_ucf(file,varargin)
function [xu,yu,zu,xv,yv,zv,xw,yw,zw,xp,yp,zp,...
nxu,nyu,nzu,nxv,nyv,nzv,nxw,nyw,nzw,nxp,nyp,nzp] = read_grid_ucf(file,varargin)
% [xu,yu,zu,xv,yv,zv,xw,yw,zw,xp,yp,zp] = read_grid_ucf(file,varargin)
% Reads staggered grid from ucf file.
% Input
@ -39,12 +40,12 @@ function [xu,yu,zu,xv,yv,zv,xw,yw,zw,xp,yp,zp] = read_grid_ucf(file,varargin)
for iset=1:nset
[data,params] = obj.readSet(1,iset);
params = cast(params,'double');
nx = params(1);
ny = params(2);
nz = params(3);
x{iset} = data(1:nx);
y{iset} = data(nx+1:nx+ny);
z{iset} = data(nx+ny+1:nx+ny+nz);
nx{iset} = params(1);
ny{iset} = params(2);
nz{iset} = params(3);
x{iset} = data(1:nx{iset});
y{iset} = data(nx{iset}+1:nx{iset}+ny{iset});
z{iset} = data(nx{iset}+ny{iset}+1:nx{iset}+ny{iset}+nz{iset});
end
% Close UCF file
@ -55,19 +56,31 @@ function [xu,yu,zu,xv,yv,zv,xw,yw,zw,xp,yp,zp] = read_grid_ucf(file,varargin)
xu = x{iset};
yu = y{iset};
zu = z{iset};
nxu = nx{iset};
nyu = ny{iset};
nzu = nz{iset};
iset = find(strcmp(sets,'v'));
xv = x{iset};
yv = y{iset};
zv = z{iset};
nxv = nx{iset};
nyv = ny{iset};
nzv = nz{iset};
iset = find(strcmp(sets,'w'));
xw = x{iset};
yw = y{iset};
zw = z{iset};
nxw = nx{iset};
nyw = ny{iset};
nzw = nz{iset};
iset = find(strcmp(sets,'p'));
xp = x{iset};
yp = y{iset};
zp = z{iset};
nxp = nx{iset};
nyp = ny{iset};
nzp = nz{iset};
end

17
matlab/read_parameters_ucf.m
View File

@ -12,16 +12,19 @@ function [params] = read_parameters_ucf(file,varargin)
% Parse optional input arguments
par = inputParser;
addParamValue(par,'verbosity',0,@isnumeric);
addParamValue(par,'tarmode',0,@isnumeric);
addParamValue(par,'tarmode',0,@isnumeric); % deprecated
parse(par,varargin{:});
% Open file
obj = ini('verbosity',par.Results.verbosity);
if par.Results.tarmode
ptr = file.pointer('parameters.asc');
obj.opentar(ptr);
else
obj.open(file);
end
switch class(file)
case 'char'
obj.open(file);
case {'ustar','ucfmulti'}
ptr = file.pointer('parameters.asc');
obj.opentar(ptr);
otherwise
error('Input file type not supported: %s',class(file));
end
% Get parsed content
params = obj.getContent();
% Close file

12
matlab/read_particles_ucf.m
View File

@ -23,19 +23,23 @@ function [pp,col,stime] = read_particles_ucf(file,varargin)
addParamValue(par,'format','array',@ischar);
addParamValue(par,'verbosity',0,@isnumeric);
addParamValue(par,'debug',0,@isnumeric);
addParamValue(par,'tarmode',0,@isnumeric);
addParamValue(par,'tarmode',0,@isnumeric); % deprecated (automatically checked now)
parse(par,varargin{:});
istep = par.Results.step;
mlformat = par.Results.format;
% Open file
obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug);
if par.Results.tarmode
switch class(file)
case 'char'
obj.open(file);
case {'ustar','ucfmulti'}
ptr = file.pointer('particles.bin');
obj.opentar(ptr);
else
obj.open(file);
otherwise
error('Input file type not supported: %s',class(file));
end
if ~obj.validateType('particle')
error('read error: no particle data.');
end

16
matlab/read_procgrid_ucf.m
View File

@ -1,11 +1,13 @@
function [ibegu,iendu,jbegu,jendu,kbegu,kendu,...
ibegv,iendv,jbegv,jendv,kbegv,kendv,...
ibegw,iendw,jbegw,jendw,kbegw,kendw,...
ibegp,iendp,jbegp,jendp,kbegp,kendp] = read_procgrid_ucf(file,varargin)
ibegp,iendp,jbegp,jendp,kbegp,kendp,...
nxprocs,nyprocs,nzprocs] = read_procgrid_ucf(file,varargin)
% [ibegu,iendu,jbegu,jendu,kbegu,kendu,...
% ibegv,iendv,jbegv,jendv,kbegv,kendv,...
% ibegw,iendw,jbegw,jendw,kbegw,kendw,...
% ibegp,iendp,jbegp,jendp,kbegp,kendp] = read_procgrid_ucf(file,varargin)
% ibegp,iendp,jbegp,jendp,kbegp,kendp,...
% nxprocs,nyprocs,nzprocs] = read_procgrid_ucf(file,varargin)
% Reads processor grids.
% Input
% file file name (if tar-mode: ustar handle)
@ -16,6 +18,7 @@ function [ibegu,iendu,jbegu,jendu,kbegu,kendu,...
% ibegv,iendv,jbegv,jendv,kbegv,kendv processor grid v
% ibegw,iendw,jbegw,jendw,kbegw,kendw processor grid w
% ibegp,iendp,jbegp,jendp,kbegp,kendp processor grid p
% nxprocs,nyprocs,nzprocs number of processors
% Parse optional input arguments
par = inputParser;
@ -30,11 +33,14 @@ function [ibegu,iendu,jbegu,jendu,kbegu,kendu,...
% Open file
obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug);
if par.Results.tarmode
switch class(file)
case 'char'
obj.open(file);
case {'ustar','ucfmulti'}
ptr = file.pointer('proc.bin');
obj.opentar(ptr);
else
obj.open(file);
otherwise
error('Input file type not supported: %s',class(file));
end
% Read raw data

11
matlab/read_scal_chunk_ucf.m
View File

@ -21,7 +21,7 @@ function [s,ib,jb,kb,nxl,nyl,nzl,ighost] = read_scal_chunk_ucf(file,varargin)
addParamValue(par,'ghost',1,@isnumeric);
addParamValue(par,'verbosity',0,@isnumeric);
addParamValue(par,'debug',0,@isnumeric);
addParamValue(par,'tarmode',0,@isnumeric);
addParamValue(par,'tarmode',0,@isnumeric); % deprecated
addParamValue(par,'rank',0,@isnumeric);
parse(par,varargin{:});
istep = par.Results.step;
@ -29,11 +29,14 @@ function [s,ib,jb,kb,nxl,nyl,nzl,ighost] = read_scal_chunk_ucf(file,varargin)
% Open file
obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug);
if par.Results.tarmode
switch class(file)
case 'char'
obj.open(file);
case {'ustar','ucfmulti'}
subfile = sprintf('scal.%05d',par.Results.rank);
obj.opentar(file.pointer(subfile));
else
obj.open(file);
otherwise
error('Input file type not supported: %s',class(file));
end
if ~obj.validateType('field')

35
matlab/read_scal_complete_ucf.m
View File

@ -15,19 +15,22 @@ function [s] = read_scal_complete_ucf(file,varargin)
addParamValue(par,'step',1,@isnumeric);
addParamValue(par,'verbosity',0,@isnumeric);
addParamValue(par,'debug',0,@isnumeric);
addParamValue(par,'tarmode',0,@isnumeric);
addParamValue(par,'tarmode',0,@isnumeric); % deprecated
parse(par,varargin{:});
istep = par.Results.step;
% Open first file
obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug);
if par.Results.tarmode
fname = sprintf('scal.%05d',0);
obj.opentar(file.pointer(fname));
else
switch class(file)
case 'char'
[fdir,fbase,fext] = fileparts(file);
fname = sprintf('%s/%s.%05d',fdir,fbase,0);
obj.open(fname);
case {'ustar','ucfmulti'}
fname = sprintf('scal.%05d',0);
obj.opentar(file.pointer(fname));
otherwise
error('Input file type not supported: %s',class(file));
end
if ~obj.validateType('field')
@ -67,20 +70,23 @@ function [s] = read_scal_complete_ucf(file,varargin)
% Now loop consecutively through files
ifile = 1;
if par.Results.tarmode
loopCondition = @(x) file.isSubfile(x);
fname = sprintf('scal.%05d',ifile);
else
switch class(file)
case 'char'
loopCondition = @(x) exist(x,'file');
fname = sprintf('%s/%s.%05d',fdir,fbase,ifile);
case {'ustar','ucfmulti'}
loopCondition = @(x) file.isSubfile(x);
fname = sprintf('scal.%05d',ifile);
end
while loopCondition(fname)
% Open file
obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug);
if par.Results.tarmode
obj.opentar(file.pointer(fname));
else
switch class(file)
case 'char'
obj.open(fname);
case {'ustar','ucfmulti'}
obj.opentar(file.pointer(fname));
end
if ~obj.validateType('field')
@ -115,9 +121,10 @@ function [s] = read_scal_complete_ucf(file,varargin)
% Move on to next file
ifile = ifile+1;
if par.Results.tarmode
switch class(file)
case 'char'
fname = sprintf('scal.%05d',ifile);
else
case {'ustar','ucfmulti'}
fname = sprintf('%s/%s.%05d',fdir,fbase,ifile);
end
end

2
matlab/read_uvwp_chunk_legacy.m
View File

@ -49,7 +49,7 @@ function [u,ibu,jbu,kbu,nxul,nyul,nzul,...
nxprocs,nyprocs,nzprocs] = read_procgrid_legacy(fproc);
% Determine processor rank from filename
[idxbeg,idxend] = regexp(fuvwp,'[\d+]$');
[idxbeg,idxend] = regexp(fuvwp,'[\d]+$');
if isempty(idxbeg)
error('Invalid file: does not contain rank. %s',fuvwp);
end

11
matlab/read_uvwp_chunk_ucf.m
View File

@ -27,7 +27,7 @@ function [u,ibu,jbu,kbu,nxul,nyul,nzul,...
addParamValue(par,'ghost',1,@isnumeric);
addParamValue(par,'verbosity',0,@isnumeric);
addParamValue(par,'debug',0,@isnumeric);
addParamValue(par,'tarmode',0,@isnumeric);
addParamValue(par,'tarmode',0,@isnumeric); % deprecated
addParamValue(par,'rank',0,@isnumeric);
parse(par,varargin{:});
istep = par.Results.step;
@ -39,11 +39,14 @@ function [u,ibu,jbu,kbu,nxul,nyul,nzul,...
% Open file
obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug);
if par.Results.tarmode
switch class(file)
case 'char'
obj.open(file);
case {'ustar','ucfmulti'}
subfile = sprintf('uvwp.%05d',par.Results.rank);
obj.opentar(file.pointer(subfile));
else
obj.open(file);
otherwise
error('Input file type not supported: %s',class(file));
end
% Read raw data

36
matlab/read_uvwp_complete_ucf.m
View File

@ -15,9 +15,9 @@ function [u,v,w,p] = read_uvwp_complete_ucf(file,varargin)
addParamValue(par,'step',1,@isnumeric);
addParamValue(par,'verbosity',0,@isnumeric);
addParamValue(par,'debug',0,@isnumeric);
addParamValue(par,'tarmode',0,@isnumeric);
addParamValue(par,'tarmode',0,@isnumeric); % deprecated
parse(par,varargin{:});
istep = par.Results.timestep;
istep = par.Results.step;
% Define sets to be read
sets = {'u','v','w','p'};
@ -25,13 +25,16 @@ function [u,v,w,p] = read_uvwp_complete_ucf(file,varargin)
% Open first file
obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug);
if par.Results.tarmode
fname = sprintf('uvwp.%05d',0);
obj.opentar(file.pointer(fname));
else
switch class(file)
case 'char'
[fdir,fbase,fext] = fileparts(file);
fname = sprintf('%s/%s.%05d',fdir,fbase,0);
obj.open(fname);
case {'ustar','ucfmulti'}
fname = sprintf('uvwp.%05d',0);
obj.opentar(file.pointer(fname));
otherwise
error('Input file type not supported: %s',class(file));
end
if ~obj.validateType('field')
@ -98,20 +101,22 @@ function [u,v,w,p] = read_uvwp_complete_ucf(file,varargin)
% Now loop consecutively through files
ifile = 1;
if par.Results.tarmode
loopCondition = @(x) file.isSubfile(x);
fname = sprintf('uvwp.%05d',ifile);
else
switch class(file)
case 'char'
loopCondition = @(x) exist(x,'file');
fname = sprintf('%s/%s.%05d',fdir,fbase,ifile);
case {'ustar','ucfmulti'}
loopCondition = @(x) file.isSubfile(x);
fname = sprintf('uvwp.%05d',ifile);
end
while loopCondition(fname)
% Open file
obj = ucf('verbosity',par.Results.verbosity,'debug',par.Results.debug);
if par.Results.tarmode
obj.opentar(file.pointer(fname));
else
switch class(file)
case 'char'
obj.open(fname);
case {'ustar','ucfmulti'}
obj.opentar(file.pointer(fname));
end
if ~obj.validateType('field')
@ -168,9 +173,10 @@ function [u,v,w,p] = read_uvwp_complete_ucf(file,varargin)
% Move on to next file
ifile = ifile+1;
if par.Results.tarmode
switch class(file)
case 'char'
fname = sprintf('uvwp.%05d',ifile);
else
case {'ustar','ucfmulti'}
fname = sprintf('%s/%s.%05d',fdir,fbase,ifile);
end
end