21 #define FILENAME_LENGTH 1024 23 static void usage(
char *name);
25 int main(
int argc,
char **argv)
28 char infilename[FILENAME_LENGTH];
29 char outfilename[FILENAME_LENGTH];
30 char prefix[FILENAME_LENGTH];
31 char fnmask[FILENAME_LENGTH];
32 char fnkmask[FILENAME_LENGTH];
36 sion_int64 chunksize = 0;
37 sion_int64 left, bread, bsumread, bwrote;
42 int useglobalranks = 0;
45 int sid, ntasks, nfiles, maxblocks;
47 sion_int64 globalskip;
48 sion_int64 start_of_varheader;
49 sion_int64 *sion_localsizes;
50 sion_int64 *sion_globalranks;
51 sion_int64 *sion_blockcount;
52 sion_int64 *sion_blocksizes;
53 sion_int64 *sionsplit_sum_bytes_per_task;
54 sion_int64 sionsplit_sum_bytes;
55 sion_int64 sionsplit_filesize;
67 if (argv[i][0] ==
'-') {
70 digits = atoi(argv[++i]);
79 fprintf(stderr,
"SIONlib utility %s (Version %d.%dp%d, fileformat version %d)\n", argv[0],
80 SION_MAIN_VERSION,SION_SUB_VERSION,
81 SION_VERSION_PATCHLEVEL,SION_FILEFORMAT_VERSION);
93 strcpy(infilename, argv[argc - 2]);
94 strcpy(prefix, argv[argc - 1]);
96 printf(
"sionsplit: filename: %-30s\n", infilename);
97 printf(
"sionsplit: prefix: %-30s\n", prefix);
99 sid =
sion_open(infilename,
"rb,posix", &ntasks, &nfiles, NULL, &fsblksize, NULL, NULL);
103 return(
_sion_errorprint(SION_NOT_SUCCESS,_SION_ERROR_RETURN,
"invalid sion_filedesc, aborting %d ...\n", sid));
106 printf(
"sionsplit: sid: %d\n", sid);
107 printf(
"sionsplit: filename: %-30s\n", infilename);
108 printf(
"sionsplit: number of tasks: %d\n", ntasks);
109 printf(
"sionsplit: number of files: %d\n", nfiles);
110 printf(
"sionsplit: number of digits: %d\n", digits);
111 printf(
"sionsplit: use global ranks: %d\n", useglobalranks);
115 printf(
"sionsplit: fsblksize: %lu bytes (%6.2f MB)\n", (
unsigned long) fsblksize, fsblksize / 1024.0 / 1024.0);
117 sion_get_locations(sid, &ntasks, &maxblocks, &globalskip, &start_of_varheader, &sion_localsizes, &sion_globalranks, &sion_blockcount,
120 printf(
"sionsplit: max number of chunks: %d\n", maxblocks);
123 sionsplit_sum_bytes_per_task = (sion_int64 *) malloc(ntasks *
sizeof(sion_int64));
124 for (rank = 0; rank < ntasks; rank++)
125 sionsplit_sum_bytes_per_task[rank] = 0;
126 for (rank = 0; rank < ntasks; rank++) {
127 for (blknum = 0; blknum < maxblocks; blknum++) {
128 sionsplit_sum_bytes_per_task[rank] += sion_blocksizes[ntasks * blknum + rank];
132 sionsplit_sum_bytes = 0;
133 for (rank = 0; rank < ntasks; rank++)
134 sionsplit_sum_bytes += sionsplit_sum_bytes_per_task[rank];
136 printf(
"sionsplit: datasize in file (aggr.): %lld bytes (%6.2f MB)\n", sionsplit_sum_bytes, sionsplit_sum_bytes / 1024.0 / 1024.0);
138 sionsplit_filesize = start_of_varheader + (maxblocks + 1) * rank *
sizeof(sion_int64);
140 printf(
"sionsplit: start_of_varheader: %lld bytes (%6.2f MB)\n", start_of_varheader, start_of_varheader / 1024.0 / 1024.0);
141 printf(
"sionsplit: size of file: %lld bytes (%6.2f MB)\n", sionsplit_filesize, sionsplit_filesize / 1024.0 / 1024.0);
143 if (sionsplit_filesize > 0) {
144 printf(
"sionsplit: file usage: %8.6f%%\n", (
double) sionsplit_sum_bytes / (
double) sionsplit_filesize * 100.0);
148 sprintf(fnmask,
"%s%%0%dd", prefix, digits);
149 printf(
"sionsplit: filename mask: %-s\n", fnmask);
151 sprintf(fnkmask,
"%s%%0%dd_%%012ld", prefix, digits);
152 printf(
"sionsplit: filename key mask: %-s\n", fnkmask);
157 for (rank = 0; rank < ntasks; rank++) {
158 if (chunksize<sion_localsizes[rank]) chunksize=sion_localsizes[rank];
159 if (chunksize<sion_blocksizes[rank]) chunksize=sion_blocksizes[rank];
161 printf(
"sionsplit: max chunksize: %lld\n", chunksize);
162 localbuffer = (
char *) malloc(chunksize *
sizeof(
char));
163 if (localbuffer == NULL) {
164 fprintf(stderr,
"cannot allocate localbuffer of size %lld , aborting ...\n", chunksize *
sizeof(
char));
169 for (rank = 0; rank < ntasks; rank++) {
171 sprintf(outfilename, fnmask, (
int) sion_globalranks[rank]);
173 sprintf(outfilename, fnmask, rank);
177 printf(
"sionsplit: generating file: %-s\n", outfilename);
180 fprintf(stderr,
"cannot open outfile %s , aborting ...\n", outfilename);
184 for (blknum = 0; blknum < sion_blockcount[rank]; blknum++) {
188 DPRINTFP((1,
"sionsplit", 0,
"after sion_seek sid=%d rank=%d blknum=%d fileposition=%lld\n", sid, rank, blknum,
sion_get_position(sid)));
191 left = sion_blocksizes[ntasks * blknum + rank];
194 DPRINTFP((8,
"sionsplit", 0,
"will read %lld bytes localbuffer+%lld\n", left, bsumread));
195 bread =
sion_fread(localbuffer + bsumread, 1, left, sid);
198 printf(
"sionsplit: %lld read left=%lld \n", bread, left);
202 left = sion_blocksizes[ntasks * blknum + rank];
204 printf(
"sionsplit: %lld wrote of left=%lld \n", bwrote, left);
218 printf(
"sionsplit: buffer size: %d\n", fsblksize);
219 localbuffer = (
char *) malloc(fsblksize *
sizeof(
char));
220 if (localbuffer == NULL) {
221 fprintf(stderr,
"cannot allocate localbuffer of size %lld , aborting ...\n", chunksize *
sizeof(
char));
225 for (rank = 0; rank < ntasks; rank++) {
228 printf(
"siondefrag: ->rank: %d\n", rank);
230 if (rank % 16 == 0) {
231 printf(
"[%d]", rank);
247 sprintf(outfilename, fnkmask, (
int) sion_globalranks[rank],(
long) key);
249 sprintf(outfilename, fnkmask, rank,(
long) key);
252 printf(
"sionsplit: generating file: '%-s'\n", outfilename);
255 fprintf(stderr,
"cannot open outfile %s , aborting ...\n", outfilename);
259 while( (bread=
sion_fread_key(localbuffer,key,1,fsblksize,sid))>0 ) {
264 printf(
"sionsplit: extracting now data of key[%12ld] (%lld bytes) (%lld bytes written)\n",
265 (
long) key, bread, bwrote);
276 free(sionsplit_sum_bytes_per_task);
282 void usage(
char *name)
284 fprintf(stderr,
"Usage: %s options <sionfn> <prefix>\n\n", name);
286 fprintf(stderr,
"Split SIONlib file <sionfn> into separate files. For each task a file is\n");
287 fprintf(stderr,
"created with the name <prefix><digits>\n\n");
289 fprintf(stderr,
"Example: %s data.sion data/file_\n", name);
290 fprintf(stderr,
" creates files data/file_00000, data/file_00001, ...\n\n");
292 fprintf(stderr,
"Options:\n");
293 fprintf(stderr,
" [-v] verbose mode\n");
294 fprintf(stderr,
" [-g] use global rank for numbering files\n");
295 fprintf(stderr,
" [-d <num>] number of digits for filename generation (default 5)\n");
296 fprintf(stderr,
" [-V] show version of SIONlib\n");
297 fprintf(stderr,
" [-h] show this help\n");
sion_int64 _sion_file_write(const void *data, sion_int64 bytes, _sion_fileptr *sion_fileptr)
Write data to file.
int sion_key_list_iterator_next(int sid, uint64_t *keyptr)
Forward to next key.
_sion_fileptr * _sion_file_open(const char *fname, unsigned int flags, unsigned int addflags)
Create and open a new file for writing.
Sion File Descriptor Structure.
int sion_get_endianness(void)
Return endianness.
sion_int64 sion_get_position(int sid)
Function that returns the current file position.
#define SION_FILE_FLAG_WRITE
int sion_get_file_endianness(int sid)
Returns edianness of data in file sid.
int sion_close(int sid)
Close a sion file.
int _sion_vcdtype(int sid)
int sion_get_locations(int sid, int *ntasks, int *maxchunks, sion_int64 *globalskip, sion_int64 *start_of_varheader, sion_int64 **sion_chunksizes, sion_int64 **sion_globalranks, sion_int64 **sion_blockcount, sion_int64 **sion_blocksizes)
Returns pointers to internal fields.
char * sion_keyval_type_to_str(int type)
Returns key value mode as string.
void * _sion_vcdtovcon(int sid)
#define SION_FILE_FLAG_ANSI
#define SION_FILE_FLAG_CREATE
int sion_seek(int sid, int rank, int currentblocknr, sion_int64 posinblk)
Function to set the file pointer to a new position.
int _sion_file_close(_sion_fileptr *sion_fileptr)
Close file and destroys fileptr structure.
int _sion_errorprint(int rc, int level, const char *format,...)
Internal SION error.
int sion_key_list_iterator_reset(int sid)
Resets key iterator.
#define SION_FILEDESCRIPTOR
size_t sion_fread(void *data, size_t size, size_t nitems, int sid)
Read data from sion file.
int sion_key_full_scan(int sid)
Performs a full scan of all meta data in current file.
size_t sion_fread_key(void *data, uint64_t key, size_t size, size_t nitems, int sid)
Read data for key.
int sion_open(char *fname, const char *file_mode, int *ntasks, int *nfiles, sion_int64 **chunksizes, sion_int32 *fsblksize, int **globalranks, FILE **fileptr)
Open a sion file in serial mode.