9 #define _XOPEN_SOURCE 700
18 #include "sion_error_handler.h"
24 #define FILENAME_LENGTH 1024
26 static void usage(
char *name);
28 int main(
int argc,
char **argv)
31 char infilename[FILENAME_LENGTH];
32 char outfilename[FILENAME_LENGTH];
33 char prefix[FILENAME_LENGTH];
34 char fnmask[FILENAME_LENGTH];
35 char fnkmask[FILENAME_LENGTH];
39 sion_int64 chunksize = 0;
40 sion_int64 left, bread, bsumread, bwrote;
45 int useglobalranks = 0;
48 int sid, ntasks, nfiles, maxblocks;
50 sion_int64 globalskip;
51 sion_int64 start_of_varheader;
52 sion_int64 *sion_localsizes;
53 sion_int64 *sion_globalranks;
54 sion_int64 *sion_blockcount;
55 sion_int64 *sion_blocksizes;
56 sion_int64 *sionsplit_sum_bytes_per_task;
57 sion_int64 sionsplit_sum_bytes;
58 sion_int64 sionsplit_filesize;
70 if (argv[i][0] ==
'-') {
73 digits = atoi(argv[++i]);
82 fprintf(stderr,
"SIONlib utility %s (Version %d.%dp%d, fileformat version %d)\n", argv[0],
83 SION_MAIN_VERSION,SION_SUB_VERSION,
84 SION_VERSION_PATCHLEVEL,SION_FILEFORMAT_VERSION);
96 strcpy(infilename, argv[argc - 2]);
97 strcpy(prefix, argv[argc - 1]);
99 printf(
"sionsplit: filename: %-30s\n", infilename);
100 printf(
"sionsplit: prefix: %-30s\n", prefix);
102 sid =
sion_open(infilename,
"rb,posix", &ntasks, &nfiles, NULL, &fsblksize, NULL, NULL);
106 return(_sion_errorprint(SION_NOT_SUCCESS,_SION_ERROR_RETURN,
"invalid sion_filedesc, aborting %d ...\n", sid));
109 printf(
"sionsplit: sid: %d\n", sid);
110 printf(
"sionsplit: filename: %-30s\n", infilename);
111 printf(
"sionsplit: number of tasks: %d\n", ntasks);
112 printf(
"sionsplit: number of files: %d\n", nfiles);
113 printf(
"sionsplit: number of digits: %d\n", digits);
114 printf(
"sionsplit: use global ranks: %d\n", useglobalranks);
118 printf(
"sionsplit: fsblksize: %lu bytes (%6.2f MB)\n", (
unsigned long) fsblksize, fsblksize / 1024.0 / 1024.0);
120 sion_get_locations(sid, &ntasks, &maxblocks, &globalskip, &start_of_varheader, &sion_localsizes, &sion_globalranks, &sion_blockcount,
123 printf(
"sionsplit: max number of chunks: %d\n", maxblocks);
126 sionsplit_sum_bytes_per_task = (sion_int64 *) malloc(ntasks *
sizeof(sion_int64));
127 for (rank = 0; rank < ntasks; rank++)
128 sionsplit_sum_bytes_per_task[rank] = 0;
129 for (rank = 0; rank < ntasks; rank++) {
130 for (blknum = 0; blknum < maxblocks; blknum++) {
131 sionsplit_sum_bytes_per_task[rank] += sion_blocksizes[ntasks * blknum + rank];
135 sionsplit_sum_bytes = 0;
136 for (rank = 0; rank < ntasks; rank++)
137 sionsplit_sum_bytes += sionsplit_sum_bytes_per_task[rank];
139 printf(
"sionsplit: datasize in file (aggr.): %lld bytes (%6.2f MB)\n", sionsplit_sum_bytes, sionsplit_sum_bytes / 1024.0 / 1024.0);
141 sionsplit_filesize = start_of_varheader + (maxblocks + 1) * rank *
sizeof(sion_int64);
143 printf(
"sionsplit: start_of_varheader: %lld bytes (%6.2f MB)\n", start_of_varheader, start_of_varheader / 1024.0 / 1024.0);
144 printf(
"sionsplit: size of file: %lld bytes (%6.2f MB)\n", sionsplit_filesize, sionsplit_filesize / 1024.0 / 1024.0);
146 if (sionsplit_filesize > 0) {
147 printf(
"sionsplit: file usage: %8.6f%%\n", (
double) sionsplit_sum_bytes / (
double) sionsplit_filesize * 100.0);
151 sprintf(fnmask,
"%s%%0%dd", prefix, digits);
152 printf(
"sionsplit: filename mask: %-s\n", fnmask);
154 sprintf(fnkmask,
"%s%%0%dd_%%012ld", prefix, digits);
155 printf(
"sionsplit: filename key mask: %-s\n", fnkmask);
160 for (rank = 0; rank < ntasks; rank++) {
161 if (chunksize<sion_localsizes[rank]) chunksize=sion_localsizes[rank];
162 if (chunksize<sion_blocksizes[rank]) chunksize=sion_blocksizes[rank];
164 printf(
"sionsplit: max chunksize: %lld\n", chunksize);
165 localbuffer = (
char *) malloc(chunksize *
sizeof(
char));
166 if (localbuffer == NULL) {
167 fprintf(stderr,
"cannot allocate localbuffer of size %lld , aborting ...\n", chunksize *
sizeof(
char));
168 free(sionsplit_sum_bytes_per_task);
173 for (rank = 0; rank < ntasks; rank++) {
175 sprintf(outfilename, fnmask, (
int) sion_globalranks[rank]);
177 sprintf(outfilename, fnmask, rank);
181 printf(
"sionsplit: generating file: %-s\n", outfilename);
184 fprintf(stderr,
"cannot open outfile %s , aborting ...\n", outfilename);
185 free(sionsplit_sum_bytes_per_task);
190 for (blknum = 0; blknum < sion_blockcount[rank]; blknum++) {
194 DPRINTFP((1,
"sionsplit", 0,
"after sion_seek sid=%d rank=%d blknum=%d fileposition=%lld\n", sid, rank, blknum,
sion_get_position(sid)));
197 left = sion_blocksizes[ntasks * blknum + rank];
200 DPRINTFP((8,
"sionsplit", 0,
"will read %lld bytes localbuffer+%lld\n", left, bsumread));
201 bread =
sion_fread(localbuffer + bsumread, 1, left, sid);
204 printf(
"sionsplit: %lld read left=%lld \n", bread, left);
208 left = sion_blocksizes[ntasks * blknum + rank];
210 printf(
"sionsplit: %lld wrote of left=%lld \n", bwrote, left);
224 printf(
"sionsplit: buffer size: %d\n", fsblksize);
225 localbuffer = (
char *) malloc(fsblksize *
sizeof(
char));
226 if (localbuffer == NULL) {
227 fprintf(stderr,
"cannot allocate localbuffer of size %lld , aborting ...\n", chunksize *
sizeof(
char));
228 free(sionsplit_sum_bytes_per_task);
232 for (rank = 0; rank < ntasks; rank++) {
235 printf(
"siondefrag: ->rank: %d\n", rank);
237 if (rank % 16 == 0) {
238 printf(
"[%d]", rank);
254 sprintf(outfilename, fnkmask, (
int) sion_globalranks[rank],(
long) key);
256 sprintf(outfilename, fnkmask, rank,(
long) key);
259 printf(
"sionsplit: generating file: '%-s'\n", outfilename);
262 fprintf(stderr,
"cannot open outfile %s , aborting ...\n", outfilename);
264 free(sionsplit_sum_bytes_per_task);
268 while( (bread=
sion_fread_key(localbuffer,key,1,fsblksize,sid))>0 ) {
273 printf(
"sionsplit: extracting now data of key[%12ld] (%lld bytes) (%lld bytes written)\n",
274 (
long) key, bread, bwrote);
285 free(sionsplit_sum_bytes_per_task);
291 void usage(
char *name)
293 fprintf(stderr,
"Usage: %s options <sionfn> <prefix>\n\n", name);
295 fprintf(stderr,
"Split SIONlib file <sionfn> into separate files. For each task a file is\n");
296 fprintf(stderr,
"created with the name <prefix><digits>\n\n");
298 fprintf(stderr,
"Example: %s data.sion data/file_\n", name);
299 fprintf(stderr,
" creates files data/file_00000, data/file_00001, ...\n\n");
301 fprintf(stderr,
"Options:\n");
302 fprintf(stderr,
" [-v] verbose mode\n");
303 fprintf(stderr,
" [-g] use global rank for numbering files\n");
304 fprintf(stderr,
" [-d <num>] number of digits for filename generation (default 5)\n");
305 fprintf(stderr,
" [-V] show version of SIONlib\n");
306 fprintf(stderr,
" [-h] show this help\n");