20 #define FILENAME_LENGTH 1024 21 #define FILE_MODE_LENGTH 1024 22 #define MAXNULLCNT 1000 26 static void usage(
char *name);
29 int main(
int argc,
char **argv)
31 char infilename[FILENAME_LENGTH];
32 char outfilename[FILENAME_LENGTH];
33 char file_mode[FILE_MODE_LENGTH];
34 char endianness[FILE_MODE_LENGTH];
35 char keyvalue[FILE_MODE_LENGTH];
39 sion_int64 max_chunksize = 0;
41 long localbuffer_size=-1;
42 sion_int64 left, bread, bsumread, bwrote, bsumwrote;
43 sion_int64 *chunksizes = NULL;
44 int *globalranks = NULL;
49 sion_int32 opt_fsblocksize = -1;
50 sion_int32 opt_nfiles = 1;
53 int ntasks, nfiles, onfiles;
55 int sid, outsid, size, blocks;
56 sion_int64 globalskip;
57 sion_int64 start_of_varheader;
58 sion_int64 *sion_chunksizes;
59 sion_int64 *sion_globalranks;
60 sion_int64 *sion_blockcount;
61 sion_int64 *sion_blocksizes;
62 sion_int64 *siondefrag_sum_bytes_per_task;
63 sion_int64 siondefrag_sum_bytes;
64 sion_int64 siondefrag_filesize;
65 sion_int64 siondefrag_sum_chunksizes;
77 if (argv[i][0] ==
'-') {
88 opt_nfiles = atoi(argv[++i]);
91 opt_fsblocksize = atoi(argv[++i]);
94 opt_fsblocksize = atoi(argv[++i]) MB;
100 fprintf(stderr,
"SIONlib utility %s (Version %d.%dp%d, fileformat version %d)\n", argv[0],
101 SION_MAIN_VERSION,SION_SUB_VERSION,
102 SION_VERSION_PATCHLEVEL,SION_FILEFORMAT_VERSION);
114 strcpy(infilename, argv[argc - 2]);
115 strcpy(outfilename, argv[argc - 1]);
117 printf(
"siondefrag: infilename: %-30s\n", infilename);
118 printf(
"siondefrag: outfilename: %-30s\n", outfilename);
123 sid =
sion_open(infilename,
"rb,posix", &ntasks, &nfiles, &chunksizes, &fsblksize, &globalranks, NULL);
127 return(
_sion_errorprint(SION_NOT_SUCCESS,_SION_ERROR_RETURN,
"invalid sion_filedesc, aborting %d ...\n", sid));
131 printf(
"siondefrag: sid: %d\n", sid);
132 printf(
"siondefrag: filename: %-30s\n", infilename);
133 printf(
"siondefrag: nfiles: %d\n", nfiles);
134 printf(
"siondefrag: fsblksize: %lu bytes (%6.2f MB)\n", (
unsigned long) fsblksize, fsblksize / 1024.0 / 1024.0);
139 sion_get_locations(sid, &size, &blocks, &globalskip, &start_of_varheader, &sion_chunksizes, &sion_globalranks, &sion_blockcount,
142 printf(
"siondefrag: max number of blocks: %d\n", blocks);
145 siondefrag_sum_bytes_per_task = (sion_int64 *) malloc(ntasks *
sizeof(sion_int64));
146 for (rank = 0; rank < size; rank++)
147 siondefrag_sum_bytes_per_task[rank] = 0;
148 for (rank = 0; rank < size; rank++) {
149 for (blknum = 0; blknum < sion_blockcount[rank]; blknum++) {
150 siondefrag_sum_bytes_per_task[rank] += sion_blocksizes[size * blknum + rank];
153 siondefrag_sum_chunksizes = 0;
154 siondefrag_sum_bytes = 0;
155 for (rank = 0; rank < size; rank++) {
156 siondefrag_sum_bytes += siondefrag_sum_bytes_per_task[rank];
157 siondefrag_sum_chunksizes+= sion_chunksizes[rank];
159 printf(
"siondefrag: datasize in file (aggr.): %lld bytes (%6.2f MB)\n", siondefrag_sum_bytes, siondefrag_sum_bytes / 1024.0 / 1024.0);
160 siondefrag_filesize = start_of_varheader + (blocks + 1) * rank *
sizeof(sion_int64);
161 printf(
"siondefrag: start_of_varheader: %lld bytes (%6.2f MB)\n", start_of_varheader, start_of_varheader / 1024.0 / 1024.0);
162 printf(
"siondefrag: size of chunks: %lld bytes (%6.2f MB)\n", siondefrag_sum_chunksizes, siondefrag_sum_chunksizes / 1024.0 / 1024.0);
163 if (siondefrag_filesize > 0) {
164 printf(
"siondefrag: file usage: %8.6f%%\n", (
double) siondefrag_sum_bytes / (
double) siondefrag_sum_chunksizes * 100.0);
168 for (rank = 0; rank < size; rank++) {
169 if (sion_chunksizes[rank] > max_chunksize)
170 max_chunksize = sion_chunksizes[rank];
171 for (blknum = 0; blknum < sion_blockcount[rank]; blknum++) {
172 if (sion_blocksizes[size * blknum + rank] > max_chunksize)
173 max_chunksize = sion_blocksizes[size * blknum + rank];
177 for (rank = 0; rank < size; rank++)
178 chunksizes[rank] = (sion_int64) siondefrag_sum_bytes_per_task[rank];
180 if (opt_fsblocksize != -1)
181 fsblksize = opt_fsblocksize;
183 onfiles = opt_nfiles;
188 sprintf(endianness,
"endianness=big");
190 sprintf(endianness,
"endianness=little");
193 sprintf(keyvalue,
"keyval=none");
195 sprintf(keyvalue,
"keyval=inline");
197 fprintf(stderr,
"unknown keyvalue mode ..., aborting\n");
206 printf(
"siondefrag: max chunksize: %lld\n", max_chunksize);
207 localbuffer = (
char *) malloc(max_chunksize *
sizeof(
char));
208 if (localbuffer == NULL) {
209 fprintf(stderr,
"cannot allocate localbuffer of size %lld , aborting ...\n", max_chunksize *
sizeof(
char));
214 sprintf(file_mode,
"wb,posix,%s",endianness);
215 outsid =
sion_open(outfilename, file_mode, &ntasks, &onfiles, &chunksizes, &fsblksize, &globalranks, NULL);
217 printf(
"siondefrag: outsid: %d\n", outsid);
218 printf(
"siondefrag: fsblksize outfile: %lu\n", (
unsigned long) fsblksize);
220 for (rank = 0; rank < size; rank++) {
224 printf(
"siondefrag: ->rank: %d\n", rank);
226 if (rank % 16 == 0) {
227 printf(
"[%d]", rank);
233 printf(
"siondefrag: copy now %lld blocks on rank=%d \n", sion_blockcount[rank], rank);
234 for (blknum = 0; blknum < sion_blockcount[rank]; blknum++) {
240 left = sion_blocksizes[size * blknum + rank];
242 printf(
"siondefrag: copy now block #%d with %lld bytes \n", blknum, left);
246 bread =
sion_fread(localbuffer + bsumread, 1, left, sid);
254 if (nullcount > MAXNULLCNT) {
255 fprintf(stderr,
"timeout on read data , aborting ...\n");
261 left = sion_blocksizes[size * blknum + rank];
264 bwrote =
sion_fwrite(localbuffer + bsumwrote, 1, left, outsid);
272 if (nullcount > MAXNULLCNT) {
273 fprintf(stderr,
"timeout on write data , aborting ...\n");
280 if (!verbose) printf(
"\n");
287 sprintf(file_mode,
"wb,posix,%s,%s",endianness,keyvalue);
288 outsid =
sion_open(outfilename, file_mode, &ntasks, &onfiles, &chunksizes, &fsblksize, &globalranks, NULL);
290 for (rank = 0; rank < size; rank++) {
293 printf(
"siondefrag: ->rank: %d\n", rank);
295 if (rank % 16 == 0) {
296 printf(
"[%d]", rank);
313 if((
long) localbuffer_size< (
long) len) {
314 if(localbuffer_size>0) {
317 localbuffer = (
char *) malloc(len *
sizeof(
char));
318 if (localbuffer == NULL) {
319 fprintf(stderr,
"cannot allocate localbuffer of size %zd , aborting ...\n", len *
sizeof(
char));
322 localbuffer_size=len;
326 printf(
"siondefrag: copy now key[%12ld] with %zd bytes \n", (
long) key, len);
332 fprintf(stderr,
"cannot read data of key %ld with len %ld, aborting ...\n",(
long) key, (
long) len *
sizeof(
char));
340 fprintf(stderr,
"cannot write data of key %ld with len %ld, aborting ...\n",(
long) key, (
long) len *
sizeof(
char));
351 if (!verbose) printf(
"\n");
354 free(siondefrag_sum_bytes_per_task);
362 void usage(
char *name)
364 fprintf(stderr,
"Usage: %s options <insionfn> <outfn>\n\n", name);
366 fprintf(stderr,
"Creates a new SIONlib file <outfn> from an existing <insionfn>. The\n");
367 fprintf(stderr,
"resulting file has only one chunk. This can be used to make SIONlib\n");
368 fprintf(stderr,
"files less sparse (e.g. by using -q 1 which effectively removes file\n");
369 fprintf(stderr,
"system block alignment).\n\n");
371 fprintf(stderr,
"Options:\n");
372 fprintf(stderr,
" [-Q <fsblksize>] filessystem blocksize for new sion file in MB\n");
373 fprintf(stderr,
" (default from input file)\n");
374 fprintf(stderr,
" [-q <fsblksize>] filessystem blocksize for new sion file in bytes\n");
375 fprintf(stderr,
" [-v] verbose mode\n");
376 fprintf(stderr,
" [-V] show version of SIONlib\n");
377 fprintf(stderr,
" [-h] show this help\n");
Sion File Descriptor Structure.
int sion_get_endianness(void)
Return endianness.
int sion_get_file_endianness(int sid)
Returns edianness of data in file sid.
int sion_fread_key_iterator_next(int sid, uint64_t *key, size_t *size)
Forward to next key.
#define SION_KEYVAL_INLINE
int sion_close(int sid)
Close a sion file.
int _sion_vcdtype(int sid)
int sion_get_locations(int sid, int *ntasks, int *maxchunks, sion_int64 *globalskip, sion_int64 *start_of_varheader, sion_int64 **sion_chunksizes, sion_int64 **sion_globalranks, sion_int64 **sion_blockcount, sion_int64 **sion_blocksizes)
Returns pointers to internal fields.
size_t sion_fwrite(const void *data, size_t size, size_t nitems, int sid)
Write data to sion file.
char * sion_keyval_type_to_str(int type)
Returns key value mode as string.
void * _sion_vcdtovcon(int sid)
int sion_seek(int sid, int rank, int currentblocknr, sion_int64 posinblk)
Function to set the file pointer to a new position.
int sion_fread_key_iterator_reset(int sid)
Resets key iterator.
int _sion_errorprint(int rc, int level, const char *format,...)
Internal SION error.
#define SION_FILEDESCRIPTOR
size_t sion_fread(void *data, size_t size, size_t nitems, int sid)
Read data from sion file.
size_t sion_fread_key(void *data, uint64_t key, size_t size, size_t nitems, int sid)
Read data for key.
size_t sion_fwrite_key(const void *data, uint64_t key, size_t size, size_t nitems, int sid)
Writes data for key.
int sion_open(char *fname, const char *file_mode, int *ntasks, int *nfiles, sion_int64 **chunksizes, sion_int32 *fsblksize, int **globalranks, FILE **fileptr)
Open a sion file in serial mode.