16 #include "sion_error_handler.h" 21 #define FILENAME_LENGTH 1024 22 #define FILE_MODE_LENGTH 1024 23 #define MAXNULLCNT 1000 27 static void usage(
char *name);
30 int main(
int argc,
char **argv)
32 char infilename[FILENAME_LENGTH];
33 char outfilename[FILENAME_LENGTH];
34 char file_mode[FILE_MODE_LENGTH];
35 char endianness[FILE_MODE_LENGTH];
36 char keyvalue[FILE_MODE_LENGTH];
40 sion_int64 max_chunksize = 0;
41 char *localbuffer = NULL;
42 long localbuffer_size=-1;
43 sion_int64 left, bread, bsumread, bwrote, bsumwrote;
44 sion_int64 *chunksizes = NULL;
45 int *globalranks = NULL;
50 sion_int32 opt_fsblocksize = -1;
51 sion_int32 opt_nfiles = 1;
54 int ntasks, nfiles, onfiles;
56 int sid, outsid, size, blocks;
57 sion_int64 globalskip;
58 sion_int64 start_of_varheader;
59 sion_int64 *sion_chunksizes;
60 sion_int64 *sion_globalranks;
61 sion_int64 *sion_blockcount;
62 sion_int64 *sion_blocksizes;
63 sion_int64 *siondefrag_sum_bytes_per_task;
64 sion_int64 siondefrag_sum_bytes;
65 sion_int64 siondefrag_filesize;
66 sion_int64 siondefrag_sum_chunksizes;
78 if (argv[i][0] ==
'-') {
89 opt_nfiles = atoi(argv[++i]);
92 opt_fsblocksize = atoi(argv[++i]);
95 opt_fsblocksize = atoi(argv[++i]) MB;
101 fprintf(stderr,
"SIONlib utility %s (Version %d.%dp%d, fileformat version %d)\n", argv[0],
102 SION_MAIN_VERSION,SION_SUB_VERSION,
103 SION_VERSION_PATCHLEVEL,SION_FILEFORMAT_VERSION);
115 strcpy(infilename, argv[argc - 2]);
116 strcpy(outfilename, argv[argc - 1]);
118 printf(
"siondefrag: infilename: %-30s\n", infilename);
119 printf(
"siondefrag: outfilename: %-30s\n", outfilename);
124 sid =
sion_open(infilename,
"rb,posix", &ntasks, &nfiles, &chunksizes, &fsblksize, &globalranks, NULL);
128 return(_sion_errorprint(SION_NOT_SUCCESS,_SION_ERROR_RETURN,
"invalid sion_filedesc, aborting %d ...\n", sid));
132 printf(
"siondefrag: sid: %d\n", sid);
133 printf(
"siondefrag: filename: %-30s\n", infilename);
134 printf(
"siondefrag: nfiles: %d\n", nfiles);
135 printf(
"siondefrag: fsblksize: %lu bytes (%6.2f MB)\n", (
unsigned long) fsblksize, fsblksize / 1024.0 / 1024.0);
140 sion_get_locations(sid, &size, &blocks, &globalskip, &start_of_varheader, &sion_chunksizes, &sion_globalranks, &sion_blockcount,
143 printf(
"siondefrag: max number of blocks: %d\n", blocks);
146 siondefrag_sum_bytes_per_task = (sion_int64 *) malloc(ntasks *
sizeof(sion_int64));
147 for (rank = 0; rank < size; rank++)
148 siondefrag_sum_bytes_per_task[rank] = 0;
149 for (rank = 0; rank < size; rank++) {
150 for (blknum = 0; blknum < sion_blockcount[rank]; blknum++) {
151 siondefrag_sum_bytes_per_task[rank] += sion_blocksizes[size * blknum + rank];
154 siondefrag_sum_chunksizes = 0;
155 siondefrag_sum_bytes = 0;
156 for (rank = 0; rank < size; rank++) {
157 siondefrag_sum_bytes += siondefrag_sum_bytes_per_task[rank];
158 siondefrag_sum_chunksizes+= sion_chunksizes[rank];
160 printf(
"siondefrag: datasize in file (aggr.): %lld bytes (%6.2f MB)\n", siondefrag_sum_bytes, siondefrag_sum_bytes / 1024.0 / 1024.0);
161 siondefrag_filesize = start_of_varheader + (blocks + 1) * rank *
sizeof(sion_int64);
162 printf(
"siondefrag: start_of_varheader: %lld bytes (%6.2f MB)\n", start_of_varheader, start_of_varheader / 1024.0 / 1024.0);
163 printf(
"siondefrag: size of chunks: %lld bytes (%6.2f MB)\n", siondefrag_sum_chunksizes, siondefrag_sum_chunksizes / 1024.0 / 1024.0);
164 if (siondefrag_filesize > 0) {
165 printf(
"siondefrag: file usage: %8.6f%%\n", (
double) siondefrag_sum_bytes / (
double) siondefrag_sum_chunksizes * 100.0);
169 for (rank = 0; rank < size; rank++) {
170 if (sion_chunksizes[rank] > max_chunksize)
171 max_chunksize = sion_chunksizes[rank];
172 for (blknum = 0; blknum < sion_blockcount[rank]; blknum++) {
173 if (sion_blocksizes[size * blknum + rank] > max_chunksize)
174 max_chunksize = sion_blocksizes[size * blknum + rank];
178 for (rank = 0; rank < size; rank++)
179 chunksizes[rank] = (sion_int64) siondefrag_sum_bytes_per_task[rank];
181 if (opt_fsblocksize != -1)
182 fsblksize = opt_fsblocksize;
184 onfiles = opt_nfiles;
189 sprintf(endianness,
"endianness=big");
191 sprintf(endianness,
"endianness=little");
194 sprintf(keyvalue,
"keyval=none");
196 sprintf(keyvalue,
"keyval=inline");
198 fprintf(stderr,
"unknown keyvalue mode ..., aborting\n");
199 free(siondefrag_sum_bytes_per_task);
208 printf(
"siondefrag: max chunksize: %lld\n", max_chunksize);
209 localbuffer = (
char *) malloc(max_chunksize *
sizeof(
char));
210 if (localbuffer == NULL) {
211 free(siondefrag_sum_bytes_per_task);
212 fprintf(stderr,
"cannot allocate localbuffer of size %lld , aborting ...\n", max_chunksize *
sizeof(
char));
217 sprintf(file_mode,
"wb,posix,%s",endianness);
218 outsid =
sion_open(outfilename, file_mode, &ntasks, &onfiles, &chunksizes, &fsblksize, &globalranks, NULL);
220 printf(
"siondefrag: outsid: %d\n", outsid);
221 printf(
"siondefrag: fsblksize outfile: %lu\n", (
unsigned long) fsblksize);
223 for (rank = 0; rank < size; rank++) {
227 printf(
"siondefrag: ->rank: %d\n", rank);
229 if (rank % 16 == 0) {
230 printf(
"[%d]", rank);
236 printf(
"siondefrag: copy now %lld blocks on rank=%d \n", sion_blockcount[rank], rank);
237 for (blknum = 0; blknum < sion_blockcount[rank]; blknum++) {
243 left = sion_blocksizes[size * blknum + rank];
245 printf(
"siondefrag: copy now block #%d with %lld bytes \n", blknum, left);
249 bread =
sion_fread(localbuffer + bsumread, 1, left, sid);
257 if (nullcount > MAXNULLCNT) {
258 fprintf(stderr,
"timeout on read data , aborting ...\n");
264 left = sion_blocksizes[size * blknum + rank];
267 bwrote =
sion_fwrite(localbuffer + bsumwrote, 1, left, outsid);
275 if (nullcount > MAXNULLCNT) {
276 fprintf(stderr,
"timeout on write data , aborting ...\n");
283 if (!verbose) printf(
"\n");
290 sprintf(file_mode,
"wb,posix,%s,%s",endianness,keyvalue);
291 outsid =
sion_open(outfilename, file_mode, &ntasks, &onfiles, &chunksizes, &fsblksize, &globalranks, NULL);
293 for (rank = 0; rank < size; rank++) {
296 printf(
"siondefrag: ->rank: %d\n", rank);
298 if (rank % 16 == 0) {
299 printf(
"[%d]", rank);
316 if((
long) localbuffer_size< (
long) len) {
317 if(localbuffer_size>0) {
320 localbuffer = (
char *) malloc(len *
sizeof(
char));
321 if (localbuffer == NULL) {
322 fprintf(stderr,
"cannot allocate localbuffer of size %zu , aborting ...\n", len *
sizeof(
char));
323 free(siondefrag_sum_bytes_per_task);
326 localbuffer_size=len;
330 printf(
"siondefrag: copy now key[%12ld] with %zu bytes \n", (
long) key, len);
336 fprintf(stderr,
"cannot read data of key %ld with len %zu, aborting ...\n",(
long) key, (
size_t) len *
sizeof(
char));
337 free(siondefrag_sum_bytes_per_task);
345 fprintf(stderr,
"cannot write data of key %ld with len %zu, aborting ...\n",(
long) key, (
size_t) len *
sizeof(
char));
346 free(siondefrag_sum_bytes_per_task);
357 if (!verbose) printf(
"\n");
360 free(siondefrag_sum_bytes_per_task);
368 void usage(
char *name)
370 fprintf(stderr,
"Usage: %s options <insionfn> <outfn>\n\n", name);
372 fprintf(stderr,
"Creates a new SIONlib file <outfn> from an existing <insionfn>. The\n");
373 fprintf(stderr,
"resulting file has only one chunk. This can be used to make SIONlib\n");
374 fprintf(stderr,
"files less sparse (e.g. by using -q 1 which effectively removes file\n");
375 fprintf(stderr,
"system block alignment).\n\n");
377 fprintf(stderr,
"Options:\n");
378 fprintf(stderr,
" [-Q <fsblksize>] filessystem blocksize for new sion file in MB\n");
379 fprintf(stderr,
" (default from input file)\n");
380 fprintf(stderr,
" [-q <fsblksize>] filessystem blocksize for new sion file in bytes\n");
381 fprintf(stderr,
" [-v] verbose mode\n");
382 fprintf(stderr,
" [-V] show version of SIONlib\n");
383 fprintf(stderr,
" [-h] show this help\n");
Sion File Descriptor Structure.
int sion_get_endianness(void)
Return endianness.
int sion_get_file_endianness(int sid)
Returns edianness of data in file sid.
int sion_fread_key_iterator_next(int sid, uint64_t *key, size_t *size)
Forward to next key.
#define SION_KEYVAL_INLINE
int sion_close(int sid)
Close a sion file.
int _sion_vcdtype(int sid)
int sion_get_locations(int sid, int *ntasks, int *maxchunks, sion_int64 *globalskip, sion_int64 *start_of_varheader, sion_int64 **sion_chunksizes, sion_int64 **sion_globalranks, sion_int64 **sion_blockcount, sion_int64 **sion_blocksizes)
Returns pointers to internal fields.
size_t sion_fwrite(const void *data, size_t size, size_t nitems, int sid)
Write data to sion file.
char * sion_keyval_type_to_str(int type)
Returns key value mode as string.
void * _sion_vcdtovcon(int sid)
int sion_seek(int sid, int rank, int currentblocknr, sion_int64 posinblk)
Function to set the file pointer to a new position.
int sion_fread_key_iterator_reset(int sid)
Resets key iterator.
#define SION_FILEDESCRIPTOR
size_t sion_fread(void *data, size_t size, size_t nitems, int sid)
Read data from sion file.
size_t sion_fread_key(void *data, uint64_t key, size_t size, size_t nitems, int sid)
Read data for key.
size_t sion_fwrite_key(const void *data, uint64_t key, size_t size, size_t nitems, int sid)
Writes data for key.
int sion_open(char *fname, const char *file_mode, int *ntasks, int *nfiles, sion_int64 **chunksizes, sion_int32 *fsblksize, int **globalranks, FILE **fileptr)
Open a sion file in serial mode.