SIONlib  1.7.3
Scalable I/O library for parallel access to task-local files
partest_opts.c
1 /****************************************************************************
2 ** SIONLIB http://www.fz-juelich.de/jsc/sionlib **
3 *****************************************************************************
4 ** Copyright (c) 2008-2019 **
5 ** Forschungszentrum Juelich, Juelich Supercomputing Centre **
6 ** **
7 ** See the file COPYRIGHT in the package base directory for details **
8 ****************************************************************************/
9 
10 #define _XOPEN_SOURCE 700
11 
12 #include <stdio.h>
13 #include <stdlib.h>
14 #include <string.h>
15 #include <ctype.h>
16 #include <unistd.h>
17 #include <mpi.h>
18 #include <time.h>
19 #include <math.h>
20 #ifdef _SION_AIX
21 #include "getopt_long.h"
22 #else
23 #include <getopt.h>
24 #endif
25 #include "sion.h"
26 #include "partest_opts.h"
27 #include "partest.h"
28 
29 int init_options ( _test_options *options) {
30  int rc=0;
31 
32  options->type = 0;
33  options->bluegene = 0;
34  options->bluegene_np = 0;
35  options->bluegene_sort = 0;
36  options->bufsize = 10 MB;
37  options->totalsize = 20 MB;
38  options->chunksize = options->totalsize;
39  options->fsblksize = 2.0 MB;
40  options->verbose = 0;
41  options->debug = 0;
42  options->Debug = 0;
43  options->numfiles = 1;
44  options->startoffset = 0;
45  options->read_task_offset = 0;
46  options->collectiveopenforread = 1;
47  options->collectivewrite = 0;
48  options->collectiveread = 0;
49  options->unlink_files = 0;
50  options->suppress_checksum = 0;
51  options->serialize_blocknum = -1;
52  options->mpiio_lb = -1;
53  options->mpiio_bs = -1;
54  options->mpiio_sa = -1;
55  options->factor = 0.0;
56 
57  options->do_write = 1;
58  options->do_read = 1;
59  options->use_posix = 0;
60 
61  options->globalsize = -1;
62 
63  strcpy(options->filename, "partest_parfile.sion");
64 
65  return(rc);
66 }
67 
68 int parse_options_std ( int argc, char **argv, _test_options *options) {
69  int rc=0;
70  int i;
71 
72  /* parse command line */
73  i = 1;
74  while (i < argc) {
75  if (argv[i][0] == '-') {
76  switch (argv[i][1]) {
77  case 'f':
78  strcpy(options->filename, argv[++i]);
79  break;
80  case 'F':
81  options->factor = atof(argv[++i]);
82  break;
83  case 'n':
84  options->numfiles = atoi(argv[++i]);
85  break;
86  case 'b':
87  options->bufsize = (sion_int64) atoi(argv[++i]);
88  break;
89  case 'B':
90  options->bufsize = (sion_int64) atoi(argv[++i]) MB;
91  break;
92  case 'g':
93  options->globalsize = (sion_int64) atoi(argv[++i]);
94  break;
95  case 'G':
96  options->globalsize = (sion_int64) atoi(argv[++i]) * 1024 MB;
97  break;
98  case 'I':
99  options->do_write = 0;
100  break;
101  case 'O':
102  options->do_read = 0;
103  break;
104  case 'L':
105  options->use_posix = 1;
106  break;
107  case 's':
108  options->totalsize = (sion_int64) atoi(argv[++i]);
109  break;
110  case 'S':
111  options->totalsize = (sion_int64) atoi(argv[++i]) MB;
112  break;
113  case 'r':
114  options->chunksize = (sion_int64) atoi(argv[++i]);
115  break;
116  case 'R':
117  options->chunksize = (sion_int64) atoi(argv[++i]) MB;
118  break;
119  case 'q':
120  options->fsblksize = atoi(argv[++i]);
121  break;
122  case 'Q':
123  options->fsblksize = atof(argv[++i]) MB;
124  break;
125  case 'o':
126  options->startoffset = (sion_int64) atoi(argv[++i]);
127  break;
128  case 'P':
129  options->bluegene = 1;
130  break;
131  case 'p':
132  options->bluegene_np = atoi(argv[++i]);
133  break;
134  case 'T':
135  options->type = atoi(argv[++i]);
136  break;
137  case 'w':
138  options->mpiio_lb = atoi(argv[++i]);
139  break;
140  case 'W':
141  options->mpiio_bs = atoi(argv[++i]);
142  break;
143  case 'x':
144  options->mpiio_sa = atoi(argv[++i]);
145  break;
146  case 'Z':
147  options->read_task_offset = atoi(argv[++i]);
148  break;
149  case 'X':
150  options->unlink_files = atoi(argv[++i]);
151  break;
152  case 'j':
153  options->serialize_blocknum = atoi(argv[++i]);
154  break;
155  case 'C':
156  options->suppress_checksum = 1;
157  break;
158  case 'M':
159  options->collectivewrite++;
160  break;
161  case 'm':
162  options->collectiveread++;
163  break;
164  case 'd':
165  options->debug++;
166  break;
167  case 'D':
168  options->Debug++;
169  break;
170  case 'v':
171  options->verbose++;
172  break;
173  default:
174  printf("Arg default: %s\n", argv[i]);
175  usage(argv[0]);
176  }
177  }
178  else {
179  printf("Arg error: %s\n", argv[i]);
180  usage(argv[0]);
181  }
182  i++;
183  }
184 
185  return(rc);
186 }
187 
188 
189 int parse_options_long ( int argc, char **argv, _test_options *options) {
190  int rc=1;
191  int c;
192  int option_index = 0;
193  static struct option long_options[] = {
194  {"filename", required_argument, 0, 'f'},
195  {"numfiles", required_argument, 0, 'n'},
196  {"chunksize", required_argument, 0, 'r'},
197  {"fsblksize", required_argument, 0, 'q'},
198  {"testtype", required_argument, 0, 'T'},
199  {"bufsize", required_argument, 0, 'b'},
200  {"totalsize", required_argument, 0, 'g'},
201  {"localsize", required_argument, 0, 's'},
202  {"factor", required_argument, 0, 'F'},
203  {"write", required_argument, 0, 'W'},
204  {"read", required_argument, 0, 'R'},
205 
206  {"verbose", required_argument, 0, 'v'},
207  {"nochecksum",required_argument, 0, 'C'},
208  {"debugtask", required_argument, 0, 'd'},
209  {"Debugtask", required_argument, 0, 'D'},
210 
211  {"posix", required_argument, 0, 'L'},
212  {"collwrite", required_argument, 0, 'M'},
213  {"collread", required_argument, 0, 'm'},
214 
215  {"taskoffset",required_argument, 0, 'Z'},
216  {"byteoffset",required_argument, 0, 'O'},
217  {"serialized",required_argument, 0, 'j'},
218  {"unlinkfiles",required_argument, 0, 'X'},
219  {"bgionode", required_argument, 0, 'P'},
220  {"bgtaskperionode", required_argument, 0, 'p'},
221  {"bgtasksort", required_argument, 0, 'Y'},
222  {"hintlargeblock", required_argument, 0, 'w'},
223  {"hintiobufsize", required_argument, 0, 'Q'},
224  {"hintsparseacess", required_argument, 0, 'x'},
225 
226 
227  {0, 0, 0, 0}
228  };
229 
230  while(1) {
231  c = getopt_long(argc, argv, "f:n:r:q:T:b:g:s:F:W:R:vCdDLMmZ:X:O:j:wQ:x",
232  long_options, &option_index);
233  if (c == -1)
234  break;
235 
236  switch (c) {
237  case 0:
238  printf("option %s", long_options[option_index].name);
239  if (optarg)
240  printf(" with arg %s", optarg);
241  printf("\n");
242  break;
243  case 'f':
244  strcpy(options->filename, optarg);
245  break;
246  case 'n':
247  options->numfiles = atoi(optarg);
248  break;
249  case 'r':
250  options->chunksize = to_bytes(optarg);
251  break;
252  case 'q':
253  options->fsblksize = to_bytes(optarg);
254  break;
255 
256  case 'T':
257  options->type = atoi(optarg);
258  break;
259  case 'b':
260  options->bufsize = to_bytes(optarg);
261  break;
262  case 'g':
263  options->globalsize = to_bytes(optarg);
264  break;
265  case 's':
266  options->totalsize = to_bytes(optarg);
267  break;
268  case 'F':
269  options->factor = atoi(optarg);
270  break;
271  case 'W':
272  options->do_write = atoi(optarg);
273  break;
274  case 'R':
275  options->do_read = atoi(optarg);
276  break;
277 
278  case 'v':
279  if(optarg) {
280  options->verbose = atoi(optarg);
281  } else options->verbose = 1;
282 
283  break;
284 
285  case 'C':
286  if(optarg) {
287  options->suppress_checksum = atoi(optarg);
288  } else options->suppress_checksum = 1;
289  break;
290 
291  case 'd':
292  if(optarg) {
293  options->debug = atoi(optarg);
294  } else options->debug = 1;
295  break;
296 
297  case 'D':
298  if(optarg) {
299  options->Debug = atoi(optarg);
300  } else options->Debug = 1;
301  break;
302 
303  case 'L':
304  if(optarg) {
305  options->use_posix = atoi(optarg);
306  } else options->use_posix = 1;
307  break;
308 
309  case 'M':
310  if(optarg) {
311  options->collectivewrite = atoi(optarg);
312  } else options->collectivewrite = 1;
313  break;
314 
315  case 'm':
316  if(optarg) {
317  options->collectiveread = atoi(optarg);
318  } else options->collectiveread = 1;
319  break;
320 
321  case 'Z':
322  options->read_task_offset = atoi(optarg);
323  break;
324 
325  case 'O':
326  options->startoffset = atoi(optarg);
327  break;
328 
329  case 'j':
330  options->serialize_blocknum = atoi(optarg);
331  break;
332 
333  case 'X':
334  if(optarg) {
335  options->unlink_files = atoi(optarg);
336  } else options->unlink_files = 1;
337  break;
338 
339  case 'P':
340  if(optarg) {
341  options->bluegene = atoi(optarg);
342  } else options->bluegene = 1;
343  break;
344 
345  case 'p':
346  options->bluegene_np = atoi(optarg);
347  break;
348 
349  case 'Y':
350  options->bluegene_sort = atoi(optarg);
351  break;
352 
353  case 'w':
354  if(optarg) {
355  options->mpiio_lb = atoi(optarg);
356  } else options->mpiio_lb = 1;
357  break;
358 
359  case 'Q':
360  options->mpiio_bs = atoi(optarg);
361  break;
362 
363  case 'x':
364  if(optarg) {
365  options->mpiio_sa = atoi(optarg);
366  } else options->mpiio_sa = 1;
367  break;
368 
369  default:
370  printf("?? getopt_long returned character code 0%o ??\n", c);
371  rc=0;
372  return(rc);
373  }
374  }
375 
376  return(rc);
377 }
378 
379 int distribute_options_mpi ( _test_options *options) {
380  int rc=0;
381 
382  MPI_Bcast(options->filename, FNAMELEN, MPI_CHAR, 0, MPI_COMM_WORLD);
383  MPI_Bcast(&options->factor, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD);
384 
385  MPI_Bcast(&options->bufsize, 1, SION_MPI_INT64, 0, MPI_COMM_WORLD);
386  MPI_Bcast(&options->totalsize, 1, SION_MPI_INT64, 0, MPI_COMM_WORLD);
387  MPI_Bcast(&options->globalsize, 1, SION_MPI_INT64, 0, MPI_COMM_WORLD);
388  MPI_Bcast(&options->chunksize, 1, SION_MPI_INT64, 0, MPI_COMM_WORLD);
389  MPI_Bcast(&options->fsblksize, 1, SION_MPI_INT32, 0, MPI_COMM_WORLD);
390 
391  MPI_Bcast(&options->type, 1, MPI_INT, 0, MPI_COMM_WORLD);
392  MPI_Bcast(&options->verbose, 1, MPI_INT, 0, MPI_COMM_WORLD);
393  MPI_Bcast(&options->bluegene, 1, MPI_INT, 0, MPI_COMM_WORLD);
394  MPI_Bcast(&options->bluegene_np, 1, MPI_INT, 0, MPI_COMM_WORLD);
395  MPI_Bcast(&options->unlink_files, 1, MPI_INT, 0, MPI_COMM_WORLD);
396  MPI_Bcast(&options->serialize_blocknum, 1, MPI_INT, 0, MPI_COMM_WORLD);
397 
398  MPI_Bcast(&options->collectivewrite, 1, MPI_INT, 0, MPI_COMM_WORLD);
399  MPI_Bcast(&options->collectiveread, 1, MPI_INT, 0, MPI_COMM_WORLD);
400 
401  MPI_Bcast(&options->numfiles, 1, MPI_INT, 0, MPI_COMM_WORLD);
402  MPI_Bcast(&options->debug, 1, MPI_INT, 0, MPI_COMM_WORLD);
403  MPI_Bcast(&options->Debug, 1, MPI_INT, 0, MPI_COMM_WORLD);
404  MPI_Bcast(&options->read_task_offset, 1, MPI_INT, 0, MPI_COMM_WORLD);
405 
406  MPI_Bcast(&options->suppress_checksum, 1, MPI_INT, 0, MPI_COMM_WORLD);
407  MPI_Bcast(&options->do_read, 1, MPI_INT, 0, MPI_COMM_WORLD);
408  MPI_Bcast(&options->do_write, 1, MPI_INT, 0, MPI_COMM_WORLD);
409  MPI_Bcast(&options->use_posix, 1, MPI_INT, 0, MPI_COMM_WORLD);
410 
411  return(rc);
412 }
413 
414 
415 void usage_long(char *name) {
416 
417  fprintf(stderr, "Usage: %s options\n\n", name);
418  fprintf(stderr, "Options:\n");
419 
420  fprintf(stderr, " Sion File Settings:\n");
421  fprintf(stderr, " [-f filename] (--filename[=]) filename of direct access file\n");
422  fprintf(stderr, " [-n <number of files>] (--numfiles[=]) number of files (def: 1)\n");
423  fprintf(stderr, " [-r <chunksize>] (--chunksize[=]) sion chunk size (see size format)\n");
424  fprintf(stderr, " [-q <fsblksize>] (--fsblksize[=]) size of filesystem blocks (see\n");
425  fprintf(stderr, " size format)\n");
426 
427  fprintf(stderr, " Test Configuration:\n");
428  fprintf(stderr, " [-T <type>] (--testtype[=]) type of test\n");
429  fprintf(stderr, " (0:SION, standard),\n");
430  fprintf(stderr, " (1: SION:, independant read),\n");
431  fprintf(stderr, " (2:MPI IO), (3: Task-Local-File)\n");
432  fprintf(stderr, " [-b <bufsize>] (--bufsize[=]) size of blocks written in ONE\n");
433  fprintf(stderr, " fwrite (see size format)\n");
434  fprintf(stderr, " [-g <totalsize>] (--totalsize[=]) global total size of data written\n");
435  fprintf(stderr, " (see size format)\n");
436  fprintf(stderr, " [-s <localsize>] (--localsize[=]) local size of data written by each\n");
437  fprintf(stderr, " processor (see size format)\n");
438  fprintf(stderr, " [-F <factor>] (--factor[=]) factor for random size\n");
439  fprintf(stderr, " (0.0 to 1.0, def: 0.0)\n");
440  fprintf(stderr, " [-R (0|1)] (--read[=]) switch read off/on\n");
441  fprintf(stderr, " [-W (0|1|2)] (--write[=]) switch write off, on, or double\n");
442  fprintf(stderr, " write\n");
443 
444  fprintf(stderr, " Special Test Options:\n");
445  fprintf(stderr, " [-v] (--verbose[=](0|1)) verbose print info for each\n");
446  fprintf(stderr, " task\n");
447  fprintf(stderr, " [-C] (--nochecksum[=](0|1)) suppress checksum\n");
448  fprintf(stderr, " [-d] (--debugtask[=](0|1)) debug task 0\n");
449  fprintf(stderr, " [-D] (--Debugtask[=](0|1)) debug task n\n");
450  fprintf(stderr, " [-L] (--posix[=](0|1)) use POSIX calls instead of\n");
451  fprintf(stderr, " ANSI calls\n");
452  fprintf(stderr, " [-M] (--collwrite[=](0|1)) use collective write if\n");
453  fprintf(stderr, " possible\n");
454  fprintf(stderr, " [-m] (--collread[=](0|1)) use collective read if\n");
455  fprintf(stderr, " possible\n");
456  fprintf(stderr, " [-Z <offset>] (--taskoffset[=]) shift tasks numbering for\n");
457  fprintf(stderr, " reading by offset to prevent\n");
458  fprintf(stderr, " data caching of file-system\n");
459  fprintf(stderr, " (default: 0)\n");
460  fprintf(stderr, " [-O <bytes>] (--byteoffset[=]) start offset, write <bytes>\n");
461  fprintf(stderr, " first before using blksize\n");
462  fprintf(stderr, " (default: 0)\n");
463  fprintf(stderr, " [-j <#tasks>] (--serialized[=]) serialize I/O, only I/O of\n");
464  fprintf(stderr, " #tasks are running in parallel\n");
465  fprintf(stderr, " (-1 -> all tasks in parallel,\n");
466  fprintf(stderr, " -2 -> use transactions,\n");
467  fprintf(stderr, " -def: -1)\n");
468  fprintf(stderr, " [-X] (--unlinkfiles[=](0|1)) remove files after test\n");
469 
470  fprintf(stderr, " Blue Gene/L, Blue Gene/P , Blue Gene/Q:\n");
471  fprintf(stderr, " [-P <mode>] (--bgionode[=](0|1|2)) order tasks by BG I/O-node\n");
472  fprintf(stderr, " (0 none, 1 ION, 2 IOB)\n");
473  fprintf(stderr, " [-Y <sort>] (--bgtaskperionode[=]) number of tasks per BG\n");
474  fprintf(stderr, " I/O-node\n");
475  fprintf(stderr, " [-p <numtasks>] (--bgtasksort[=](0|1)) sort task inside local\n");
476  fprintf(stderr, " communicator (0 distance to\n");
477  fprintf(stderr, " ION, 1 global rank)\n");
478 
479  fprintf(stderr, " MPI-IO, GPFS options:\n");
480  fprintf(stderr, " [-w] (--hintlargeblock[=](0|1)) Hint MPI-IO, IBM, Large\n");
481  fprintf(stderr, " Block IO\n");
482  fprintf(stderr, " [-Q <size>] (--hintiobufsize[=]) Hint MPI-IO, IBM, IO\n");
483  fprintf(stderr, " bufsize in KB\n");
484  fprintf(stderr, " [-x] (--hintsparseacess[=](0|1)) Hint MPI-IO, IBM, sparse\n");
485  fprintf(stderr, " access\n");
486 
487  fprintf(stderr, " Size Formats: <d>[k, K, kib, kiB, Kib, KiB] for kibi bytes\n");
488  fprintf(stderr, " <d>[kb, kB, Kb, KB] for kilo bytes\n");
489  fprintf(stderr, " similarly for [M, G, T]\n");
490 
491 }
492 
493 void usage(char *name) {
494 
495  fprintf(stderr, "Usage: %s options\n\n", name);
496  fprintf(stderr, "Options:\n\n");
497 
498  fprintf(stderr, " Sion File Settings:\n");
499  fprintf(stderr, " [-f filename] filename of direct access file\n");
500  fprintf(stderr, " [-n <number of files>] number of files\n");
501  fprintf(stderr, " [-r <chunksize>] sion chunk size in bytes\n");
502  fprintf(stderr, " [-R <chunksize>] sion chunk size in MBytes\n");
503  fprintf(stderr, " [-q <fsblksize>] size of filesystem blocks in bytes\n");
504  fprintf(stderr, " [-Q <fsblksize>] size of filesystem blocks in MBytes\n");
505 
506  fprintf(stderr, " Test Configuration:\n");
507  fprintf(stderr, " [-F <factor>] factor for random size (0.0 to 1.0, def: 0.0)\n");
508  fprintf(stderr, " [-b <bufsize>] size of blocks written in ONE fwrite in bytes\n");
509  fprintf(stderr, " [-B <bufsize>] size of blocks written in ONE fwrite in MBytes\n");
510  fprintf(stderr, " [-g <globaltotalsize>] global total size of data written in bytes\n");
511  fprintf(stderr, " [-G <globaltotalsize>] global total size of data written in GBytes\n");
512  fprintf(stderr, " [-s <totalsize>] total size of data written by each processor in bytes\n");
513  fprintf(stderr, " [-S <totalsize>] total size of data written by each processor in MBytes\n");
514  fprintf(stderr, " [-T <type>] type of test (0): w/o collective read; (2): MPI IO\n");
515  fprintf(stderr, " [-v] verbose print info for each task\n");
516  fprintf(stderr, " [-C] suppress checksum\n");
517  fprintf(stderr, " [-I] only read data\n");
518  fprintf(stderr, " [-O] only write data\n");
519  fprintf(stderr, " [-d] debug task 0\n");
520  fprintf(stderr, " [-D] debug task n\n");
521  fprintf(stderr, " [-L] use POSIX calls instead of ANSI call\n");
522  fprintf(stderr, " [-M] use collective write if possible\n");
523  fprintf(stderr, " [-m] use collective read if possible\n");
524  fprintf(stderr, " [-Z <offset>] shift tasks numbering for reading by offset to ommit\n");
525  fprintf(stderr, " data caching of file-system (default: 0)\n");
526  fprintf(stderr, " [-o <bytes>] start offset, write <bytes> first before using blksize\n");
527  fprintf(stderr, " (default: 0)\n");
528  fprintf(stderr, " [-j <#tasks>] serialize I/O, only I/O of #tasks are running in\n");
529  fprintf(stderr, " parallel (-1 -> all tasks in parallel, -2 -> use\n");
530  fprintf(stderr, " transactions, def: -1)\n");
531 
532  fprintf(stderr, " Blue Gene/L, Blue Gene/P , Blue Gene/Q:\n");
533  fprintf(stderr, " [-P <mode>] order tasks by BG I/O-node (0 none, 1 ION, 2 IOB)\n");
534  fprintf(stderr, " [-Y <sort>] number of tasks per BG I/O-node\n");
535  fprintf(stderr, " [-p <numtasks>] number of tasks per BG I/O-node\n");
536 
537  fprintf(stderr, " MPI-IO, GPFS options:\n");
538  fprintf(stderr, " [-w <1|0>] Hint MPI-IO, IBM, Large Block IO\n");
539  fprintf(stderr, " [-W <size>] Hint MPI-IO, IBM, IO bufsize in KB\n");
540  fprintf(stderr, " [-x <1|0>] Hint MPI-IO, IBM, sparse access\n");
541 
542  exit(1);
543 
544 }
545 
546 
547 /* convert following formats to bytes:
548  t,T,Ti,Tib,TiB -> 1024*1024*1024*1024 bytes
549  g,G,Gi,Gib,GiB -> 1024*1024*1024 bytes
550  m,M,Mi,Mib,MiB -> 1024*1024 bytes
551  k,K,Ki,Kib,KiB -> 1024 bytes
552 
553  Tb,TB -> 1000*1000*1000*1000 bytes
554  Gb,GB -> 1000*1000*1000 bytes
555  Mb,MB -> 1000*1000 bytes
556  Kb,KB -> 1000 bytes
557 */
558 sion_int64 to_bytes ( char *option) {
559  int rc;
560  sion_int64 size = -1;
561  sion_int64 factor = 1024;
562  char char1='\0',char2='\0',char3='\0';
563  rc = sscanf(option, "%lld%c%c%c", &size, &char1,&char2,&char3);
564  if (rc == 3 && ( (char2=='b') || (char2=='B') )) {
565  factor = 1000;
566  }
567 
568  if ( (char1=='T') || (char1=='t') ) size*=factor*factor*factor*factor;
569  if ( (char1=='G') || (char1=='g') ) size*=factor*factor*factor;
570  if ( (char1=='M') || (char1=='m') ) size*=factor*factor;
571  if ( (char1=='K') || (char1=='k') ) size*=factor;
572 
573  return size;
574 }
_test_options_struct
Definition: partest_opts.h:15
Generated by   doxygen 1.8.15