SIONlib  1.7.1
Scalable I/O library for parallel access to task-local files
partest_opts.c
1 /****************************************************************************
2 ** SIONLIB http://www.fz-juelich.de/jsc/sionlib **
3 *****************************************************************************
4 ** Copyright (c) 2008-2016 **
5 ** Forschungszentrum Juelich, Juelich Supercomputing Centre **
6 ** **
7 ** See the file COPYRIGHT in the package base directory for details **
8 ****************************************************************************/
9 
10 #include <stdio.h>
11 #include <stdlib.h>
12 #include <string.h>
13 #include <ctype.h>
14 #include <unistd.h>
15 #include <mpi.h>
16 #include <time.h>
17 #include <math.h>
18 #ifdef _SION_AIX
19 #include "getopt_long.h"
20 #else
21 #include <getopt.h>
22 #endif
23 #include "sion.h"
24 #include "partest_opts.h"
25 #include "partest.h"
26 
27 int init_options ( _test_options *options) {
28  int rc=0;
29 
30  options->type = 0;
31  options->bluegene = 0;
32  options->bluegene_np = 0;
33  options->bluegene_sort = 0;
34  options->bufsize = 10 MB;
35  options->totalsize = 20 MB;
36  options->chunksize = options->totalsize;
37  options->fsblksize = 2.0 MB;
38  options->verbose = 0;
39  options->debug = 0;
40  options->Debug = 0;
41  options->numfiles = 1;
42  options->startoffset = 0;
43  options->read_task_offset = 0;
44  options->collectiveopenforread = 1;
45  options->collectivewrite = 0;
46  options->collectiveread = 0;
47  options->unlink_files = 0;
48  options->suppress_checksum = 0;
49  options->serialize_blocknum = -1;
50  options->mpiio_lb = -1;
51  options->mpiio_bs = -1;
52  options->mpiio_sa = -1;
53  options->factor = 0.0;
54 
55  options->do_write = 1;
56  options->do_read = 1;
57  options->use_posix = 0;
58 
59  options->globalsize = -1;
60 
61  strcpy(options->filename, "partest_parfile.sion");
62 
63  return(rc);
64 }
65 
66 int parse_options_std ( int argc, char **argv, _test_options *options) {
67  int rc=0;
68  int i;
69 
70  /* parse command line */
71  i = 1;
72  while (i < argc) {
73  if (argv[i][0] == '-') {
74  switch (argv[i][1]) {
75  case 'f':
76  strcpy(options->filename, argv[++i]);
77  break;
78  case 'F':
79  options->factor = atof(argv[++i]);
80  break;
81  case 'n':
82  options->numfiles = atoi(argv[++i]);
83  break;
84  case 'b':
85  options->bufsize = (sion_int64) atoi(argv[++i]);
86  break;
87  case 'B':
88  options->bufsize = (sion_int64) atoi(argv[++i]) MB;
89  break;
90  case 'g':
91  options->globalsize = (sion_int64) atoi(argv[++i]);
92  break;
93  case 'G':
94  options->globalsize = (sion_int64) atoi(argv[++i]) * 1024 MB;
95  break;
96  case 'I':
97  options->do_write = 0;
98  break;
99  case 'O':
100  options->do_read = 0;
101  break;
102  case 'L':
103  options->use_posix = 1;
104  break;
105  case 's':
106  options->totalsize = (sion_int64) atoi(argv[++i]);
107  break;
108  case 'S':
109  options->totalsize = (sion_int64) atoi(argv[++i]) MB;
110  break;
111  case 'r':
112  options->chunksize = (sion_int64) atoi(argv[++i]);
113  break;
114  case 'R':
115  options->chunksize = (sion_int64) atoi(argv[++i]) MB;
116  break;
117  case 'q':
118  options->fsblksize = atoi(argv[++i]);
119  break;
120  case 'Q':
121  options->fsblksize = atof(argv[++i]) MB;
122  break;
123  case 'o':
124  options->startoffset = (sion_int64) atoi(argv[++i]);
125  break;
126  case 'P':
127  options->bluegene = 1;
128  break;
129  case 'p':
130  options->bluegene_np = atoi(argv[++i]);
131  break;
132  case 'T':
133  options->type = atoi(argv[++i]);
134  break;
135  case 'w':
136  options->mpiio_lb = atoi(argv[++i]);
137  break;
138  case 'W':
139  options->mpiio_bs = atoi(argv[++i]);
140  break;
141  case 'x':
142  options->mpiio_sa = atoi(argv[++i]);
143  break;
144  case 'Z':
145  options->read_task_offset = atoi(argv[++i]);
146  break;
147  case 'X':
148  options->unlink_files = atoi(argv[++i]);
149  break;
150  case 'j':
151  options->serialize_blocknum = atoi(argv[++i]);
152  break;
153  case 'C':
154  options->suppress_checksum = 1;
155  break;
156  case 'M':
157  options->collectivewrite++;
158  break;
159  case 'm':
160  options->collectiveread++;
161  break;
162  case 'd':
163  options->debug++;
164  break;
165  case 'D':
166  options->Debug++;
167  break;
168  case 'v':
169  options->verbose++;
170  break;
171  default:
172  printf("Arg default: %s\n", argv[i]);
173  usage(argv[0]);
174  }
175  }
176  else {
177  printf("Arg error: %s\n", argv[i]);
178  usage(argv[0]);
179  }
180  i++;
181  }
182 
183  return(rc);
184 }
185 
186 
187 int parse_options_long ( int argc, char **argv, _test_options *options) {
188  int rc=1;
189  int c;
190  int option_index = 0;
191  static struct option long_options[] = {
192  {"filename", required_argument, 0, 'f'},
193  {"numfiles", required_argument, 0, 'n'},
194  {"chunksize", required_argument, 0, 'r'},
195  {"fsblksize", required_argument, 0, 'q'},
196  {"testtype", required_argument, 0, 'T'},
197  {"bufsize", required_argument, 0, 'b'},
198  {"totalsize", required_argument, 0, 'g'},
199  {"localsize", required_argument, 0, 's'},
200  {"factor", required_argument, 0, 'F'},
201  {"write", required_argument, 0, 'W'},
202  {"read", required_argument, 0, 'R'},
203 
204  {"verbose", required_argument, 0, 'v'},
205  {"nochecksum",required_argument, 0, 'C'},
206  {"debugtask", required_argument, 0, 'd'},
207  {"Debugtask", required_argument, 0, 'D'},
208 
209  {"posix", required_argument, 0, 'L'},
210  {"collwrite", required_argument, 0, 'M'},
211  {"collread", required_argument, 0, 'm'},
212 
213  {"taskoffset",required_argument, 0, 'Z'},
214  {"byteoffset",required_argument, 0, 'O'},
215  {"serialized",required_argument, 0, 'j'},
216  {"unlinkfiles",required_argument, 0, 'X'},
217  {"bgionode", required_argument, 0, 'P'},
218  {"bgtaskperionode", required_argument, 0, 'p'},
219  {"bgtasksort", required_argument, 0, 'Y'},
220  {"hintlargeblock", required_argument, 0, 'w'},
221  {"hintiobufsize", required_argument, 0, 'Q'},
222  {"hintsparseacess", required_argument, 0, 'x'},
223 
224 
225  {0, 0, 0, 0}
226  };
227 
228  while(1) {
229  c = getopt_long(argc, argv, "f:n:r:q:T:b:g:s:F:W:R:vCdDLMmZ:X:O:j:wQ:x",
230  long_options, &option_index);
231  if (c == -1)
232  break;
233 
234  switch (c) {
235  case 0:
236  printf("option %s", long_options[option_index].name);
237  if (optarg)
238  printf(" with arg %s", optarg);
239  printf("\n");
240  break;
241  case 'f':
242  strcpy(options->filename, optarg);
243  break;
244  case 'n':
245  options->numfiles = atoi(optarg);
246  break;
247  case 'r':
248  options->chunksize = to_bytes(optarg);
249  break;
250  case 'q':
251  options->fsblksize = to_bytes(optarg);
252  break;
253 
254  case 'T':
255  options->type = atoi(optarg);
256  break;
257  case 'b':
258  options->bufsize = to_bytes(optarg);
259  break;
260  case 'g':
261  options->globalsize = to_bytes(optarg);
262  break;
263  case 's':
264  options->totalsize = to_bytes(optarg);
265  break;
266  case 'F':
267  options->factor = atoi(optarg);
268  break;
269  case 'W':
270  options->do_write = atoi(optarg);
271  break;
272  case 'R':
273  options->do_read = atoi(optarg);
274  break;
275 
276  case 'v':
277  if(optarg) {
278  options->verbose = atoi(optarg);
279  } else options->verbose = 1;
280 
281  break;
282 
283  case 'C':
284  if(optarg) {
285  options->suppress_checksum = atoi(optarg);
286  } else options->suppress_checksum = 1;
287  break;
288 
289  case 'd':
290  if(optarg) {
291  options->debug = atoi(optarg);
292  } else options->debug = 1;
293  break;
294 
295  case 'D':
296  if(optarg) {
297  options->Debug = atoi(optarg);
298  } else options->Debug = 1;
299  break;
300 
301  case 'L':
302  if(optarg) {
303  options->use_posix = atoi(optarg);
304  } else options->use_posix = 1;
305  break;
306 
307  case 'M':
308  if(optarg) {
309  options->collectivewrite = atoi(optarg);
310  } else options->collectivewrite = 1;
311  break;
312 
313  case 'm':
314  if(optarg) {
315  options->collectiveread = atoi(optarg);
316  } else options->collectiveread = 1;
317  break;
318 
319  case 'Z':
320  options->read_task_offset = atoi(optarg);
321  break;
322 
323  case 'O':
324  options->startoffset = atoi(optarg);
325  break;
326 
327  case 'j':
328  options->serialize_blocknum = atoi(optarg);
329  break;
330 
331  case 'X':
332  if(optarg) {
333  options->unlink_files = atoi(optarg);
334  } else options->unlink_files = 1;
335  break;
336 
337  case 'P':
338  if(optarg) {
339  options->bluegene = atoi(optarg);
340  } else options->bluegene = 1;
341  break;
342 
343  case 'p':
344  options->bluegene_np = atoi(optarg);
345  break;
346 
347  case 'Y':
348  options->bluegene_sort = atoi(optarg);
349  break;
350 
351  case 'w':
352  if(optarg) {
353  options->mpiio_lb = atoi(optarg);
354  } else options->mpiio_lb = 1;
355  break;
356 
357  case 'Q':
358  options->mpiio_bs = atoi(optarg);
359  break;
360 
361  case 'x':
362  if(optarg) {
363  options->mpiio_sa = atoi(optarg);
364  } else options->mpiio_sa = 1;
365  break;
366 
367  default:
368  printf("?? getopt_long returned character code 0%o ??\n", c);
369  rc=0;
370  return(rc);
371  }
372  }
373 
374  return(rc);
375 }
376 
377 int distribute_options_mpi ( _test_options *options) {
378  int rc=0;
379 
380  MPI_Bcast(options->filename, FNAMELEN, MPI_CHAR, 0, MPI_COMM_WORLD);
381  MPI_Bcast(&options->factor, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD);
382 
383  MPI_Bcast(&options->bufsize, 1, SION_MPI_INT64, 0, MPI_COMM_WORLD);
384  MPI_Bcast(&options->totalsize, 1, SION_MPI_INT64, 0, MPI_COMM_WORLD);
385  MPI_Bcast(&options->globalsize, 1, SION_MPI_INT64, 0, MPI_COMM_WORLD);
386  MPI_Bcast(&options->chunksize, 1, SION_MPI_INT64, 0, MPI_COMM_WORLD);
387  MPI_Bcast(&options->fsblksize, 1, SION_MPI_INT32, 0, MPI_COMM_WORLD);
388 
389  MPI_Bcast(&options->type, 1, MPI_INT, 0, MPI_COMM_WORLD);
390  MPI_Bcast(&options->verbose, 1, MPI_INT, 0, MPI_COMM_WORLD);
391  MPI_Bcast(&options->bluegene, 1, MPI_INT, 0, MPI_COMM_WORLD);
392  MPI_Bcast(&options->bluegene_np, 1, MPI_INT, 0, MPI_COMM_WORLD);
393  MPI_Bcast(&options->unlink_files, 1, MPI_INT, 0, MPI_COMM_WORLD);
394  MPI_Bcast(&options->serialize_blocknum, 1, MPI_INT, 0, MPI_COMM_WORLD);
395 
396  MPI_Bcast(&options->collectivewrite, 1, MPI_INT, 0, MPI_COMM_WORLD);
397  MPI_Bcast(&options->collectiveread, 1, MPI_INT, 0, MPI_COMM_WORLD);
398 
399  MPI_Bcast(&options->numfiles, 1, MPI_INT, 0, MPI_COMM_WORLD);
400  MPI_Bcast(&options->debug, 1, MPI_INT, 0, MPI_COMM_WORLD);
401  MPI_Bcast(&options->Debug, 1, MPI_INT, 0, MPI_COMM_WORLD);
402  MPI_Bcast(&options->read_task_offset, 1, MPI_INT, 0, MPI_COMM_WORLD);
403 
404  MPI_Bcast(&options->suppress_checksum, 1, MPI_INT, 0, MPI_COMM_WORLD);
405  MPI_Bcast(&options->do_read, 1, MPI_INT, 0, MPI_COMM_WORLD);
406  MPI_Bcast(&options->do_write, 1, MPI_INT, 0, MPI_COMM_WORLD);
407  MPI_Bcast(&options->use_posix, 1, MPI_INT, 0, MPI_COMM_WORLD);
408 
409  return(rc);
410 }
411 
412 
413 void usage_long(char *name) {
414 
415  fprintf(stderr, "Usage: %s options\n\n", name);
416  fprintf(stderr, "Options:\n");
417 
418  fprintf(stderr, " Sion File Settings:\n");
419  fprintf(stderr, " [-f filename] (--filename[=]) filename of direct access file\n");
420  fprintf(stderr, " [-n <number of files>] (--numfiles[=]) number of files (def: 1)\n");
421  fprintf(stderr, " [-r <chunksize>] (--chunksize[=]) sion chunk size (see size format)\n");
422  fprintf(stderr, " [-q <fsblksize>] (--fsblksize[=]) size of filesystem blocks (see\n");
423  fprintf(stderr, " size format)\n");
424 
425  fprintf(stderr, " Test Configuration:\n");
426  fprintf(stderr, " [-T <type>] (--testtype[=]) type of test\n");
427  fprintf(stderr, " (0:SION, standard),\n");
428  fprintf(stderr, " (1: SION:, independant read),\n");
429  fprintf(stderr, " (2:MPI IO), (3: Task-Local-File)\n");
430  fprintf(stderr, " [-b <bufsize>] (--bufsize[=]) size of blocks written in ONE\n");
431  fprintf(stderr, " fwrite (see size format)\n");
432  fprintf(stderr, " [-g <totalsize>] (--totalsize[=]) global total size of data written\n");
433  fprintf(stderr, " (see size format)\n");
434  fprintf(stderr, " [-s <localsize>] (--localsize[=]) local size of data written by each\n");
435  fprintf(stderr, " processor (see size format)\n");
436  fprintf(stderr, " [-F <factor>] (--factor[=]) factor for random size\n");
437  fprintf(stderr, " (0.0 to 1.0, def: 0.0)\n");
438  fprintf(stderr, " [-R (0|1)] (--read[=]) switch read off/on\n");
439  fprintf(stderr, " [-W (0|1|2)] (--write[=]) switch write off, on, or double\n");
440  fprintf(stderr, " write\n");
441 
442  fprintf(stderr, " Special Test Options:\n");
443  fprintf(stderr, " [-v] (--verbose[=](0|1)) verbose print info for each\n");
444  fprintf(stderr, " task\n");
445  fprintf(stderr, " [-C] (--nochecksum[=](0|1)) suppress checksum\n");
446  fprintf(stderr, " [-d] (--debugtask[=](0|1)) debug task 0\n");
447  fprintf(stderr, " [-D] (--Debugtask[=](0|1)) debug task n\n");
448  fprintf(stderr, " [-L] (--posix[=](0|1)) use POSIX calls instead of\n");
449  fprintf(stderr, " ANSI calls\n");
450  fprintf(stderr, " [-M] (--collwrite[=](0|1)) use collective write if\n");
451  fprintf(stderr, " possible\n");
452  fprintf(stderr, " [-m] (--collread[=](0|1)) use collective read if\n");
453  fprintf(stderr, " possible\n");
454  fprintf(stderr, " [-Z <offset>] (--taskoffset[=]) shift tasks numbering for\n");
455  fprintf(stderr, " reading by offset to prevent\n");
456  fprintf(stderr, " data caching of file-system\n");
457  fprintf(stderr, " (default: 0)\n");
458  fprintf(stderr, " [-O <bytes>] (--byteoffset[=]) start offset, write <bytes>\n");
459  fprintf(stderr, " first before using blksize\n");
460  fprintf(stderr, " (default: 0)\n");
461  fprintf(stderr, " [-j <#tasks>] (--serialized[=]) serialize I/O, only I/O of\n");
462  fprintf(stderr, " #tasks are running in parallel\n");
463  fprintf(stderr, " (-1 -> all tasks in parallel,\n");
464  fprintf(stderr, " -2 -> use transactions,\n");
465  fprintf(stderr, " -def: -1)\n");
466  fprintf(stderr, " [-X] (--unlinkfiles[=](0|1)) remove files after test\n");
467 
468  fprintf(stderr, " Blue Gene/L, Blue Gene/P , Blue Gene/Q:\n");
469  fprintf(stderr, " [-P <mode>] (--bgionode[=](0|1|2)) order tasks by BG I/O-node\n");
470  fprintf(stderr, " (0 none, 1 ION, 2 IOB)\n");
471  fprintf(stderr, " [-Y <sort>] (--bgtaskperionode[=]) number of tasks per BG\n");
472  fprintf(stderr, " I/O-node\n");
473  fprintf(stderr, " [-p <numtasks>] (--bgtasksort[=](0|1)) sort task inside local\n");
474  fprintf(stderr, " communicator (0 distance to\n");
475  fprintf(stderr, " ION, 1 global rank)\n");
476 
477  fprintf(stderr, " MPI-IO, GPFS options:\n");
478  fprintf(stderr, " [-w] (--hintlargeblock[=](0|1)) Hint MPI-IO, IBM, Large\n");
479  fprintf(stderr, " Block IO\n");
480  fprintf(stderr, " [-Q <size>] (--hintiobufsize[=]) Hint MPI-IO, IBM, IO\n");
481  fprintf(stderr, " bufsize in KB\n");
482  fprintf(stderr, " [-x] (--hintsparseacess[=](0|1)) Hint MPI-IO, IBM, sparse\n");
483  fprintf(stderr, " access\n");
484 
485  fprintf(stderr, " Size Formats: <d>[k, K, kib, kiB, Kib, KiB] for kibi bytes\n");
486  fprintf(stderr, " <d>[kb, kB, Kb, KB] for kilo bytes\n");
487  fprintf(stderr, " similarly for [M, G, T]\n");
488 
489 }
490 
491 void usage(char *name) {
492 
493  fprintf(stderr, "Usage: %s options\n\n", name);
494  fprintf(stderr, "Options:\n\n");
495 
496  fprintf(stderr, " Sion File Settings:\n");
497  fprintf(stderr, " [-f filename] filename of direct access file\n");
498  fprintf(stderr, " [-n <number of files>] number of files\n");
499  fprintf(stderr, " [-r <chunksize>] sion chunk size in bytes\n");
500  fprintf(stderr, " [-R <chunksize>] sion chunk size in MBytes\n");
501  fprintf(stderr, " [-q <fsblksize>] size of filesystem blocks in bytes\n");
502  fprintf(stderr, " [-Q <fsblksize>] size of filesystem blocks in MBytes\n");
503 
504  fprintf(stderr, " Test Configuration:\n");
505  fprintf(stderr, " [-F <factor>] factor for random size (0.0 to 1.0, def: 0.0)\n");
506  fprintf(stderr, " [-b <bufsize>] size of blocks written in ONE fwrite in bytes\n");
507  fprintf(stderr, " [-B <bufsize>] size of blocks written in ONE fwrite in MBytes\n");
508  fprintf(stderr, " [-g <globaltotalsize>] global total size of data written in bytes\n");
509  fprintf(stderr, " [-G <globaltotalsize>] global total size of data written in GBytes\n");
510  fprintf(stderr, " [-s <totalsize>] total size of data written by each processor in bytes\n");
511  fprintf(stderr, " [-S <totalsize>] total size of data written by each processor in MBytes\n");
512  fprintf(stderr, " [-T <type>] type of test (0): w/o collective read; (2): MPI IO\n");
513  fprintf(stderr, " [-v] verbose print info for each task\n");
514  fprintf(stderr, " [-C] suppress checksum\n");
515  fprintf(stderr, " [-I] only read data\n");
516  fprintf(stderr, " [-O] only write data\n");
517  fprintf(stderr, " [-d] debug task 0\n");
518  fprintf(stderr, " [-D] debug task n\n");
519  fprintf(stderr, " [-L] use POSIX calls instead of ANSI call\n");
520  fprintf(stderr, " [-M] use collective write if possible\n");
521  fprintf(stderr, " [-m] use collective read if possible\n");
522  fprintf(stderr, " [-Z <offset>] shift tasks numbering for reading by offset to ommit\n");
523  fprintf(stderr, " data caching of file-system (default: 0)\n");
524  fprintf(stderr, " [-o <bytes>] start offset, write <bytes> first before using blksize\n");
525  fprintf(stderr, " (default: 0)\n");
526  fprintf(stderr, " [-j <#tasks>] serialize I/O, only I/O of #tasks are running in\n");
527  fprintf(stderr, " parallel (-1 -> all tasks in parallel, -2 -> use\n");
528  fprintf(stderr, " transactions, def: -1)\n");
529 
530  fprintf(stderr, " Blue Gene/L, Blue Gene/P , Blue Gene/Q:\n");
531  fprintf(stderr, " [-P <mode>] order tasks by BG I/O-node (0 none, 1 ION, 2 IOB)\n");
532  fprintf(stderr, " [-Y <sort>] number of tasks per BG I/O-node\n");
533  fprintf(stderr, " [-p <numtasks>] number of tasks per BG I/O-node\n");
534 
535  fprintf(stderr, " MPI-IO, GPFS options:\n");
536  fprintf(stderr, " [-w <1|0>] Hint MPI-IO, IBM, Large Block IO\n");
537  fprintf(stderr, " [-W <size>] Hint MPI-IO, IBM, IO bufsize in KB\n");
538  fprintf(stderr, " [-x <1|0>] Hint MPI-IO, IBM, sparse access\n");
539 
540  exit(1);
541 
542 }
543 
544 
545 /* convert following formats to bytes:
546  t,T,Ti,Tib,TiB -> 1024*1024*1024*1024 bytes
547  g,G,Gi,Gib,GiB -> 1024*1024*1024 bytes
548  m,M,Mi,Mib,MiB -> 1024*1024 bytes
549  k,K,Ki,Kib,KiB -> 1024 bytes
550 
551  Tb,TB -> 1000*1000*1000*1000 bytes
552  Gb,GB -> 1000*1000*1000 bytes
553  Mb,MB -> 1000*1000 bytes
554  Kb,KB -> 1000 bytes
555 */
556 sion_int64 to_bytes ( char *option) {
557  int rc;
558  sion_int64 size = -1;
559  sion_int64 factor = 1024;
560  char char1='\0',char2='\0',char3='\0';
561  rc = sscanf(option, "%lld%c%c%c", &size, &char1,&char2,&char3);
562  if (rc == 3 && ( (char2=='b') || (char2=='B') )) {
563  factor = 1000;
564  }
565 
566  if ( (char1=='T') || (char1=='t') ) size*=factor*factor*factor*factor;
567  if ( (char1=='G') || (char1=='g') ) size*=factor*factor*factor;
568  if ( (char1=='M') || (char1=='m') ) size*=factor*factor;
569  if ( (char1=='K') || (char1=='k') ) size*=factor;
570 
571  return size;
572 }
_test_options_struct
Definition: partest_opts.h:15
Generated by   doxygen 1.8.15