21 #include "sion_error_handler.h" 37 #include "sion_generic_buddy.h" 61 #define DFUNCTION "_sion_paropen_generic_buddy" 62 int _sion_paropen_generic_buddy(
69 sion_int64 *chunksize,
70 sion_int32 *fsblksize,
77 int rc = SION_SUCCESS;
78 int b, capability, pass, buddylevel;
79 sion_int32 file_globalrank;
80 sion_int64 file_chunksize;
81 char *buddy_fn, *nfname=NULL;
86 buddylevel = atoi(_sion_flags_get(flags_store,
"buddy")->val);
87 if (buddylevel==0) buddylevel=1;
89 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"allocate memory for %d buddy levels\n", buddylevel));
92 if (buddies == NULL) {
93 return(_sion_errorprint(SION_NOT_SUCCESS,_SION_ERROR_RETURN,
"cannot allocate buddies structure of size %lu (_sion_generic_buddy), aborting ...\n",
98 buddy_fn = calloc(SION_FILENAME_LENGTH+10,1);
99 if (buddy_fn == NULL) {
101 return(_sion_errorprint(SION_ID_NOT_VALID,_SION_ERROR_RETURN,
"_sion_paropen_generic_buddy: cannot allocate temporary memory of size %lu (buddy_fn), aborting ...\n",
102 (
unsigned long) SION_FILENAME_LENGTH+10));
106 sion_apidesc=sion_gendata->apidesc;
108 if (flags_store->mask&_SION_FMODE_WRITE) {
112 (fname,sion_gendata->filenumber);
113 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"call parallel open of %d files (current name %s)\n",
114 sion_gendata->numfiles, nfname));
116 &sion_gendata->numfiles, &sion_gendata->filenumber,
117 chunksize, fsblksize,
118 sion_gendata->lrank, sion_gendata->lsize, globalrank,
119 _SION_INTERNAL_FLAG_BUDDY_NORMAL, fileptr, sion_gendata, NULL);
120 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"leave parallel open of %d files in #tasks=%d sid=%d globalrank=%d\n", sion_gendata->numfiles,
121 sion_gendata->lsize, sid, sion_gendata->grank));
128 return(_sion_errorprint(SION_ID_NOT_VALID,_SION_ERROR_RETURN,
"_sion_paropen_generic_buddy: invalid sion_filedesc %d", sid));
132 sion_filedesc->
buddies=buddies;
134 capability=sion_apidesc->get_capability_cb(sion_gendata->comm_data_global);
138 buddyptr=&buddies[b];
141 buddyptr->buddy_send.commgroup=NULL; buddyptr->buddy_coll.commgroup=NULL;
142 rc=_sion_buddy_map(sion_gendata,capability,b+1,&buddyptr->buddy_send,&buddyptr->buddy_coll);
145 sprintf(buddy_fn,
"%s_BUDDY_%02d", fname,b);
148 for(pass=1;pass<=_SION_BW_SCHED_NUM_PASSES;pass++) {
150 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"BUDDYO pass=%d [grank=%2d] op=%d\n",pass,*globalrank,
151 _sion_buddy_bwsched(buddyptr->buddy_send.groupnum, sion_gendata->numfiles, pass)));
154 switch (_sion_buddy_bwsched(buddyptr->buddy_send.groupnum, sion_gendata->numfiles, pass)) {
155 case _SION_BW_SCHED_ACTIONA:
157 file_globalrank=-1*(*globalrank+1);
160 (buddy_fn,buddyptr->buddy_coll.filenum);
161 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"BUDDYO call open_one_file COLL [gendata: b=%d grank=%2d, group=%d of %d, lrank=%d of %d] fn=%s (%d)\n",
162 b,*globalrank, buddyptr->buddy_coll.groupnum,sion_gendata->numfiles,
163 buddyptr->buddy_coll.rank, buddyptr->buddy_coll.size,nfname,buddyptr->buddy_coll.filenum));
166 &sion_gendata->numfiles, &buddyptr->buddy_coll.filenum,
167 &file_chunksize, fsblksize,
168 buddyptr->buddy_coll.rank,buddyptr->buddy_coll.size, &file_globalrank,
169 _SION_INTERNAL_FLAG_BUDDY_COLL, NULL, sion_gendata,
173 case _SION_BW_SCHED_ACTIONB:
175 file_globalrank=*globalrank;
176 file_chunksize=*chunksize;
178 (buddy_fn,buddyptr->buddy_send.filenum);
179 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"BUDDYO call open_one_file SEND [gendata: b=%d grank=%2d, group=%d of %d, lrank=%d of %d] fn=%s \n",
180 b,*globalrank, buddyptr->buddy_send.groupnum,sion_gendata->numfiles,
181 buddyptr->buddy_send.rank, buddyptr->buddy_send.size,nfname));
184 &sion_gendata->numfiles, &buddyptr->buddy_send.filenum,
185 &file_chunksize, fsblksize,
186 buddyptr->buddy_send.rank,buddyptr->buddy_send.size, &file_globalrank,
187 _SION_INTERNAL_FLAG_BUDDY_SEND, NULL, sion_gendata,
192 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"BUDDYO no-op [grank=%2d]\n",*globalrank));
198 }
else if (flags_store->mask&_SION_FMODE_READ) {
199 int filefound, lgroup=*filenumber, root=0, task, tmpsize, group, masterfile_found;
200 sion_int32 filenumber, numfiles, lrank, lsize, file, file_lgroup, myrole, steprank;
201 sion_int32 helpint32=0, numsteps=0, numgroups=0, numreaderinthisfile;
202 sion_int32 *sion_tmpintfield1 = NULL;
205 int *stepvectorlist[MAXREADSTEPS];
206 int *stepvector, readercount=0, readyflag=0, datafound=0;
207 int stepcount_coll,stepcount_collreader,stepcount_reader,stepcount_reader_rank,stepcount_noreader,stepcount_noreader_rank, groupcounter;
208 int step,fit,newrank;
210 sion_int32 helpint7[7];
212 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"enter buddy open in read mode\n"));
214 if(sion_gendata->grank==0) {
215 tmpsize=1*sion_gendata->gsize;
216 sion_tmpintfield1 = (sion_int32 *) malloc(tmpsize *
sizeof(sion_int32));
217 if (sion_tmpintfield1 == NULL) {
220 return(_sion_errorprint(SION_ID_NOT_VALID,_SION_ERROR_RETURN,
"_sion_paropen_generic_buddy: cannot allocate temporary memory of size %lu (sion_tmpintfield), aborting ...\n",
221 (
unsigned long) tmpsize *
sizeof(sion_int32)));
227 for(b=-1; ( (b<buddylevel) && (!readyflag));b++) {
230 if(b==-1) sprintf(buddy_fn,
"%s", fname);
231 else sprintf(buddy_fn,
"%s_BUDDY_%02d", fname,b);
232 nfname=(sion_apidesc->get_multi_filename_cb?sion_apidesc->get_multi_filename_cb:
_sion_get_multi_filename)(buddy_fn,0);
233 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"BUDDY[%2d] start checking buddy level (file=%s)\n",b,nfname));
236 if(sion_gendata->lrank==0) {
239 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"check for existence of file %s --> file found\n",nfname));
245 sion_gendata->apidesc->gatherr_cb(&helpint32, sion_tmpintfield1, sion_gendata->comm_data_global, _SION_INT32, 1, 0);
246 if (sion_gendata->grank == 0) {
248 if (sion_tmpintfield1 == NULL) {
251 return(_sion_errorprint(SION_NOT_SUCCESS,_SION_ERROR_ABORT, DFUNCTION
": gatherr_cb returned sion_tmpintfield1 == NULL"));
253 for(task=0;task<sion_gendata->gsize;task++) {
254 if(sion_tmpintfield1[task]==1) root=task;
257 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"after scan master: root for this file is %d #tasks=%d\n",root,sion_gendata->gsize));
259 sion_gendata->apidesc->bcastr_cb(&helpint32, sion_gendata->comm_data_global, _SION_INT32, 1, 0); root=helpint32;
260 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"root for this file is %d\n",root));
264 _sion_generic_buddy_get_and_distribute_info_from_file(sion_gendata,nfname,root, &filenumber, &numfiles, &lrank, &lsize);
265 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"BUDDY[%d] after read mapping file=%d of %d, lrank=%d of %d\n",b,filenumber, numfiles, lrank, lsize));
268 filenumber=lrank=lsize=-1;
275 DPRINTFP((32, DFUNCTION, sion_gendata->grank,
"Info avail: b=%d numfiles=%d root=%d lgroup=%d\n",b,numfiles,root,lgroup));
278 for(file=0;( (file<numfiles) && (!readyflag) );file++) {
281 nfname=(sion_apidesc->get_multi_filename_cb?sion_apidesc->get_multi_filename_cb:
_sion_get_multi_filename)(buddy_fn,file);
283 if(sion_gendata->lrank==0) {
286 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"check for existence of file %s --> file found\n",nfname));
292 sion_gendata->apidesc->gatherr_cb(&helpint32, sion_tmpintfield1, sion_gendata->comm_data_global, _SION_INT32, 1, 0);
293 if (sion_gendata->grank == 0) {
295 for(task=0;task<sion_gendata->gsize;task++) {
296 if(sion_tmpintfield1[task]>-1) {
298 file_lgroup=sion_tmpintfield1[task];
301 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"after scan one file: root for this file is %d file_lgroup=%d\n",root,file_lgroup));
303 if (sion_gendata->grank == 0) helpint32=root;
304 sion_gendata->apidesc->bcastr_cb(&helpint32, sion_gendata->comm_data_global, _SION_INT32, 1, 0); root=helpint32;
305 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"root for this file is %d\n",root));
310 if(!masterfile_found) {
312 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"BUDDY[%d] befre read globalranks file=%d of %d, lrank=%d of %d\n",
313 b, filenumber, numfiles, lrank, lsize));
314 _sion_generic_buddy_get_and_distribute_info_from_one_file(sion_gendata, nfname, root, &filenumber, &numfiles, &lrank, &lsize);
315 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"BUDDY[%d] after read globalranks file=%d of %d, lrank=%d of %d\n",
316 b, filenumber, numfiles, lrank, lsize));
320 if (sion_gendata->grank == 0) helpint32=file_lgroup;
321 sion_gendata->apidesc->bcastr_cb(&helpint32, sion_gendata->comm_data_global, _SION_INT32, 1, 0); file_lgroup=helpint32;
322 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"file_lgroup for this file is %d\n",file_lgroup));
325 myrole=SION_ROLE_NONE;
326 if((filenumber==file) && (!datafound)) {
328 if(sion_gendata->grank==root) myrole=SION_ROLE_COLLECTOR_READER;
329 else myrole=SION_ROLE_READER;
331 if(sion_gendata->grank==root) myrole=SION_ROLE_COLLECTOR;
332 else if(file_lgroup==lgroup) myrole=SION_ROLE_NOREADER;
337 sion_gendata->apidesc->gatherr_cb(&helpint32, sion_tmpintfield1, sion_gendata->comm_data_global, _SION_INT32, 1, 0);
340 if(sion_gendata->grank == 0) {
341 numreaderinthisfile=0;
342 if (sion_tmpintfield1 == NULL) {
343 return(_sion_errorprint(SION_NOT_SUCCESS,_SION_ERROR_ABORT, DFUNCTION
": gatherr_cb returned sion_tmpintfield1 == NULL"));
345 for(task=0; ( (task<sion_gendata->gsize) ) ;task++)
346 if ( (sion_tmpintfield1[task]==SION_ROLE_COLLECTOR_READER) || (sion_tmpintfield1[task]==SION_ROLE_READER) ) numreaderinthisfile++;
348 sion_gendata->apidesc->bcastr_cb(&numreaderinthisfile, sion_gendata->comm_data_global, _SION_INT32, 1, 0);
350 if (numreaderinthisfile>0) {
352 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"myrole for file %s is %s\n",nfname,_sion_buddy_role_to_str(myrole)));
355 if (sion_gendata->grank == 0) {
358 for(step=0; ( (step<numsteps) && (fit==0) ) ;step++) {
359 stepvector=stepvectorlist[step];
361 for(task=0; ( (task<sion_gendata->gsize) && (fit==1) ) ;task++) {
362 if(sion_tmpintfield1[task]!=SION_ROLE_NONE) {
363 if(stepvector[task]!=SION_ROLE_NONE) fit=0;
367 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"BUDDY STEP assign: after scanning %d steps fit=%d\n",numsteps,fit));
370 stepvector = (
int *) malloc(sion_gendata->gsize *
sizeof(
int));
371 if (stepvector == NULL) {
372 return(_sion_errorprint(SION_ID_NOT_VALID,_SION_ERROR_RETURN,
"_sion_paropen_generic_buddy: cannot allocate temporary memory of size %lu (stepvector), aborting ...\n",
373 (
unsigned long) sion_gendata->gsize *
sizeof(
int)));
375 for(task=0; (task<sion_gendata->gsize);task++) stepvector[task]=SION_ROLE_NONE;
377 stepvectorlist[numsteps-1]=stepvector;
378 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"BUDDY STEP assign: allocating new step %d\n",numsteps-1));
381 stepcount_coll=stepcount_collreader=stepcount_reader=stepcount_noreader=0;
382 for(task=0; (task<sion_gendata->gsize);task++) {
383 if(sion_tmpintfield1[task]!=SION_ROLE_NONE) {
384 stepvector[task]=sion_tmpintfield1[task];
385 if (sion_tmpintfield1[task]==SION_ROLE_COLLECTOR) stepcount_coll++;
386 if (sion_tmpintfield1[task]==SION_ROLE_READER) {readercount++;stepcount_reader++;}
387 if (sion_tmpintfield1[task]==SION_ROLE_COLLECTOR_READER) {readercount++;stepcount_collreader++;}
388 if (sion_tmpintfield1[task]==SION_ROLE_NOREADER) stepcount_noreader++;
393 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"BUDDY STEP assign: group assigned to step %d (tasks with no data: %d)\n",numsteps-1,sion_gendata->gsize-readercount));
394 if(sion_gendata->gsize-readercount==0) readyflag=1;
397 stepcount_noreader_rank = stepcount_coll;
398 stepcount_reader_rank = stepcount_noreader_rank+stepcount_noreader;
399 for(task=0; (task<sion_gendata->gsize);task++) {
402 if(sion_tmpintfield1[task]==SION_ROLE_NONE) newrank=-1;
403 if(sion_tmpintfield1[task]==SION_ROLE_COLLECTOR) newrank=0;
404 if(sion_tmpintfield1[task]==SION_ROLE_COLLECTOR_READER) newrank=stepcount_reader_rank++;
405 if(sion_tmpintfield1[task]==SION_ROLE_NOREADER) newrank=stepcount_noreader_rank++;
406 if(sion_tmpintfield1[task]==SION_ROLE_READER) newrank=stepcount_reader_rank++;
407 sion_tmpintfield1[task]=newrank;
414 sion_gendata->apidesc->scatterr_cb(sion_tmpintfield1, &steprank, sion_gendata->comm_data_global, _SION_INT32, 1, 0);
418 if (sion_gendata->grank == 0) {
421 helpint7[1]=readyflag;
422 helpint7[2]=stepcount_coll+stepcount_collreader+stepcount_reader+stepcount_noreader;
423 helpint7[3]=stepcount_collreader+stepcount_reader;
424 helpint7[4]=stepcount_coll+stepcount_noreader;
425 helpint7[5]=stepcount_coll+stepcount_noreader+stepcount_collreader+stepcount_reader-1;
426 helpint7[6]=groupcounter++;
427 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"define group #%d (step=%d, readyflag=%d, size=%d, collsize=%d, from=%d, to=%d) \n",
428 helpint7[6], helpint7[0], helpint7[1], helpint7[2], helpint7[3], helpint7[4], helpint7[5]));
430 sion_gendata->apidesc->bcastr_cb(helpint7, sion_gendata->comm_data_global, _SION_INT32, 7, 0);
431 readyflag=helpint7[1];
434 if(myrole!=SION_ROLE_NONE) {
436 if (buddy_info == NULL) {
438 return(_sion_errorprint(SION_ID_NOT_VALID,_SION_ERROR_RETURN,
"_sion_paropen_generic_buddy: cannot allocate temporary memory of size %lu (buddy_info), aborting ...\n",
441 buddies->groups[buddies->numgroups]=buddy_info; buddies->numgroups++;
442 buddy_info->groupid=helpint7[6];
443 buddy_info->stepnum=helpint7[0];
444 buddy_info->rank=steprank;
445 buddy_info->size=helpint7[2];
446 buddy_info->commgroup=NULL;
447 buddy_info->collsize=helpint7[3];
448 buddy_info->from_index=helpint7[4];
449 buddy_info->to_index=helpint7[5];
450 buddy_info->myrole=myrole;
452 if( (myrole==SION_ROLE_READER) || (myrole==SION_ROLE_COLLECTOR_READER)) {
453 buddy_info->filelrank=lrank;
455 buddy_info->filelrank=-1;
457 buddy_info->filenum=file;
458 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"create group #%d (step=%d, rank=%d, size=%d, collsize=%d, from=%2d, to=%2d) (bnum=%2d file# %2d) with role %s \n",
459 buddy_info->groupid, buddy_info->stepnum, buddy_info->rank, buddy_info->size, buddy_info->collsize,
460 buddy_info->from_index, buddy_info->to_index, buddy_info->bnum, buddy_info->filenum, _sion_buddy_role_to_str(buddy_info->myrole)));
472 return(_sion_errorprint(SION_ID_NOT_VALID,_SION_ERROR_RETURN,
"_sion_paropen_generic_buddy: cannot open file, not all files available"));
476 sion_gendata->apidesc->bcastr_cb(&numsteps, sion_gendata->comm_data_global, _SION_INT32, 1, 0);
477 buddies->numsteps=numsteps;
480 if (sion_gendata->grank == 0) numgroups=groupcounter;
481 sion_gendata->apidesc->bcastr_cb(&numgroups, sion_gendata->comm_data_global, _SION_INT32, 1, 0);
484 for(step=0;step<numsteps;step++) {
487 for(b=0; ( (b<buddies->numgroups) && (group==-1) ); b++ ) {
489 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"check group #%d of %d in stepnum=%d step=%d\n",b, buddies->numgroups, buddies->groups[b]->stepnum,step));
490 if (buddies->groups[b]->stepnum==step) group=b;
495 buddy_info=buddies->groups[group];
496 rc=sion_gendata->apidesc->create_lcg_cb(&buddy_info->commgroup,sion_gendata->comm_data_global,
497 sion_gendata->grank,sion_gendata->gsize,
498 buddy_info->rank,buddy_info->size,
499 buddy_info->groupid,numgroups);
500 DPRINTFP((256, DFUNCTION, _SION_DEFAULT_RANK,
"COMM_created active [grank=%2d of %2d, group=%2d of %2d, lrank=%2d of %2d]\n",
501 sion_gendata->grank,sion_gendata->gsize, buddy_info->groupid,numgroups, buddy_info->rank,buddy_info->size));
504 void *dummycommgroup=NULL;
505 rc=sion_gendata->apidesc->create_lcg_cb(&dummycommgroup,sion_gendata->comm_data_global,
506 sion_gendata->grank,sion_gendata->gsize,
509 DPRINTFP((256, DFUNCTION, _SION_DEFAULT_RANK,
"COMM_created dummy [grank=%2d of %2d, group=%2d of %2d, lrank=%2d of %2d]\n",
510 sion_gendata->grank,sion_gendata->gsize,
518 buddy_info=buddies->groups[group];
519 buddies->currentgroup=group;
521 if(buddy_info->bnum==-1) sprintf(buddy_fn,
"%s", fname);
522 else sprintf(buddy_fn,
"%s_BUDDY_%02d", fname,buddy_info->bnum);
523 nfname=(sion_apidesc->get_multi_filename_cb?sion_apidesc->get_multi_filename_cb:
_sion_get_multi_filename)(buddy_fn,buddy_info->filenum);
525 DPRINTFP((256, DFUNCTION, _SION_DEFAULT_RANK,
"open file for group #%d %s [step=%d grank=%2d of %2d, lrank=%2d of %2d filelrank=%2d]\n",
526 buddy_info->groupid,nfname,buddy_info->stepnum,
527 sion_gendata->grank,sion_gendata->gsize, buddy_info->rank,buddy_info->size,buddy_info->filelrank));
530 if( (buddy_info->myrole == SION_ROLE_READER)
531 || (buddy_info->myrole == SION_ROLE_COLLECTOR_READER)) {
534 &sion_gendata->numfiles, &buddy_info->filenum,
535 chunksize, fsblksize,
536 buddy_info->rank, buddy_info->size, &sion_gendata->grank,
537 _SION_INTERNAL_FLAG_BUDDY_READ, NULL, sion_gendata,
544 return(_sion_errorprint(SION_ID_NOT_VALID,_SION_ERROR_RETURN,
"_sion_paropen_generic_buddy: invalid sion_filedesc %d", sid));
548 sion_filedesc->
buddies=buddies;
551 sion_int64 dummy_chunksize;
552 sion_int32 dummy_fsblksize;
555 &sion_gendata->numfiles, &buddy_info->filenum,
556 &dummy_chunksize, &dummy_fsblksize,
557 buddy_info->rank, buddy_info->size, &sion_gendata->grank,
558 _SION_INTERNAL_FLAG_BUDDY_READ, NULL, sion_gendata,
563 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"BUDDYO afteropen [grank=%2d step=%d] rc=%d\n",sion_gendata->grank,step,rc));
568 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"BUDDYO no-op [grank=%2d]\n",sion_gendata->grank));
572 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"BUDDYO beforegbarrier [grank=%2d]\n",sion_gendata->grank));
573 sion_apidesc->barrier_cb(sion_gendata->comm_data_global);
574 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"BUDDYO aftergbarrier [grank=%2d]\n",sion_gendata->grank));
583 return(_sion_errorprint(SION_ID_NOT_VALID,_SION_ERROR_RETURN,
"_sion_paropen_generic_buddy: unknown file mode"));
588 sion_apidesc->barrier_cb(sion_gendata->comm_data_global);
607 #define DFUNCTION "_sion_parclose_generic_buddy" 608 int _sion_parclose_generic_buddy(
int sid,
618 int b_mapping_size=0,group,step;
619 sion_int32 *b_mapping=NULL;
626 return(_sion_errorprint(SION_ID_NOT_VALID,_SION_ERROR_RETURN,
"_sion_parclose_generic_buddy: invalid sion_filedesc %d", sid));
629 buddies=sion_filedesc->
buddies;
636 buddyptr=&buddies[b];
640 _sion_generic_collect_mapping_buddy(buddyptr, sion_gendata, &b_mapping_size, &b_mapping);
644 for(pass=1;pass<=_SION_BW_SCHED_NUM_PASSES;pass++) {
646 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"BUDDYC pass=%d [grank=%2d] op=%d\n",pass,sion_gendata->grank,
647 _sion_buddy_bwsched(buddyptr->buddy_send.groupnum, sion_gendata->numfiles, pass)));
649 switch (_sion_buddy_bwsched(buddyptr->buddy_send.groupnum, sion_gendata->numfiles, pass)) {
650 case _SION_BW_SCHED_ACTIONA:
651 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"BUDDYc call parallel close COLL [gendata: grank=%2d, group=%d of %d, lrank=%d of %d]\n",
652 sion_gendata->grank, buddyptr->buddy_coll.groupnum,sion_gendata->numfiles,
653 buddyptr->buddy_coll.rank, buddyptr->buddy_coll.size));
655 b_mapping_size, b_mapping, _SION_INTERNAL_FLAG_BUDDY_COLL, sion_gendata, buddyptr );
657 case _SION_BW_SCHED_ACTIONB:
658 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"BUDDYc call parallel close SEND [gendata: grank=%2d, group=%d of %d, lrank=%d of %d]\n",
659 sion_gendata->grank, buddyptr->buddy_send.groupnum,sion_gendata->numfiles,
660 buddyptr->buddy_send.rank, buddyptr->buddy_send.size));
662 0, NULL, _SION_INTERNAL_FLAG_BUDDY_SEND, sion_gendata, buddyptr );
665 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"BUDDYC no-op [grank=%2d]\n",sion_gendata->grank));
672 _SION_SAFE_FREE(b_mapping, NULL);
673 _SION_SAFE_FREE(buddies, NULL);
677 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"call parallel close of %d files, sid=%d\n",
678 sion_gendata->numfiles, sid));
680 sion_gendata, NULL );
681 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"leave parallel close of %d files in #tasks=%d globalrank=%d\n", sion_gendata->numfiles,
682 sion_gendata->lsize, sion_gendata->grank));
687 for(step=0;step<buddies->numsteps;step++) {
690 for(b=0; ( (b<buddies->numgroups) && (group==-1) ); b++ ) {
692 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"check group #%d of %d %d==%d %x\n",b, buddies->numgroups, buddies->groups[b]->stepnum,step, buddies->groups[b]));
693 if (buddies->groups[b]->stepnum==step) group=b;
695 buddy_info=buddies->groups[group];
699 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"call parallel close of %d files, sid=%d\n",
700 sion_gendata->numfiles, sid));
701 buddies->currentgroup=group;
702 rc =
_sion_parclose_generic( buddy_info->sid, buddy_info->rank,buddy_info->size, -1, NULL, _SION_INTERNAL_FLAG_BUDDY_READ,
703 sion_gendata, buddies );
704 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"leave parallel close of %d files in #tasks=%d globalrank=%d\n", sion_gendata->numfiles,
705 sion_gendata->lsize, sion_gendata->grank));
711 return(_sion_errorprint(SION_ID_NOT_VALID,_SION_ERROR_RETURN,
"_sion_parclose_generic_buddy: unknown file mode"));
728 #define DFUNCTION "_sion_coll_fwrite_buddy" 729 int _sion_coll_fwrite_buddy(
const void *data,
735 _sion_filedesc *sion_filedesc,*sion_filedesc_coll,*sion_filedesc_send;
738 sion_int64 spec[2], ownnewposition;
739 int b, pass, collector=0;
741 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"enter parallel write buddy\n"));
745 return(_sion_errorprint(SION_ID_NOT_VALID,_SION_ERROR_RETURN,
"_sion_coll_fwrite_buddy: invalid sion_filedesc %d", sid));
747 sion_apidesc=sion_gendata->apidesc;
748 buddies=sion_filedesc->
buddies;
752 buddyptr=&buddies[b];
756 return(_sion_errorprint(SION_ID_NOT_VALID,_SION_ERROR_RETURN,
"_sion_coll_fwrite_buddy: invalid sion_filedesc %d", buddyptr->buddy_coll.sid));
759 return(_sion_errorprint(SION_ID_NOT_VALID,_SION_ERROR_RETURN,
"_sion_coll_fwrite_buddy: invalid sion_filedesc %d", buddyptr->buddy_send.sid));
764 for(pass=1;pass<=_SION_BW_SCHED_NUM_PASSES;pass++) {
766 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"BUDDYw pass=%d [grank=%2d] op=%d\n",pass,sion_gendata->grank,
767 _sion_buddy_bwsched(buddyptr->buddy_send.groupnum, sion_gendata->numfiles, pass)));
769 switch (_sion_buddy_bwsched(buddyptr->buddy_send.groupnum, sion_gendata->numfiles, pass)) {
770 case _SION_BW_SCHED_ACTIONA:
771 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"BUDDYw call parallel write COLL [gendata: grank=%2d, group=%d of %d, lrank=%d of %d] %d..%d\n",
772 sion_gendata->grank, buddyptr->buddy_coll.groupnum,sion_gendata->numfiles,
773 buddyptr->buddy_coll.rank, buddyptr->buddy_coll.size,buddyptr->buddy_coll.from_index,buddyptr->buddy_coll.to_index));
775 if(sion_filedesc_coll->
rank == collector) {
776 ownnewposition=sion_filedesc_coll->
currentpos;
777 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"BUDDYw call parallel write COLL currentpos=%d\n",(
int) sion_filedesc_coll->
currentpos));
779 rc = sion_apidesc->gather_execute_cb(data,spec,2, sion_filedesc_coll->
fsblksize,
780 buddyptr->buddy_coll.commgroup,collector,buddyptr->buddy_coll.from_index,buddyptr->buddy_coll.to_index,
781 buddyptr->buddy_coll.sid, _sion_generic_collective_process_write);
782 if(sion_filedesc_coll->
rank == collector) {
785 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"BUDDYw call parallel write COLL currentpos=%d\n",(
int) sion_filedesc_coll->
currentpos));
788 case _SION_BW_SCHED_ACTIONB:
789 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"BUDDYw call parallel write SEND [gendata: grank=%2d, group=%d of %d, lrank=%d of %d] %d..%d\n",
790 sion_gendata->grank, buddyptr->buddy_send.groupnum,sion_gendata->numfiles,
791 buddyptr->buddy_send.rank, buddyptr->buddy_send.size,buddyptr->buddy_send.from_index,buddyptr->buddy_send.to_index));
795 _sion_errorprint(SION_SIZE_NOT_VALID,_SION_ERROR_RETURN,
"could not ensure free space for this block, returning %d ...\n", sid);
801 rc = sion_apidesc->gather_execute_cb(data,spec,2, sion_filedesc_send->
fsblksize,
802 buddyptr->buddy_send.commgroup,collector,buddyptr->buddy_send.from_index,buddyptr->buddy_send.to_index,
803 buddyptr->buddy_send.sid, _sion_generic_collective_process_write);
807 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"BUDDYw no-op [grank=%2d]\n",sion_gendata->grank));
816 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"leave parallel write buddy\n"));
833 #define DFUNCTION "_sion_coll_fread_buddy" 834 int _sion_coll_fread_buddy(
void *data,
size_t size,
size_t nitems,
int sid) {
838 sion_int64 bread=-1, spec[2], ownnewposition, items_read = 0;
839 int rc_own=SION_STD_SUCCESS,rc_cb=SION_STD_SUCCESS;
840 int collector, firstsender, lastsender, step, b, group;
846 return(_sion_errorprint(SION_SIZE_NOT_VALID,_SION_ERROR_RETURN,
"_sion_coll_fread_buddy: invalid sion_filedesc %d", sid));
848 DPRINTFP((4, DFUNCTION, _SION_DEFAULT_RANK,
"enter usecoll=%d collector=%d collsize=%d (%d tasks, %d files)\n",
851 sion_gendata=sion_filedesc->
dataptr;
852 sion_apidesc=sion_gendata->apidesc;
853 buddies=sion_filedesc->
buddies;
860 return(_sion_errorprint(SION_SIZE_NOT_VALID,_SION_ERROR_RETURN,
"_sion_coll_fread_buddy: collsize=%d <= 0, returning ...\n",
866 for(step=0;step<buddies->numsteps;step++) {
869 for(b=0; ( (b<buddies->numgroups) && (group==-1) ); b++ ) {
871 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"check group #%d of %d %d==%d %x\n",b, buddies->numgroups, buddies->groups[b]->stepnum,step, buddies->groups[b]));
872 if (buddies->groups[b]->stepnum==step) group=b;
878 buddy_info=buddies->groups[group];
881 return(_sion_errorprint(SION_SIZE_NOT_VALID,_SION_ERROR_RETURN,
"_sion_coll_fread_buddy: invalid sion_filedesc %d", buddy_info->sid));
886 firstsender=buddy_info->from_index;
887 lastsender=buddy_info->to_index;
891 if( (buddy_info->myrole == SION_ROLE_READER)
892 || (buddy_info->myrole == SION_ROLE_COLLECTOR_READER)) {
897 _sion_errorprint(SION_SIZE_NOT_VALID,_SION_ERROR_RETURN,
"early eof found for this block, returning %d ...\n", buddy_info->sid);
914 if( (buddy_info->myrole == SION_ROLE_COLLECTOR_READER) ) {
915 rc_own=_sion_generic_collective_process_read(data,spec,buddy_info->sid);
919 if( (buddy_info->myrole == SION_ROLE_COLLECTOR)
920 || (buddy_info->myrole == SION_ROLE_COLLECTOR_READER)) {
926 if(!sion_apidesc->execute_scatter_cb ) {
927 return(_sion_errorprint(SION_SIZE_NOT_VALID,_SION_ERROR_RETURN,
928 "_sion_coll_fread_buddy: API %s not correctly initalized, collective I/O calls missing, aborting",sion_apidesc->name));
930 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"call execute_gather #%d (step=%d, rank=%d, size=%d, collsize=%d, from=%2d, to=%2d) (bnum=%2d file# %2d) with role %s \n",
931 buddy_info->groupid, buddy_info->stepnum, buddy_info->rank, buddy_info->size, buddy_info->collsize,
932 buddy_info->from_index, buddy_info->to_index, buddy_info->bnum, buddy_info->filenum, _sion_buddy_role_to_str(buddy_info->myrole)));
934 rc_cb=sion_apidesc->execute_scatter_cb(data,spec,2, b_sion_filedesc->
fsblksize,
935 buddy_info->commgroup,collector,firstsender,lastsender,buddy_info->sid,
936 _sion_generic_collective_process_read);
941 if( (buddy_info->myrole == SION_ROLE_COLLECTOR)
942 || (buddy_info->myrole == SION_ROLE_COLLECTOR_READER)) {
948 if(buddy_info->myrole == SION_ROLE_READER) {
955 if( (buddy_info->myrole == SION_ROLE_READER)
956 || (buddy_info->myrole == SION_ROLE_COLLECTOR_READER)) {
957 if( (rc_own == SION_STD_SUCCESS) && (rc_cb == SION_STD_SUCCESS) ) {
962 items_read = size ? bread / size : 0;
968 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"BUDDYO beforegbarrier [grank=%2d]\n",sion_gendata->grank));
969 sion_apidesc->barrier_cb(sion_gendata->comm_data_global);
970 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"BUDDYO aftergbarrier [grank=%2d]\n",sion_gendata->grank));
974 DPRINTFP((4, DFUNCTION, _SION_DEFAULT_RANK,
"leave usecoll=%d collector=%d collsize=%d (%d tasks, %d files) rc=%d\n",
982 #define DFUNCTION "_sion_buddy_map" 990 int tmpsize,t,g, g_map, p, p_map, n, n_map, orig, distance,distance_1step,pass;
991 int *size_per_group = NULL, *group_map_orig_to_new, *group_map_new_to_orig, mynewgroupnr;
992 sion_int32 helpint2[2];
993 sion_int32 *tasktogroup = NULL;
994 sion_int32 *tmpintfield1 = NULL;
995 int comm_send_rank, comm_send_size, comm_send_grpnum;
996 int comm_coll_rank, comm_coll_size, comm_coll_grpnum;
997 void *dummycommgroup=NULL;
999 DPRINTFP((256, DFUNCTION, _SION_DEFAULT_RANK,
"buddy_idx=%d capability=%d [gendata: grank=%d of %d, group=%d of %d, lrank=%d of %d]\n",
1000 buddy_idx, capability,
1001 sion_gendata->grank, sion_gendata->gsize,
1002 sion_gendata->filenumber, sion_gendata->numfiles,
1003 sion_gendata->lrank, sion_gendata->lsize));
1009 if(sion_gendata->grank==0) {
1011 tmpsize=1*sion_gendata->gsize;
1012 tasktogroup = (sion_int32 *) malloc(tmpsize *
sizeof(sion_int32));
1013 if (tasktogroup == NULL) {
1014 return(_sion_errorprint(SION_ID_NOT_VALID,_SION_ERROR_RETURN,
"_sion_buddy_map: cannot allocate temporary memory of size %lu (tasktogroup), aborting ...\n",
1015 (
unsigned long) tmpsize *
sizeof(sion_int32)));
1018 tmpsize=2*sion_gendata->gsize;
1019 tmpintfield1 = (sion_int32 *) malloc(tmpsize *
sizeof(sion_int32));
1020 if (tmpintfield1 == NULL) {
1022 return(_sion_errorprint(SION_ID_NOT_VALID,_SION_ERROR_RETURN,
"_sion_buddy_map: cannot allocate temporary memory of size %lu (tmpintfield1), aborting ...\n",
1023 (
unsigned long) tmpsize *
sizeof(sion_int32)));
1026 size_per_group = (
int *) malloc(sion_gendata->numfiles *
sizeof(
int));
1027 if (size_per_group == NULL) {
1030 return(_sion_errorprint(SION_ID_NOT_VALID,_SION_ERROR_RETURN,
"_sion_buddy_map: cannot allocate temporary memory of size %lu (size_per_group), aborting ...\n",
1031 (
unsigned long) sion_gendata->numfiles *
sizeof(
int)));
1037 group_map_orig_to_new = (
int *) malloc(sion_gendata->numfiles *
sizeof(
int));
1038 if (group_map_orig_to_new == NULL) {
1040 _SION_SAFE_FREE(size_per_group, NULL);
1042 return(_sion_errorprint(SION_ID_NOT_VALID,_SION_ERROR_RETURN,
"_sion_buddy_map: cannot allocate temporary memory of size %lu (group_map_orig_to_new), aborting ...\n",
1043 (
unsigned long) sion_gendata->numfiles *
sizeof(
int)));
1046 group_map_new_to_orig = (
int *) malloc(sion_gendata->numfiles *
sizeof(
int));
1047 if (group_map_new_to_orig == NULL) {
1048 _SION_SAFE_FREE(size_per_group, NULL);
1049 free(group_map_orig_to_new);
1052 return(_sion_errorprint(SION_ID_NOT_VALID,_SION_ERROR_RETURN,
"_sion_buddy_map: cannot allocate temporary memory of size %lu (group_map_new_to_orig), aborting ...\n",
1053 (
unsigned long) sion_gendata->numfiles *
sizeof(
int)));
1057 helpint2[0]=sion_gendata->filenumber;
1058 sion_gendata->apidesc->gatherr_cb(helpint2, tasktogroup, sion_gendata->comm_data_global, _SION_INT32,1,0);
1062 helpint2[0]=sion_gendata->lsize;
1063 helpint2[1]=capability;
1064 sion_gendata->apidesc->gatherr_cb(helpint2, tmpintfield1, sion_gendata->comm_data_global, _SION_INT32,2,0);
1065 if(sion_gendata->grank==0) {
1066 for(t=0;t<sion_gendata->gsize;t++) {
1067 DPRINTFP((256, DFUNCTION, _SION_DEFAULT_RANK,
"task-map --> t=%2d grpnr=%2d lsize=%2d capability=%d\n",
1068 t,tasktogroup[t],tmpintfield1[2*t+0],tmpintfield1[2*t+1]));
1074 if(sion_gendata->grank==0) {
1075 for(t=0;t<sion_gendata->gsize;t++) {
1076 size_per_group[tasktogroup[t]] = tmpintfield1[2*t+0];
1080 for(g=0;g<sion_gendata->numfiles;g++) {
1081 orig=(g*distance) % sion_gendata->numfiles;
1082 group_map_orig_to_new[orig]=g;
1083 group_map_new_to_orig[g]=orig;
1086 for(g=0;g<sion_gendata->numfiles;g++) {
1087 DPRINTFP((256, DFUNCTION, _SION_DEFAULT_RANK,
"grp-map --> g=%2d->%2d\n",g,group_map_orig_to_new[g]));
1092 if(sion_gendata->grank==0) {
1093 for(t=0;t<sion_gendata->gsize;t++) {
1095 g_map=group_map_orig_to_new[g];
1096 p_map=(g_map-distance_1step + sion_gendata->numfiles) % sion_gendata->numfiles;
1097 p=group_map_new_to_orig[p_map];
1098 tmpintfield1[2*t+0] = g_map;
1099 tmpintfield1[2*t+1] = size_per_group[p];
1100 DPRINTFP((256, DFUNCTION, _SION_DEFAULT_RANK,
"comm-send --> t=%2d grpnr=%2d grpmap=%2d prevgr=%2d prevgrmap=%2d prev_lsize=%2d\n",
1101 t, g, tmpintfield1[2*t+0], p, p_map, tmpintfield1[2*t+1]));
1107 sion_gendata->apidesc->scatterr_cb(tmpintfield1, helpint2, sion_gendata->comm_data_global, _SION_INT32,2,0);
1110 mynewgroupnr=comm_send_grpnum=helpint2[0];
1111 buddy_send->rank=comm_send_rank=helpint2[1]+sion_gendata->lrank;
1112 buddy_send->size=comm_send_size=helpint2[1]+sion_gendata->lsize;
1113 buddy_send->collsize=sion_gendata->lsize;
1114 buddy_send->groupnum=mynewgroupnr;
1115 buddy_send->filenum=group_map_new_to_orig[mynewgroupnr];
1116 buddy_send->from_index=helpint2[1];
1117 buddy_send->to_index=helpint2[1]+sion_gendata->lsize-1;
1118 DPRINTFP((256, DFUNCTION, _SION_DEFAULT_RANK,
"COMM_SEND d=%d grpnum=%2d lrank=%2d lsize=%2d mynewgroupnr=%2d collsize=%2d %d..%d fnum=%d\n",
1119 distance,comm_send_grpnum,comm_send_rank,comm_send_size,mynewgroupnr,
1120 buddy_send->collsize,buddy_send->from_index,buddy_send->to_index,
1121 buddy_send->filenum));
1124 if(sion_gendata->grank==0) {
1125 for(t=0;t<sion_gendata->gsize;t++) {
1127 g_map=group_map_orig_to_new[g];
1128 n_map=(g_map+distance_1step) % sion_gendata->numfiles;
1129 n=group_map_new_to_orig[n_map];
1130 tmpintfield1[2*t+0] = n_map;
1131 tmpintfield1[2*t+1] = size_per_group[n];
1132 DPRINTFP((256, DFUNCTION, _SION_DEFAULT_RANK,
"comm-coll --> t=%2d grpnr=%2d grpmap=%2d nextgr=%2d nextgrmap=%2d next_lsize=%2d\n",
1133 t, g, g_map, n, tmpintfield1[2*t+0], tmpintfield1[2*t+1]));
1139 sion_gendata->apidesc->scatterr_cb(tmpintfield1, helpint2, sion_gendata->comm_data_global, _SION_INT32,2,0);
1142 comm_coll_grpnum=helpint2[0];
1143 buddy_coll->rank=comm_coll_rank=sion_gendata->lrank;
1144 buddy_coll->size=comm_coll_size=sion_gendata->lsize+helpint2[1];
1145 buddy_coll->collsize=helpint2[1];
1146 buddy_coll->groupnum=comm_coll_grpnum;
1147 buddy_coll->filenum=group_map_new_to_orig[comm_coll_grpnum];
1148 buddy_coll->from_index=sion_gendata->lsize;
1149 buddy_coll->to_index=sion_gendata->lsize+helpint2[1]-1;
1150 DPRINTFP((256, DFUNCTION, _SION_DEFAULT_RANK,
"COMM_COLL d=%d grpnum=%2d lrank=%2d lsize=%2d collsize=%2d %d..%d fnum=%d\n",
1151 distance,comm_coll_grpnum,comm_coll_rank,comm_coll_size,
1152 buddy_coll->collsize,buddy_coll->from_index,buddy_coll->to_index,
1153 buddy_coll->filenum));
1157 for(pass=1;pass<=_SION_BW_SCHED_NUM_PASSES;pass++) {
1158 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"pass=%d [grank=%2d] op=%d\n",pass,sion_gendata->grank,
1159 _sion_buddy_bwsched(mynewgroupnr, sion_gendata->numfiles, pass)));
1161 switch (_sion_buddy_bwsched(mynewgroupnr, sion_gendata->numfiles, pass)) {
1162 case _SION_BW_SCHED_ACTIONA:
1163 rc=sion_gendata->apidesc->create_lcg_cb(&buddy_coll->commgroup,sion_gendata->comm_data_global,
1164 sion_gendata->grank,sion_gendata->gsize,
1165 comm_coll_rank,comm_coll_size,
1166 comm_coll_grpnum,sion_gendata->numfiles);
1167 DPRINTFP((256, DFUNCTION, _SION_DEFAULT_RANK,
"COMM_created comm_coll (d=%1d, p=%1d) [gendata: mygrp=%2d grank=%2d of %2d, group=%2d of %2d, lrank=%2d of %2d]\n",
1168 distance,pass,mynewgroupnr,sion_gendata->grank,sion_gendata->gsize,
1169 comm_coll_grpnum,sion_gendata->numfiles,
1170 comm_coll_rank,comm_coll_size ));
1172 case _SION_BW_SCHED_ACTIONB:
1173 rc=sion_gendata->apidesc->create_lcg_cb(&buddy_send->commgroup,sion_gendata->comm_data_global,
1174 sion_gendata->grank,sion_gendata->gsize,
1175 comm_send_rank,comm_send_size,
1176 comm_send_grpnum,sion_gendata->numfiles);
1177 DPRINTFP((256, DFUNCTION, _SION_DEFAULT_RANK,
"COMM_created comm_send (d=%1d, p=%1d) [gendata: mygrp=%2d grank=%2d of %2d, group=%2d of %2d, lrank=%2d of %2d]\n",
1178 distance,pass,mynewgroupnr,sion_gendata->grank,sion_gendata->gsize,
1179 comm_send_grpnum,sion_gendata->numfiles,
1180 comm_send_rank,comm_send_size ));
1182 case _SION_BW_SCHED_NOACTION:
1183 rc=sion_gendata->apidesc->create_lcg_cb(&dummycommgroup,sion_gendata->comm_data_global,
1184 sion_gendata->grank,sion_gendata->gsize,
1186 -1,sion_gendata->numfiles);
1187 DPRINTFP((256, DFUNCTION, _SION_DEFAULT_RANK,
"COMM_created dummy (d=%1d, p=%1d) [gendata: mygrp=%2d grank=%2d of %2d, group=%2d of %2d, lrank=%2d of %2d]\n",
1188 distance,pass,mynewgroupnr,sion_gendata->grank,sion_gendata->gsize,
1189 -1,sion_gendata->numfiles,0,1 ));
1197 if(sion_gendata->grank==0) {
1200 free(size_per_group);
1202 free(group_map_new_to_orig);
1203 free(group_map_orig_to_new);
1210 #define DFUNCTION "_sion_buddy_bwsched" 1211 int _sion_buddy_bwsched(
int groupnr,
int numgroups,
int pass) {
1212 int res=_SION_BW_SCHED_NOACTION;
1213 if(numgroups%2==0) {
1215 if(pass==1) res=_SION_BW_SCHED_ACTIONA;
1216 if(pass==2) res=_SION_BW_SCHED_ACTIONB;
1218 if(pass==1) res=_SION_BW_SCHED_ACTIONB;
1219 if(pass==2) res=_SION_BW_SCHED_ACTIONA;
1223 if((pass==1) && (groupnr != (numgroups-1))) res=_SION_BW_SCHED_ACTIONA;
1224 if((pass==2) && (groupnr != 0)) res=_SION_BW_SCHED_ACTIONB;
1225 if((pass==3) && (groupnr == 0)) res=_SION_BW_SCHED_ACTIONB;
1226 if((pass==3) && (groupnr == (numgroups-1))) res=_SION_BW_SCHED_ACTIONA;
1228 if(pass==1) res=_SION_BW_SCHED_ACTIONB;
1229 if(pass==2) res=_SION_BW_SCHED_ACTIONA;
1240 #define DFUNCTION "_sion_generic_collect_mapping" 1244 sion_int32 **mapping) {
1245 int rc=SION_SUCCESS;
1248 sion_int32 lpos[2], *receivemap=NULL, iamreceiver, receiver = -1;
1250 sion_apidesc=sion_gendata->apidesc;
1252 *mapping = NULL; *mapping_size = 0;
1256 if((buddyptr->buddy_coll.groupnum==0) && (buddyptr->buddy_coll.rank==0)) {
1258 *mapping_size=sion_gendata->gsize;
1259 *mapping = (sion_int32 *) malloc(*mapping_size * 2 *
sizeof(sion_int32));
1260 if (*mapping == NULL) {
1261 return(_sion_errorprint(SION_ID_NOT_VALID,_SION_ERROR_RETURN,
"_sion_generic_collect_mapping_buddy: Cannot allocate memory for mapping"));
1266 if(sion_gendata->grank==0) {
1267 receivemap = (sion_int32 *) malloc(sion_gendata->gsize *
sizeof(sion_int32));
1268 if (receivemap == NULL) {
1269 return(_sion_errorprint(SION_ID_NOT_VALID,_SION_ERROR_RETURN,
"_sion_generic_collect_mapping_buddy: Cannot allocate memory for receivemap"));
1273 if((buddyptr->buddy_coll.filenum==0) && (buddyptr->buddy_coll.rank==0)) iamreceiver=sion_gendata->grank;
1274 else iamreceiver=-1;
1275 sion_apidesc->gatherr_cb(&iamreceiver, receivemap, sion_gendata->comm_data_global, _SION_INT32, 1, 0);
1276 if(sion_gendata->grank==0) {
1277 for(t=0;t<sion_gendata->gsize;t++) {
1278 if(receivemap[t]>=0) {
1279 receiver=receivemap[t];
1283 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"receiver of mapping grank=%d\n", receiver));
1285 sion_apidesc->bcastr_cb(&receiver, sion_gendata->comm_data_global, _SION_INT32, 1, 0);
1288 lpos[0] = buddyptr->buddy_send.filenum;
1289 lpos[1] = buddyptr->buddy_send.rank;
1290 sion_apidesc->gatherr_cb(&lpos, *mapping, sion_gendata->comm_data_global, _SION_INT32, 2, receiver);
1292 if(receivemap!=NULL) free(receivemap);
1299 #define DFUNCTION "_sion_generic_buddy_get_and_distribute_info_from_file" 1300 int _sion_generic_buddy_get_and_distribute_info_from_file(
_sion_generic_gendata *sion_gendata,
char *fname,
int root,
1301 sion_int32 *filenumber, sion_int32 *numfiles,
1302 sion_int32 *lrank, sion_int32 *lsize) {
1304 int sid = -1, ntasks, nfiles, t;
1305 int rc = SION_SUCCESS;
1307 sion_int32 fsblksize;
1308 int *tasksinfile = NULL;
1309 int mapping_size = -1;
1310 sion_int32 *mapping = NULL;
1314 sion_apidesc=sion_gendata->apidesc;
1316 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"entering function sion_gendata->grank=%d\n",sion_gendata->grank));
1318 if(sion_gendata->grank == root) {
1320 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"before open\n"));
1321 sid=
_sion_open_read(fname,_SION_FMODE_READ|_SION_FMODE_ANSI,_SION_READ_MASTER_ONLY_OF_MULTI_FILES,
1322 &ntasks,&nfiles,NULL,&fsblksize,NULL,&fileptr);
1324 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"after open\n"));
1326 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"sion file %d files\n", *numfiles));
1334 sion_apidesc->bcastr_cb(numfiles, sion_gendata->comm_data_global, _SION_INT32, 1, root);
1335 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"%s: numfiles=%d\n",fname,*numfiles));
1337 if((sion_gendata->grank == root) && (*numfiles>1)) {
1338 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"%s: mapping_size=%d sion_gendata->gsize=%d\n",fname,*numfiles,mapping_size,sion_gendata->gsize));
1339 if(mapping_size!=sion_gendata->gsize) {
1340 return(_sion_errorprint(SION_NOT_SUCCESS,_SION_ERROR_RETURN,
1341 "_sion_generic_buddy_get_and_distribute_info_from_file: Incorrect sum of ntasks of files %d <> %d\n",
1342 mapping_size, sion_gendata->gsize));
1347 return(_sion_errorprint(SION_NOT_SUCCESS,_SION_ERROR_RETURN,
1348 "_sion_generic_buddy_get_and_distribute_info_from_file: could not get numfiles from sion file\n"));
1352 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"before scatter\n"));
1353 if(sion_gendata->grank==root) {
1354 for(t=0;t<mapping_size;t++) {
1355 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
" %d -> (%d,%d)\n",t,mapping[t*2],mapping[t*2+1]));
1360 sion_apidesc->scatterr_cb(mapping, lpos, sion_gendata->comm_data_global, _SION_INT32, 2, root);
1361 *filenumber=lpos[0];
1363 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"after scatter filenum+lrank (%d,%d)\n",*filenumber,*lrank));
1366 if(sion_gendata->grank==root) {
1367 tasksinfile = (
int *) malloc(*numfiles *
sizeof(
int));
1368 if (tasksinfile == NULL) {
1369 return(_sion_errorprint(SION_ID_NOT_VALID,_SION_ERROR_RETURN,
"_sion_generic_get_and_distribute_info_from_file: Cannot allocate memory for tasksinfile counter vector"));
1371 for(t=0;t<*numfiles;t++) tasksinfile[t]=0;
1372 for(t=0;t<mapping_size;t++) tasksinfile[ mapping[t*2] ]++;
1373 for(t=0;t<mapping_size;t++) mapping[t*2+1]=tasksinfile[ mapping[t*2] ];
1375 sion_apidesc->scatterr_cb(mapping, lpos, sion_gendata->comm_data_global, _SION_INT32, 2, root);
1377 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"after scatter lsize (%d, %d of %d)\n",*filenumber, *lrank, *lsize));
1379 if(sion_gendata->grank==root) {
1380 if(tasksinfile) free(tasksinfile);
1386 *lrank = sion_gendata->grank;
1387 *lsize = sion_gendata->gsize;
1388 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"only one file -> filenumber=%d lRank=%d\n",*filenumber,*lrank));
1391 if(sion_gendata->grank == root) {
1393 if (sid>=0) _sion_close_sid(sid);
1401 #define DFUNCTION "_sion_generic_buddy_get_and_distribute_info_from_one_file" 1402 int _sion_generic_buddy_get_and_distribute_info_from_one_file(
_sion_generic_gendata *sion_gendata,
char *fname,
int root,
1403 sion_int32 *filenumber, sion_int32 *numfiles,
1404 sion_int32 *lrank, sion_int32 *lsize) {
1406 int rc = SION_SUCCESS;
1407 int t, mapping_size = 0, grank, *mapping = NULL;
1408 sion_int32 file_filenumber,file_numfiles,file_lrank,file_lsize;
1413 sion_apidesc=sion_gendata->apidesc;
1415 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"entering function sion_gendata->grank=%d\n",sion_gendata->grank));
1417 if(sion_gendata->grank == root) {
1419 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"before open\n"));
1422 if (sion_filedesc == NULL) {
1423 return(_sion_errorprint(SION_ID_UNDEF,SION_ID_UNDEF,
1424 "sion_open: cannot allocate filedescriptor structure of size %lu (sion_filedesc), aborting ...\n",
1425 (
unsigned long)
sizeof(sion_filedesc)));
1431 if (!sion_fileptr) {
1432 return(_sion_errorprint(SION_ID_UNDEF,_SION_ERROR_RETURN,
"sion_open: cannot open %s for reading, aborting ...\n", fname));
1434 sion_filedesc->
fileptr = sion_fileptr;
1438 if (rc!=SION_SUCCESS) {
1439 return(_sion_errorprint(SION_ID_UNDEF,_SION_ERROR_RETURN,
"sion_open: cannot read header from file %s, aborting ...\n", fname));
1441 sion_filedesc->
rank = 0;
1450 if (rc!=SION_SUCCESS) {
1451 return(_sion_errorprint(SION_ID_UNDEF,_SION_ERROR_RETURN,
"sion_open: cannot read header (var part) from file %s, aborting ...\n", fname));
1454 file_numfiles=sion_filedesc->
nfiles;
1456 file_lsize=sion_filedesc->
ntasks;
1459 mapping_size=sion_gendata->gsize;
1460 mapping = (sion_int32 *) malloc(mapping_size * 2 *
sizeof(sion_int32));
1461 if (mapping == NULL) {
1462 return(_sion_errorprint(SION_ID_NOT_VALID,_SION_ERROR_RETURN,
"_sion_generic_buddy_get_and_distribute_info_from_one_file: cannot allocate temporary memory of size %lu (mapping), aborting ...\n",
1463 (
unsigned long) mapping_size * 2 *
sizeof(sion_int32)));
1470 sion_apidesc->bcastr_cb(&file_numfiles, sion_gendata->comm_data_global, _SION_INT32, 1, root);
1471 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"%s: numfiles=%d\n",fname,file_numfiles));
1472 sion_apidesc->bcastr_cb(&file_filenumber, sion_gendata->comm_data_global, _SION_INT32, 1, root);
1473 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"%s: filenumber=%d\n",fname,file_filenumber));
1474 sion_apidesc->bcastr_cb(&file_lsize, sion_gendata->comm_data_global, _SION_INT32, 1, root);
1475 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"%s: file_lsize=%d\n",fname,file_lsize));
1478 if(file_numfiles!=*numfiles) {
1479 _SION_SAFE_FREE(mapping, NULL);
1480 return(_sion_errorprint(SION_NOT_SUCCESS,_SION_ERROR_RETURN,
1481 "_sion_generic_buddy_get_and_distribute_info_from_one_file: Incorrect number of files %d <> %d\n",
1482 file_numfiles,*numfiles));
1486 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"before scatter\n"));
1487 if(sion_gendata->grank==root) {
1488 for(t=0;t<mapping_size;t++) mapping[t]=-1;
1489 for(t=0;t<file_lsize;t++) {
1493 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
" file=%s mapping[%d] -> (%d)\n",fname,grank,mapping[grank]));
1499 sion_apidesc->scatterr_cb(mapping, &file_lrank, sion_gendata->comm_data_global, _SION_INT32, 1, root);
1500 if(file_lrank!=-1) {
1501 *filenumber=file_filenumber;
1504 DPRINTFP((1, DFUNCTION, sion_gendata->grank,
"after scatter filenum=%d lrank=%d lsize=%d\n",*filenumber,*lrank,*lsize));
1507 if(sion_gendata->grank == root) {
int sion_feof(int sid)
Function that indicates whether the end of file is reached for this task.
sion_int64 _sion_file_set_position(_sion_fileptr *sion_fileptr, sion_int64 startpointer)
Set new position in file.
_sion_filedesc * _sion_alloc_filedesc()
Allocates memory for internal sion structure.
_sion_fileptr * _sion_file_open(const char *fname, unsigned int flags, unsigned int addflags)
Create and open a new file for writing.
Sion File Descriptor Structure.
int sion_ensure_free_space(int sid, sion_int64 bytes)
Funtion to ensure that enough space is available for writing.
int _sion_paropen_generic_one_file(int sid, char *fname, _sion_flags_store *flags_store, char *prefix, int *numFiles, int *filenumber, sion_int64 *chunksize, sion_int32 *fsblksize, int rank, int ntasks, int *globalrank, int flag, FILE **fileptr, _sion_generic_gendata *sion_gendata, _sion_generic_buddy *buddy_data)
Generic parallel open of one direct access file.
int _sion_parclose_generic(int sid, int rank, int ntasks, int mapping_size, sion_int32 *mapping, int flag, _sion_generic_gendata *sion_gendata, _sion_generic_buddy *buddy_data)
Internal function to close parallel opened SION file.
#define SION_FILESTATE_SEROPEN
sion_int64 * all_globalranks
int _sion_alloc_filedesc_arrays(_sion_filedesc *sion_filedesc)
Allocate memory for the internal sion arrays.
int _sion_free_filedesc_arrays(_sion_filedesc *sion_filedesc)
free memory for the internal sion arrays
#define SION_FILE_FLAG_READ
int _sion_vcdtype(int sid)
sion_int32 fileptr_exported
#define SION_FILE_FLAG_POSIX
int sion_get_mapping(int sid, int *mapping_size, sion_int32 **mapping, int *numfiles)
Returns pointers to the internal field mapping.
void * _sion_vcdtovcon(int sid)
#define SION_FILEMODE_READ
int _sion_file_close(_sion_fileptr *sion_fileptr)
Close file and destroys fileptr structure.
int _sion_newvcd(void *data, int type)
char * _sion_get_multi_filename(const char *fname, int filenumber)
generates the multi filename
#define SION_FILEDESCRIPTOR
int _sion_init_filedesc(_sion_filedesc *sion_filedesc)
Initialize the sion file description.
#define SION_FILEMODE_WRITE
int _sion_file_stat_file(const char *fname)
Check if file exists (LARGE_FILE support on BlueGene)
int _sion_file_flush(_sion_fileptr *sion_fileptr)
Flush data to file.
int _sion_open_read(const char *fname, sion_int64 file_mode_flags, int read_all, int *ntasks, int *nfiles, sion_int64 **chunksizes, sion_int32 *fsblksize, int **globalranks, FILE **fileptr)
internal sion serial open function for reading on one or more files