SIONlib  1.7.7
Scalable I/O library for parallel access to task-local files
sion_mpi_internal_gen.c
Go to the documentation of this file.
1 /****************************************************************************
2 ** SIONLIB http://www.fz-juelich.de/jsc/sionlib **
3 *****************************************************************************
4 ** Copyright (c) 2008-2019 **
5 ** Forschungszentrum Juelich, Juelich Supercomputing Centre **
6 ** **
7 ** See the file COPYRIGHT in the package base directory for details **
8 ****************************************************************************/
9 
14 #define _XOPEN_SOURCE 700
15 
16 #include <stdlib.h>
17 #include <stdio.h>
18 #include <stdarg.h>
19 #include <string.h>
20 #include <time.h>
21 
22 #include "mpi.h"
23 
24 #include <sys/time.h>
25 #include <sys/types.h>
26 #include <fcntl.h>
27 
28 #include <unistd.h>
29 
30 #include "sion.h"
31 #include "sion_debug.h"
32 #include "sion_error_handler.h"
33 #include "sion_internal.h"
34 #include "sion_fd.h"
35 #include "sion_filedesc.h"
36 #include "sion_printts.h"
37 
38 #include "sion_mpi_internal_gen.h"
39 
40 
41 #ifdef SION_MPI
42 
43 
44 int _sion_errorprint_mpi(int rc, int level, const char *format, ...)
45 {
46  int rank=-1, thread=-1;
47  va_list ap;
48 
49 #ifdef SION_MPI
50  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
51 #endif
52 
53  va_start(ap, format);
54  rc=__sion_errorprint_vargs(rc, level, rank, thread, format, ap);
55  va_end(ap);
56 
57  return (rc);
58 }
59 
60 
61 /* Help Functions */
62 
73 int _sion_gen_info_from_gcomm_mpi(int numFiles, MPI_Comm gComm, int *filenumber, int *lrank, int *lsize)
74 {
75  int gtasks, gRank;
76  int rc = SION_SUCCESS;
77  int task_per_file;
78 
79 
80  MPI_Comm_size(gComm, &gtasks);
81  MPI_Comm_rank(gComm, &gRank);
82 
83  if (gtasks < numFiles) {
84  return(_sion_errorprint_mpi(SION_NOT_SUCCESS,_SION_ERROR_RETURN,
85  "_sion_gen_info_from_gcomm_mpi: Number of files(%d) cannot be bigger the the number of tasks(%d), aborting...\n",
86  numFiles, gtasks));
87  }
88 
89  task_per_file = gtasks / numFiles;
90 
91  /* remaining tasks are added to last communicator */
92  if (gRank >= (numFiles * task_per_file)) {
93  *filenumber = numFiles - 1;
94  }
95  else {
96  *filenumber = gRank / task_per_file;
97  }
98 
99  if(*filenumber == numFiles - 1) {
100  *lrank=gRank-(numFiles - 1)*task_per_file;
101  *lsize=gtasks-(numFiles - 1)*task_per_file;
102  } else {
103  *lrank=gRank%task_per_file;
104  *lsize=task_per_file;
105  }
106 
107  DPRINTFP((1, "_sion_get_info_from_splitted_comm_mpi", gRank, "Global communicator divided in %d local communicators (%f: %d of %d)\n",
108  numFiles,*filenumber,*lrank,*lsize));
109 
110  return (rc);
111 }
112 
113 int _sion_get_info_from_splitted_comm_mpi(MPI_Comm gComm, MPI_Comm lComm, int *numComm, int *CommNumber, int *lrank, int *lsize) {
114  int gSize, gRank, lSize, lRank;
115  int *allRanks=NULL, *allSizes=NULL, i, ntasks;
116  int ncomms, color = 0;
117  int rc = SION_SUCCESS;
118  MPI_Status status;
119 
120  MPI_Comm_size(gComm, &gSize);
121  MPI_Comm_rank(gComm, &gRank);
122 
123  if (lComm != MPI_COMM_NULL) {
124  MPI_Comm_size(lComm, &lSize);
125  MPI_Comm_rank(lComm, &lRank);
126  }
127  else {
128  lSize = gSize;
129  lRank = gRank;
130  }
131 
132  DPRINTFP((32, "_sion_get_info_from_splitted_comm_mpi", gRank, "lRank: %d\n", lRank));
133 
134  if (gRank == 0) {
135 
136  allRanks = (int *) malloc(gSize * sizeof(int));
137  allSizes = (int *) malloc(gSize * sizeof(int));
138  if ((allRanks == NULL) || (allSizes == NULL)) {
139  free(allRanks);
140  free(allSizes);
141  return(_sion_errorprint_mpi(SION_NOT_SUCCESS,_SION_ERROR_RETURN,"_sion_get_info_from_splitted_comm_mpi: Cannot allocate temp arrays of size %lu\n", (unsigned long) gSize * sizeof(int)));
142  }
143  }
144 
145  /* Gather all local ranks to task 0 for counting.. */
146  MPI_Gather(&lRank, 1, MPI_INT, allRanks, 1, MPI_INT, 0, gComm);
147  MPI_Gather(&lSize, 1, MPI_INT, allSizes, 1, MPI_INT, 0, gComm);
148 
149  ncomms = 0;
150  ntasks = 0;
151  if (gRank == 0) {
152  for (i = 0; i < gSize; i++) {
153  if (allRanks[i] == 0) {
154  /* Use the current number of the communicator for the file suffix!! */
155  if (i != 0) {
156  MPI_Send(&ncomms, 1, MPI_INT, i, i, gComm);
157  DPRINTFP((32, "sion_paropen_comms_mpi", gRank, "Sent ncomms=%05d to %d with TAG %d\n", ncomms, i, i));
158  }
159  else {
160  /* it's my self */
161  color = ncomms;
162  }
163  ncomms++;
164  ntasks += allSizes[i];
165  }
166  }
167  }
168  /* Recv the num of communicator from global 0 =>used for the suffix */
169  if ((lRank == 0) && (gRank != 0)) {
170  MPI_Recv(&color, 1, MPI_INT, 0, gRank, gComm, &status);
171  DPRINTFP((32, "sion_paropen_comms_mpi", gRank, "Received ncomms=%05d from %d with TAG %d\n", color, status.MPI_SOURCE, status.MPI_TAG));
172 
173  }
174 
175  MPI_Bcast(&ncomms, 1, MPI_INT, 0, gComm);
176  MPI_Bcast(&ntasks, 1, MPI_INT, 0, gComm);
177 
178 
179  DPRINTFP((32, "_sion_get_info_from_splitted_comm_mpi", gRank, "#Comms=%05d #Tasks=%05d #TotalTasks=%05d\n", ncomms, ntasks, gSize));
180 
181  if (lComm != MPI_COMM_NULL) {
182  MPI_Bcast(&color, 1, MPI_INT, 0, lComm);
183  }
184 
185  if (gRank == 0) {
186  if(allRanks) free(allRanks);
187  if(allSizes) free(allSizes);
188  }
189  /* return parameters */
190  *numComm = ncomms;
191  *CommNumber = color;
192  *lrank = lRank;
193  *lsize = lSize;
194 
195  return (rc);
196 }
197 
198 int _sion_get_numfiles_from_file_mpi(char *fname) {
199 
200  int rank, sid;
201  FILE *fileptr;
202  int numfiles = 0;
203  sion_int64 chunksize;
204  sion_int32 fsblksize;
205 
206  rank = 0;
207  sid = _sion_open_rank(fname, "br", &chunksize, &fsblksize, &rank, &fileptr);
208 
209  numfiles = sion_get_number_of_files(sid);
210 
211  _sion_close_sid(sid);
212 
213  DPRINTFP((1, "_sion_get_numfiles_from_file_mpi", _SION_DEFAULT_RANK, "sion file %s has %d files\n", fname, numfiles));
214 
215  return (numfiles);
216 }
217 
218 int _sion_get_filenumber_from_files_mpi(char *fname, MPI_Comm gComm, int *numfiles, int *filenumber, int *lRank) {
219 
220  int sid, gSize, gRank, ntasks, nfiles;
221  int rc = SION_SUCCESS;
222  FILE *fileptr;
223  sion_int64 *chunksizes;
224  sion_int32 fsblksize;
225  int *globalranks;
226  int mapping_size = 0;
227  sion_int32 *mapping = NULL;
228  sion_int32 lpos[2];
229 
230  MPI_Comm_size(gComm, &gSize);
231  MPI_Comm_rank(gComm, &gRank);
232 
233  DPRINTFP((1, "_sion_get_filenumber_from_files_mpi", gRank, "before open\n"));
234  if(gRank == 0) {
235  /* open and get mapping of sion file */
236  chunksizes=NULL;globalranks=NULL;
237  sid=_sion_open_read(fname,_SION_FMODE_READ|_SION_FMODE_ANSI,_SION_READ_MASTER_ONLY_OF_MULTI_FILES,
238  &ntasks,&nfiles,&chunksizes,&fsblksize,&globalranks,&fileptr);
239  /* sid = sion_open(fname, "br", &ntasks, &nfiles, &chunksizes, &fsblksize, &globalranks, &fileptr); */
240  if(sid>=0) {
241  DPRINTFP((1, "_sion_get_filenumber_from_files_mpi", gRank, "after open\n"));
242  rc=sion_get_mapping(sid,&mapping_size,&mapping,numfiles);
243  DPRINTFP((1, "_sion_get_filenumber_from_files_mpi", gRank, "sion file %d files rc=%d\n", *numfiles, rc));
244  } else {
245  *numfiles=-1;
246  }
247  }
248 
249  /* each task has to know if more than file was used in sion file */
250  MPI_Bcast(numfiles, 1, MPI_INT, 0, gComm);
251 
252  if((gRank == 0) && (*numfiles>1)) {
253  if(mapping_size!=gSize) {
254  return(_sion_errorprint_mpi(SION_NOT_SUCCESS,_SION_ERROR_RETURN,"_sion_get_filenumber_from_files_mpi: Incorrect sum of ntasks of files %d <> %d\n", mapping_size, gSize));
255  }
256  }
257 
258  if(*numfiles<0) {
259  return(_sion_errorprint_mpi(SION_NOT_SUCCESS,_SION_ERROR_RETURN,"_sion_get_filenumber_from_files_mpi: could not get numfiles from sion file\n"));
260  }
261 
262  if(*numfiles>1) {
263  DPRINTFP((1, "_sion_get_filenumber_from_files_mpi", gRank, "before scatter\n"));
264  MPI_Scatter(mapping, 2, SION_MPI_INT32, lpos, 2, SION_MPI_INT32, 0, gComm);
265  *filenumber=lpos[0];
266  *lRank =lpos[1];
267  DPRINTFP((1, "_sion_get_filenumber_from_files_mpi", gRank, "after scatter\n"));
268  } else {
269  *filenumber=0;
270  *lRank =gRank;
271  DPRINTFP((1, "_sion_get_filenumber_from_files_mpi", gRank, "only one file -> filenumber=%d lRank=%d\n",*filenumber,*lRank));
272  }
273 
274  if(gRank == 0) {
275  /* frees also mapping vector */
276  if (sid>=0) _sion_close_sid(sid);
277  }
278 
279  DPRINTFP((1, "_sion_get_filenumber_from_files_mpi", gRank, "close rc=%d\n",rc));
280  return(rc);
281 }
282 
283 /* end of ifdef MPI */
284 #endif
int sion_get_mapping(int sid, int *mapping_size, sion_int32 **mapping, int *numfiles)
Returns pointers to the internal field mapping.
Definition: sion_common.c:219
int _sion_open_read(const char *fname, sion_int64 file_mode_flags, int read_all, int *ntasks, int *nfiles, sion_int64 **chunksizes, sion_int32 *fsblksize, int **globalranks, FILE **fileptr)
internal sion serial open function for reading on one or more files
int _sion_gen_info_from_gcomm_mpi(int numFiles, MPI_Comm gComm, int *filenumber, int *lrank, int *lsize)
Splits a Communicator in numfiles different communicators.
Sion Time Stamp Header.