sion_generic.c

/****************************************************************************
**  SIONLIB     http://www.fz-juelich.de/jsc/sionlib                       **
*****************************************************************************
**  Copyright (c) 2008-2018                                                **
**  Forschungszentrum Juelich, Juelich Supercomputing Centre               **
**                                                                         **
**  See the file COPYRIGHT in the package base directory for details       **
****************************************************************************/

#define _XOPEN_SOURCE 700

#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "sion_common.h"
#include "sion_const.h"
#include "sion_debug.h"
#include "sion_error_handler.h"
#include "sion_fd.h"
#include "sion_filedesc.h"
#include "sion_flags.h"
#include "sion_generic.h"
#include "sion_generic_apidesc.h"
#include "sion_generic_buddy.h"
#include "sion_generic_internal.h"
#include "sion_generic_mapped.h"
#include "sion_internal.h"

int sion_generic_create_api(char *name)
{
  int apiid = SION_ID_UNDEF;
  _sion_generic_apidesc *sion_apidesc;

  sion_apidesc = _sion_generic_alloc_apidesc();
  if (!sion_apidesc) {
    return _sion_errorprint(
      SION_ID_NOT_VALID, _SION_ERROR_RETURN, "_sion_generic_create_api: problems to alloc apidesc, aborting ...\n");
  }
  _sion_generic_init_apidesc(sion_apidesc);

  if (name == NULL) {
    return _sion_errorprint(
      SION_ID_NOT_VALID, _SION_ERROR_RETURN, "_sion_generic_create_api: name for API not given, aborting ...\n");
  }

  sion_apidesc->name = strdup(name);

  apiid = _sion_newvcd(sion_apidesc, SION_APIDESCRIPTOR);
  sion_apidesc->aid = apiid;
  sion_apidesc->level = SION_GENERIC_API_LEVEL_NONE;

  DPRINTFP((2, "_sion_generic_create_api", _SION_DEFAULT_RANK, "API created with name %s apiid=%d\n", sion_apidesc->name, apiid));

  return apiid;
}

int sion_generic_free_api(int aid)
{
  _sion_generic_apidesc *sion_apidesc;
  int rc = SION_SUCCESS;

  if ((aid < 0) || (_sion_vcdtype(aid) != SION_APIDESCRIPTOR) || !(sion_apidesc = _sion_vcdtovcon(aid))) {
    return _sion_errorprint(SION_NOT_SUCCESS, _SION_ERROR_RETURN, "sion_free_api: invalid sion_apidesc %d", aid);
  }
  _sion_freevcd(aid);
  rc = _sion_generic_free_apidesc(sion_apidesc);

  return rc;
}

int sion_generic_register_create_local_commgroup_cb(int aid, int create_lcg_cb(void **, void *, int, int, int, int, int, int))
{
  _sion_generic_apidesc *sion_apidesc;
  int rc = SION_SUCCESS;

  if ((aid < 0) || (_sion_vcdtype(aid) != SION_APIDESCRIPTOR) || !(sion_apidesc = _sion_vcdtovcon(aid))) {
    return _sion_errorprint(
      SION_NOT_SUCCESS, _SION_ERROR_RETURN, "sion_generic_register_create_local_commgroup_cb: invalid sion_apidesc %d", aid);
  }

  if (sion_apidesc->create_lcg_cb != NULL) {
    return _sion_errorprint(SION_NOT_SUCCESS, _SION_ERROR_RETURN,
      "sion_generic_register_create_local_commgroup_cb: update_cb already registered for apiid=%d", aid);
  }
  sion_apidesc->create_lcg_cb = create_lcg_cb;

  _sion_generic_update_api_level(sion_apidesc);

  DPRINTFP((2, "sion_generic_register_create_local_commgroup_cb", _SION_DEFAULT_RANK,
    "create_lcg_cb registered for apiid=%s (%d)\n", sion_apidesc->name, aid));

  return rc;
}

int sion_generic_register_free_local_commgroup_cb(int aid, int free_lcg_cb(void *))
{
  _sion_generic_apidesc *sion_apidesc;
  int rc = SION_SUCCESS;

  if ((aid < 0) || (_sion_vcdtype(aid) != SION_APIDESCRIPTOR) || !(sion_apidesc = _sion_vcdtovcon(aid))) {
    return _sion_errorprint(
      SION_NOT_SUCCESS, _SION_ERROR_RETURN, "sion_generic_register_free_local_commgroup_cb: invalid sion_apidesc %d", aid);
  }

  if (sion_apidesc->free_lcg_cb != NULL) {
    return _sion_errorprint(SION_NOT_SUCCESS, _SION_ERROR_RETURN,
      "sion_generic_register_free_local_commgroup_cb: update_cb already registered for apiid=%d", aid);
  }
  sion_apidesc->free_lcg_cb = free_lcg_cb;

  _sion_generic_update_api_level(sion_apidesc);

  DPRINTFP((2, "sion_generic_register_free_local_commgroup_cb", _SION_DEFAULT_RANK, "free_lcg_cb registered for apiid=%s (%d)\n",
    sion_apidesc->name, aid));

  return rc;
}

int sion_generic_register_barrier_cb(int aid, int barrier_cb(void *))
{
  _sion_generic_apidesc *sion_apidesc;
  int rc = SION_SUCCESS;

  if ((aid < 0) || (_sion_vcdtype(aid) != SION_APIDESCRIPTOR) || !(sion_apidesc = _sion_vcdtovcon(aid))) {
    return _sion_errorprint(
      SION_NOT_SUCCESS, _SION_ERROR_RETURN, "sion_generic_register_barrier_cb: invalid sion_apidesc %d", aid);
  }

  if (sion_apidesc->barrier_cb != NULL) {
    return _sion_errorprint(
      SION_NOT_SUCCESS, _SION_ERROR_RETURN, "sion_generic_register_barrier_cb: barrier_cb already registered for apiid=%d", aid);
  }
  sion_apidesc->barrier_cb = barrier_cb;

  _sion_generic_update_api_level(sion_apidesc);

  DPRINTFP((2, "sion_generic_register_barrier_cb", _SION_DEFAULT_RANK, "barrier_cb registered for apiid=%s (%d)\n",
    sion_apidesc->name, aid));

  return rc;
}

int sion_generic_register_bcastr_cb(int aid, int bcastr_cb(void *, void *, int, int, int))
{
  _sion_generic_apidesc *sion_apidesc;
  int rc = SION_SUCCESS;

  if ((aid < 0) || (_sion_vcdtype(aid) != SION_APIDESCRIPTOR) || !(sion_apidesc = _sion_vcdtovcon(aid))) {
    return _sion_errorprint(
      SION_NOT_SUCCESS, _SION_ERROR_RETURN, "sion_generic_register_bcastr_cb: invalid sion_apidesc %d", aid);
  }

  if (sion_apidesc->bcastr_cb != NULL) {
    return _sion_errorprint(
      SION_NOT_SUCCESS, _SION_ERROR_RETURN, "sion_generic_register_bcastr_cb: bcastr_cb already registered for apiid=%d", aid);
  }
  sion_apidesc->bcastr_cb = bcastr_cb;

  _sion_generic_update_api_level(sion_apidesc);

  DPRINTFP((2, "sion_generic_register_bcastr_cb", _SION_DEFAULT_RANK, "bcastr_cb registered for apiid=%s (%d)\n",
    sion_apidesc->name, aid));

  return rc;
}

int sion_generic_register_gatherr_cb(int aid, int gatherr_cb(void *, void *, void *, int, int, int))
{
  _sion_generic_apidesc *sion_apidesc;
  int rc = SION_SUCCESS;

  if ((aid < 0) || (_sion_vcdtype(aid) != SION_APIDESCRIPTOR) || !(sion_apidesc = _sion_vcdtovcon(aid))) {
    return _sion_errorprint(
      SION_NOT_SUCCESS, _SION_ERROR_RETURN, "sion_generic_register_gatherr_cb: invalid sion_apidesc %d", aid);
  }

  if (sion_apidesc->gatherr_cb != NULL) {
    return _sion_errorprint(
      SION_NOT_SUCCESS, _SION_ERROR_RETURN, "sion_generic_register_gatherr_cb: gatherr_cb already registered for apiid=%d", aid);
  }
  sion_apidesc->gatherr_cb = gatherr_cb;

  _sion_generic_update_api_level(sion_apidesc);

  DPRINTFP((2, "sion_generic_register_gatherr_cb", _SION_DEFAULT_RANK, "gatherr_cb registered for apiid=%s (%d)\n",
    sion_apidesc->name, aid));

  return rc;
}

int sion_generic_register_gathervr_cb(int aid, int gathervr_cb(void *, void *, void *, int, int *, int, int))
{
  _sion_generic_apidesc *sion_apidesc;
  int rc = SION_SUCCESS;

  if ((aid < 0) || (_sion_vcdtype(aid) != SION_APIDESCRIPTOR) || !(sion_apidesc = _sion_vcdtovcon(aid))) {
    return _sion_errorprint(
      SION_NOT_SUCCESS, _SION_ERROR_RETURN, "sion_generic_register_gathervr_cb: invalid sion_apidesc %d", aid);
  }

  if (sion_apidesc->gathervr_cb != NULL) {
    return _sion_errorprint(SION_NOT_SUCCESS, _SION_ERROR_RETURN,
      "sion_generic_register_gathervr_cb: gathervr_cb already registered for apiid=%d", aid);
  }
  sion_apidesc->gathervr_cb = gathervr_cb;

  _sion_generic_update_api_level(sion_apidesc);

  DPRINTFP((2, "sion_generic_register_gathervr_cb", _SION_DEFAULT_RANK, "gathervr_cb registered for apiid=%s (%d)\n",
    sion_apidesc->name, aid));

  return rc;
}

int sion_generic_register_scatterr_cb(int aid, int scatterr_cb(void *, void *, void *, int, int, int))
{
  _sion_generic_apidesc *sion_apidesc;
  int rc = SION_SUCCESS;

  if ((aid < 0) || (_sion_vcdtype(aid) != SION_APIDESCRIPTOR) || !(sion_apidesc = _sion_vcdtovcon(aid))) {
    return _sion_errorprint(
      SION_NOT_SUCCESS, _SION_ERROR_RETURN, "sion_generic_register_scatterr_cb: invalid sion_apidesc %d", aid);
  }

  if (sion_apidesc->scatterr_cb != NULL) {
    return _sion_errorprint(SION_NOT_SUCCESS, _SION_ERROR_RETURN,
      "sion_generic_register_scatterr_cb: scatterr_cb already registered for apiid=%d", aid);
  }
  sion_apidesc->scatterr_cb = scatterr_cb;

  _sion_generic_update_api_level(sion_apidesc);

  DPRINTFP((2, "sion_generic_register_scatterr_cb", _SION_DEFAULT_RANK, "scatterr_cb registered for apiid=%s (%d)\n",
    sion_apidesc->name, aid));

  return rc;
}

int sion_generic_register_scattervr_cb(int aid, int scattervr_cb(void *, void *, void *, int, int *, int, int))
{
  _sion_generic_apidesc *sion_apidesc;
  int rc = SION_SUCCESS;

  if ((aid < 0) || (_sion_vcdtype(aid) != SION_APIDESCRIPTOR) || !(sion_apidesc = _sion_vcdtovcon(aid))) {
    return _sion_errorprint(
      SION_NOT_SUCCESS, _SION_ERROR_RETURN, "sion_generic_register_scattervr_cb: invalid sion_apidesc %d", aid);
  }

  if (sion_apidesc->scattervr_cb != NULL) {
    return _sion_errorprint(SION_NOT_SUCCESS, _SION_ERROR_RETURN,
      "sion_generic_register_scattervr_cb: scattervr_cb already registered for apiid=%d", aid);
  }
  sion_apidesc->scattervr_cb = scattervr_cb;

  _sion_generic_update_api_level(sion_apidesc);

  DPRINTFP((2, "sion_generic_register_scattervr_cb", _SION_DEFAULT_RANK, "scattervr_cb registered for apiid=%s (%d)\n",
    sion_apidesc->name, aid));

  return rc;
}

int sion_generic_register_gather_and_execute_cb(int aid,
  int64_t gather_execute_cb(const void *, int64_t, void *, int, int, int, int, int64_t process_cb(const void *, int64_t *, int),
    void *, bool spec_iterator_cb(void *, int64_t *)))
{
  _sion_generic_apidesc *sion_apidesc;
  int rc = SION_SUCCESS;

  if ((aid < 0) || (_sion_vcdtype(aid) != SION_APIDESCRIPTOR) || !(sion_apidesc = _sion_vcdtovcon(aid))) {
    return _sion_errorprint(
      SION_NOT_SUCCESS, _SION_ERROR_RETURN, "sion_generic_register_gather_and_execute_cb: invalid sion_apidesc %d", aid);
  }

  if (sion_apidesc->gather_execute_cb != NULL) {
    return _sion_errorprint(SION_NOT_SUCCESS, _SION_ERROR_RETURN,
      "sion_generic_register_gather_and_execute_cb: scattervr_cb already registered for apiid=%d", aid);
  }
  sion_apidesc->gather_execute_cb = gather_execute_cb;

  _sion_generic_update_api_level(sion_apidesc);

  DPRINTFP((2, "sion_generic_register_gather_and_execute_cb", _SION_DEFAULT_RANK,
    "gather_execute_cb registered for apiid=%s (%d)\n", sion_apidesc->name, aid));

  return rc;
}

int sion_generic_register_execute_and_scatter_cb(int aid,
  int64_t execute_scatter_cb(void *, int64_t, void *, int, int, int, int, int64_t process_cb(void *, int64_t *, int), void *,
    bool spec_iterator_cb(void *, int64_t *)))
{
  _sion_generic_apidesc *sion_apidesc;
  int rc = SION_SUCCESS;

  if ((aid < 0) || (_sion_vcdtype(aid) != SION_APIDESCRIPTOR) || !(sion_apidesc = _sion_vcdtovcon(aid))) {
    return _sion_errorprint(
      SION_NOT_SUCCESS, _SION_ERROR_RETURN, "sion_generic_register_execute_and_scatter_cb: invalid sion_apidesc %d", aid);
  }

  if (sion_apidesc->execute_scatter_cb != NULL) {
    return _sion_errorprint(SION_NOT_SUCCESS, _SION_ERROR_RETURN,
      "sion_generic_register_execute_and_scatter_cb: scattervr_cb already registered for apiid=%d", aid);
  }
  sion_apidesc->execute_scatter_cb = execute_scatter_cb;

  DPRINTFP((2, "sion_generic_register_execute_and_scatter_cb", _SION_DEFAULT_RANK,
    "gather_execute_cb registered for apiid=%s (%d)\n", sion_apidesc->name, aid));

  return rc;
}

int sion_generic_register_get_multi_filename_cb(int aid, char *get_multi_filename_cb(const char *, int))
{
  _sion_generic_apidesc *sion_apidesc;
  int rc = 0;

  if ((aid < 0) || (_sion_vcdtype(aid) != SION_APIDESCRIPTOR) || !(sion_apidesc = _sion_vcdtovcon(aid))) {
    return _sion_errorprint(-1, _SION_ERROR_RETURN, "sion_generic_register_get_multi_filename_cb: invalid sion_apidesc %d", aid);
  }

  if (sion_apidesc->get_multi_filename_cb != NULL) {
    return _sion_errorprint(-1, _SION_ERROR_RETURN,
      "sion_generic_register_get_multi_filename_cb: get_multi_filename_cb already registered for apiid=%d", aid);
  }
  sion_apidesc->get_multi_filename_cb = get_multi_filename_cb;

  _sion_generic_update_api_level(sion_apidesc);

  DPRINTFP((2, "sion_generic_register_get_multi_filename_cb", -1, "get_multi_filename_cb registered for apiid=%s (%d)\n",
    sion_apidesc->name, aid));

  return rc;
}

int sion_generic_register_get_capability_cb(int aid, int get_capability_cb(void *))
{
  _sion_generic_apidesc *sion_apidesc;
  int rc = SION_SUCCESS;

  if ((aid < 0) || (_sion_vcdtype(aid) != SION_APIDESCRIPTOR) || !(sion_apidesc = _sion_vcdtovcon(aid))) {
    return _sion_errorprint(
      SION_NOT_SUCCESS, _SION_ERROR_RETURN, "sion_generic_register_get_capability_cb: invalid sion_apidesc %d", aid);
  }

  if (sion_apidesc->get_capability_cb != NULL) {
    return _sion_errorprint(SION_NOT_SUCCESS, _SION_ERROR_RETURN,
      "sion_generic_register_get_capability_cb: get_capability_cb already registered for apiid=%d", aid);
  }
  sion_apidesc->get_capability_cb = get_capability_cb;

  _sion_generic_update_api_level(sion_apidesc);

  DPRINTFP((2, "sion_generic_register_get_capability_cb", _SION_DEFAULT_RANK, "get_capability_cb registered for apiid=%s (%d)\n",
    sion_apidesc->name, aid));

  return rc;
}

int sion_generic_paropen(int aid, const char *fname, const char *file_mode, int64_t *chunksize, int32_t *fsblksize,
  void *gcommgroup, int grank, int gsize, int *filenumber, int *numfiles, const int *lrank, const int *lsize, FILE **fileptr,
  char **newfname)
{
  int sid;
  _sion_flags_store *flags_store = _sion_parse_flags(file_mode);
  if (!flags_store) {
    return _sion_errorprint(
      SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_paropen: could not parse file mode in %s, aborting ...\n", file_mode);
  }
  sid = _sion_generic_paropen(aid, fname, flags_store, chunksize, fsblksize, gcommgroup, grank, gsize, filenumber, numfiles,
    lrank, lsize, fileptr, newfname);
  _sion_flags_destroy_store(&flags_store);
  return sid;
}

int _sion_generic_paropen(int aid, const char *fname, const _sion_flags_store *flags_store, int64_t *chunksize,
  int32_t *fsblksize, void *gcommgroup, int grank, int gsize, int *filenumber, int *numfiles, const int *lrank, const int *lsize,
  FILE **fileptr, char **newfname)
{
  int rc, sid = SION_ID_UNDEF;
  int32_t lfsblksize;
  int64_t lchunksize;
  int help_globalrank;
  char *nfname = NULL;
  _sion_filedesc *sion_filedesc;
  _sion_generic_apidesc *sion_apidesc;
  _sion_generic_gendata *sion_gendata;

  DPRINTFTS(grank, "enter sion_generic_paropen");
  DPRINTFP((1, "sion_generic_paropen", grank,
    "enter generic parallel open of FILE %s (global: %d of %d, local: %d of %d, file: %d of %d )\n", fname, grank, gsize, *lrank,
    *lsize, *filenumber, *numfiles));

  if ((aid < 0) || (_sion_vcdtype(aid) != SION_APIDESCRIPTOR) || !(sion_apidesc = _sion_vcdtovcon(aid))) {
    return _sion_errorprint(
      SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_register_scattervr_cb: invalid sion_apidesc %d", aid);
  }

  if ((sion_apidesc->level != SION_GENERIC_API_LEVEL_STD) && (sion_apidesc->level != SION_GENERIC_API_LEVEL_FULL)) {
    return _sion_errorprint(SION_ID_NOT_VALID, _SION_ERROR_RETURN,
      "sion_generic_paropen: API %s not correctly initalized, aborting", sion_apidesc->name);
  }

  /* check parameters */
  if ((grank < 0) || (grank >= gsize)) {
    return _sion_errorprint(
      SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_paropen: global rank %d not valid (0..%d)", grank, gsize);
  }

#ifdef HACK_SCALASCA
  /* hack for scalasca */
  if (strstr(fname, ".def") != NULL) {
    lfsblksize = 65536;
    lchunksize = 65536;
  } else {
#endif
    lfsblksize = *fsblksize;
    lchunksize = *chunksize;
#ifdef HACK_SCALASCA
  }
#endif
  DPRINTFP((1, "sion_generic_paropen", grank, "setting fsblksize = %d for file %s\n", lfsblksize, fname));

  /* create data container */
  sion_gendata = _sion_generic_alloc_gendata();
  if (!sion_gendata) {
    return _sion_errorprint(
      SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_paropen: problems to alloc gendata, aborting ...\n");
  }
  _sion_generic_init_gendata(sion_gendata);

  sion_gendata->aid = aid;
  sion_gendata->apidesc = sion_apidesc;
  sion_gendata->comm_data_global = gcommgroup;
  sion_gendata->comm_data_local = NULL;
  sion_gendata->grank = grank;
  sion_gendata->gsize = gsize;

  if (flags_store->mask & _SION_FMODE_WRITE) {
    if (lrank == NULL) {
      return _sion_errorprint(SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_paropen: No lrank variable given");
    }
    if (lsize == NULL) {
      return _sion_errorprint(SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_paropen: No lsize variable given");
    }
    if ((*lrank < 0) || (*lrank >= *lsize)) {
      return _sion_errorprint(
        SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_paropen: local rank %d not valid (0..%d)", *lrank, *lsize);
    }
    sion_gendata->lrank = *lrank;
    sion_gendata->lsize = *lsize;

    if (filenumber == NULL) {
      return _sion_errorprint(SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_paropen: No filenumber variable given");
    }
    if (numfiles == NULL) {
      return _sion_errorprint(SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_paropen: No numfiles variable given");
    }
    if ((*filenumber < 0) || (*filenumber >= *numfiles)) {
      return _sion_errorprint(
        SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_paropen: file number %d not valid (0..%d)", *filenumber, *numfiles);
    }
    sion_gendata->filenumber = *filenumber;
    sion_gendata->numfiles = *numfiles;

  } else if (flags_store->mask & _SION_FMODE_READ) {
    /* file mode READ */

    DPRINTFP((1, "sion_generic_paropen", grank, "READ: buddy=%d\n", flags_store->mask & _SION_FMODE_BUDDY));

    /* get number of files and filenumber */
    if (!(flags_store->mask & _SION_FMODE_BUDDY)) {
      rc = _sion_generic_get_and_distribute_info_from_file(sion_gendata, fname);
      *numfiles = sion_gendata->numfiles;
      *filenumber = sion_gendata->filenumber;
      if (rc != SION_SUCCESS) {
        return _sion_errorprint(SION_ID_NOT_VALID, _SION_ERROR_RETURN,
          "sion_generic_paropen: error in _sion_generic_get_and_distribute_info_from_file");
      }

    } else {
      /* buddy will determine file layout later, but need local commgroup to optimize file operations */

      sion_gendata->lrank = *lrank;
      sion_gendata->lsize = *lsize;

      if (filenumber == NULL) {
        return _sion_errorprint(SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_paropen: No filenumber variable given");
      }
      if (numfiles == NULL) {
        return _sion_errorprint(SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_paropen: No numfiles variable given");
      }
      if ((*filenumber < 0) || (*filenumber >= *numfiles)) {
        return _sion_errorprint(SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_paropen: file number %d not valid (0..%d)",
          *filenumber, *numfiles);
      }
      sion_gendata->filenumber = *filenumber;
      sion_gendata->numfiles = *numfiles;
    }

  } else {
    return _sion_errorprint(SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_paropen: unknown file mode");
  }

  /* generate local commgroup according to information from file (read) or parameter (write) */
  rc = sion_apidesc->create_lcg_cb(&sion_gendata->comm_data_local, sion_gendata->comm_data_global, sion_gendata->grank,
    sion_gendata->gsize, sion_gendata->lrank, sion_gendata->lsize, sion_gendata->filenumber, sion_gendata->numfiles);
  if (rc) {
    return _sion_errorprint(
      SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_paropen_generic: error in create_local_commgroup callback function");
  }

  /* set up parameters of call to generic open  */
  nfname = (sion_apidesc->get_multi_filename_cb ? sion_apidesc->get_multi_filename_cb : _sion_get_multi_filename)(
    fname, sion_gendata->filenumber);
  help_globalrank = sion_gendata->grank;

  sid = _sion_newvcd(NULL, SION_FILEDESCRIPTOR);

  if (!(flags_store->mask & _SION_FMODE_BUDDY)) {
    /* STANDARD  */
    /* FIXME: add debug output for flags_store
    DPRINTFP((1, "sion_generic_paropen", grank, "call parallel open of %d files (current name %s) in %s mode\n", sion_gendata->numfiles, nfname, file_mode)); */
    DPRINTFP(
      (1, "sion_generic_paropen", grank, "call parallel open of %d files (current name %s)\n", sion_gendata->numfiles, nfname));
    rc = _sion_paropen_generic_one_file(sid, nfname, flags_store, &sion_gendata->numfiles, &sion_gendata->filenumber, &lchunksize,
      &lfsblksize, sion_gendata->lrank, sion_gendata->lsize, &help_globalrank, _SION_INTERNAL_FLAG_NORMAL, fileptr, sion_gendata,
      NULL);
    /* FIXME: add debug output for flags_store
    DPRINTFP((1, "sion_generic_paropen", sion_gendata->grank, "leave parallel open of %d files in %s mode #tasks=%d sid=%d globalrank=%d\n", sion_gendata->numfiles,
        file_mode, sion_gendata->lsize, sid, sion_gendata->grank)); */
    DPRINTFP((1, "sion_generic_paropen", sion_gendata->grank, "leave parallel open of %d files #tasks=%d sid=%d globalrank=%d\n",
      sion_gendata->numfiles, sion_gendata->lsize, sid, sion_gendata->grank));

  } else {
    /* branch BUDDY Checkpointing  */
    /* FIXME: add debug output for flags_store
    DPRINTFP((1, "sion_generic_paropen", grank, "call parallel buddy open of %d files (current name %s) in %s mode\n", sion_gendata->numfiles, nfname, file_mode)); */
    DPRINTFP((1, "sion_generic_paropen", grank, "call parallel buddy open of %d files (current name %s)\n",
      sion_gendata->numfiles, nfname));
    rc = _sion_paropen_generic_buddy(sid, fname, flags_store, &sion_gendata->numfiles, &sion_gendata->filenumber, &lchunksize,
      &lfsblksize, sion_gendata->lrank, sion_gendata->lsize, &help_globalrank, fileptr, sion_gendata);
    /* FIXME: add debug output for flags_store
    DPRINTFP((1, "sion_generic_paropen", sion_gendata->grank, "leave parallel buddy open of %d files in %s mode #tasks=%d sid=%d globalrank=%d rc=%d\n", sion_gendata->numfiles,
        file_mode, sion_gendata->lsize, sid, sion_gendata->grank, rc)); */
    DPRINTFP((1, "sion_generic_paropen", sion_gendata->grank,
      "leave parallel buddy open of %d files #tasks=%d sid=%d globalrank=%d rc=%d\n", sion_gendata->numfiles, sion_gendata->lsize,
      sid, sion_gendata->grank, rc));
  }

  /* test sid and get internal data structure */
  if ((rc < 0) || (sid < 0) || (_sion_vcdtype(sid) != SION_FILEDESCRIPTOR) || !(sion_filedesc = _sion_vcdtovcon(sid))) {
    DPRINTFP((1, "sion_generic_paropen", sion_gendata->grank, "invalid rc %d or sion_filedesc %d", rc, sid));
    return _sion_errorprint(
      SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_paropen: invalid rc %d or sion_filedesc %d", rc, sid);
  }

  if (newfname != NULL) {
    *newfname = nfname;
  } else {
    free(nfname);
  }

  /* OUTPUT parameters */
  *fsblksize = lfsblksize;
  *chunksize = lchunksize;

  /* Bug? */

  DPRINTFP((1, "sion_generic_paropen", grank, "leave parallel open of file %s sid=%d\n", fname, sid));
  DPRINTFTS(-1, "leave sion_generic_paropen");

  return sid;
}

int sion_generic_parclose(int sid)
{
  ONLY_DEBUG(int grank);
  int rc = SION_SUCCESS;
  _sion_filedesc *sion_filedesc;
  _sion_generic_gendata *sion_gendata;
  _sion_generic_apidesc *sion_apidesc;

  DPRINTFTS(-1, "enter sion_generic_parclose");

  if ((sid < 0) || (_sion_vcdtype(sid) != SION_FILEDESCRIPTOR) || !(sion_filedesc = _sion_vcdtovcon(sid))) {
    return _sion_errorprint(SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_parclose: invalid sion_filedesc %d", sid);
  }
  sion_gendata = sion_filedesc->dataptr;
  sion_apidesc = sion_gendata->apidesc;

  ONLY_DEBUG(grank = sion_gendata->grank);

  DPRINTFP((1, "sion_generic_parclose", grank, "enter parallel close of sid %d (%d file)\n", sid, sion_filedesc->nfiles));

  if (!sion_filedesc->usebuddy) {
    rc = _sion_parclose_generic(sid, sion_filedesc->rank, sion_filedesc->ntasks, _SION_INTERNAL_FLAG_NORMAL, sion_gendata, NULL);
  } else {
    rc = _sion_parclose_generic_buddy(sid, sion_filedesc->rank, sion_filedesc->ntasks, sion_gendata);
  }

  if (sion_apidesc->free_lcg_cb && sion_gendata->comm_data_local) {
    sion_apidesc->free_lcg_cb(sion_gendata->comm_data_local);
  }

  _sion_freevcd(sid);

  sion_apidesc->barrier_cb(sion_gendata->comm_data_global);

  free(sion_gendata);

  DPRINTFP((1, "sion_generic_parclose", grank, "leave parallel close of sid %d\n", sid));
  DPRINTFTS(-1, "leave sion_generic_parclose");

  return rc;
}

int sion_generic_paropen_mapped(int aid, char *fname, const char *file_mode, int *numFiles, void *gcommgroup, int grank,
  int gsize, int *nlocaltasks, int **globalranks, int64_t **chunksizes, int **mapping_filenrs, int **mapping_lranks,
  int32_t *fsblksize, FILE **fileptr)
{
  int sid;
  _sion_flags_store *flags_store = _sion_parse_flags(file_mode);
  if (!flags_store) {
    return _sion_errorprint(SION_ID_NOT_VALID, _SION_ERROR_RETURN,
      "sion_generic_paropen_mapped: could not parse file mode in %s, aborting ...\n", file_mode);
  }
  sid = _sion_generic_paropen_mapped(aid, fname, flags_store, numFiles, gcommgroup, grank, gsize, nlocaltasks, globalranks,
    chunksizes, mapping_filenrs, mapping_lranks, fsblksize, fileptr);
  _sion_flags_destroy_store(&flags_store);
  return sid;
}

int _sion_generic_paropen_mapped(int aid, char *fname, const _sion_flags_store *flags_store, int *numFiles, void *gcommgroup,
  int grank, int gsize, int *nlocaltasks, int **globalranks, int64_t **chunksizes, int **mapping_filenrs, int **mapping_lranks,
  int32_t *fsblksize, FILE **fileptr)
{
  int rc, sid = SION_ID_UNDEF;
  _sion_filedesc *sion_filedesc;
  _sion_generic_apidesc *sion_apidesc;
  _sion_generic_gendata *sion_gendata;

  DPRINTFTS(grank, "enter sion_generic_paropen_mapped");
  DPRINTFP((1, "sion_generic_paropen_mapped", grank,
    "enter generic parallel mapped open of file %s (global: %d of %d, nlocaltasks %d)\n", fname, grank, gsize, *nlocaltasks));

  if ((aid < 0) || (_sion_vcdtype(aid) != SION_APIDESCRIPTOR) || !(sion_apidesc = _sion_vcdtovcon(aid))) {
    return _sion_errorprint(
      SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_register_scattervr_cb: invalid sion_apidesc %d", aid);
  }

  if ((!sion_apidesc->create_lcg_cb) || (!sion_apidesc->free_lcg_cb) || (!sion_apidesc->barrier_cb) || (!sion_apidesc->bcastr_cb)
    || (!sion_apidesc->gatherr_cb) || (!sion_apidesc->gathervr_cb) || (!sion_apidesc->scatterr_cb)
    || (!sion_apidesc->scattervr_cb)) {
    return _sion_errorprint(SION_ID_NOT_VALID, _SION_ERROR_RETURN,
      "sion_generic_paropen_mapped: API %s not correctly initalized, aborting", sion_apidesc->name);
  }

  /* check parameters */
  if ((grank < 0) || (grank >= gsize)) {
    return _sion_errorprint(
      SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_paropen_mapped: global rank %d not valid (0..%d)", grank, gsize);
  }

  /* create data container */
  sion_gendata = _sion_generic_alloc_gendata();
  if (!sion_gendata) {
    return _sion_errorprint(
      SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_paropen_mapped: problems to alloc gendata, aborting ...\n");
  }
  _sion_generic_init_gendata(sion_gendata);

  sion_gendata->aid = aid;
  sion_gendata->apidesc = sion_apidesc;
  sion_gendata->comm_data_global = gcommgroup;
  sion_gendata->grank = grank;
  sion_gendata->gsize = gsize;
  sion_gendata->lrank = -1;
  sion_gendata->lsize = -1;
  sion_gendata->filenumber = -1;
  sion_gendata->numfiles = -1;
  sion_gendata->comm_data_local = NULL;

  if (!(flags_store->mask & (_SION_FMODE_WRITE | _SION_FMODE_READ))) {
    return _sion_errorprint(SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_paropen_mapped: unknown file mode");
  }

  sid = _sion_newvcd(NULL, SION_FILEDESCRIPTOR);

  // FIXME: re-enable debug output of file mode
  // DPRINTFP((1, "sion_generic_paropen_mapped", grank, "enter parallel mapped open  in %s mode\n", file_mode));
  DPRINTFP((1, "sion_generic_paropen_mapped", grank, "enter parallel mapped open\n"));
  rc = _sion_paropen_mapped_generic(sid, fname, flags_store->mask, numFiles, nlocaltasks, globalranks, chunksizes,
    mapping_filenrs, mapping_lranks, fsblksize, grank, gsize, _SION_INTERNAL_FLAG_NORMAL, fileptr, sion_gendata);
  sion_gendata->numfiles = *numFiles;
  // FIXME: re-enable debug output of file mode
  // DPRINTFP((1, "sion_generic_paropen_mapped", sion_gendata->grank, "leave parallel open of %d files in %s mode #tasks=%d sid=%d globalrank=%d\n", sion_gendata->numfiles,
  //                               file_mode, sion_gendata->lsize, sid, sion_gendata->grank));
  DPRINTFP(
    (1, "sion_generic_paropen_mapped", sion_gendata->grank, "leave parallel open of %d files #tasks=%d sid=%d globalrank=%d\n",
      sion_gendata->numfiles, sion_gendata->lsize, sid, sion_gendata->grank));

  /* test sid again and get internal data structure */
  if ((sid < 0) || (_sion_vcdtype(sid) != SION_FILEDESCRIPTOR) || !(sion_filedesc = _sion_vcdtovcon(sid))) {
    return _sion_errorprint(SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_paropen_mapped: invalid sion_filedesc %d", sid);
  }

  /* store additional data */
  sion_filedesc->dataptr = sion_gendata;

  /* Bug? */

  DPRINTFP((1, "sion_generic_paropen_mapped", grank, "leave parallel open of file %s sid=%d\n", fname, sid));
  DPRINTFTS(-1, "leave sion_generic_paropen_mapped");

  return rc;
}

int sion_generic_parclose_mapped(int sid)
{
  int grank, gsize, rc;
  _sion_filedesc *sion_filedesc;
  _sion_generic_gendata *sion_gendata;

  DPRINTFTS(-1, "enter sion_generic_parclose_mapped");
  if ((sid < 0) || (_sion_vcdtype(sid) != SION_FILEDESCRIPTOR) || !(sion_filedesc = _sion_vcdtovcon(sid))) {
    return _sion_errorprint(SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_parclose_mapped: invalid sion_filedesc %d", sid);
  }
  sion_gendata = sion_filedesc->dataptr;

  grank = sion_gendata->grank;
  gsize = sion_gendata->gsize;

  DPRINTFP((1, "sion_generic_parclose_mapped", grank, "enter parallel close of sid %d\n", sid));

  DPRINTFP((1, "sion_generic_parclose_mapped", grank, "closing %d file(s)\n", sion_filedesc->nfiles));

  rc = _sion_parclose_mapped_generic(sid, grank, gsize, sion_gendata);

  _sion_freevcd(sid);

  _sion_generic_free_gendata(sion_gendata);

  DPRINTFP((1, "sion_generic_parclose_mapped", grank, "leave parallel close of sid %d\n", sid));
  DPRINTFTS(-1, "leave sion_generic_parclose_mapped");

  return rc;
}

int _sion_generic_get_and_distribute_info_from_file(_sion_generic_gendata *sion_gendata, const char *fname)
{
  int sid = -1, ntasks = -1, nfiles = -1, t = 0;
  int rc = SION_SUCCESS;
  int32_t fsblksize;
  int *tasksinfile;
  int mapping_size = -1;
  int32_t *mapping = NULL;
  int32_t lpos[2];
  _sion_generic_apidesc *sion_apidesc;

  sion_apidesc = sion_gendata->apidesc;

  if (sion_gendata->grank == 0) {
    /* open and get mapping of sion file */
    DPRINTFP((1, "_sion_generic_get_and_distribute_info_from_file", sion_gendata->grank, "before open\n"));
    sid = _sion_open_read(fname, _SION_FMODE_READ | _SION_FMODE_ANSI, _SION_READ_MASTER_ONLY_OF_MULTI_FILES, &ntasks, &nfiles,
      NULL, &fsblksize, NULL, NULL);
    if (sid >= 0) {
      DPRINTFP((1, "_sion_generic_get_and_distribute_info_from_file", sion_gendata->grank, "after open\n"));
      rc = sion_get_mapping(sid, &mapping_size, &mapping, &sion_gendata->numfiles);
      DPRINTFP((1, "_sion_generic_get_and_distribute_info_from_file", sion_gendata->grank, "sion file %d files\n",
        sion_gendata->numfiles));
    } else {
      sion_gendata->numfiles = -1;
    }
  }

  /* each task has to know if more than file was used in sion file */
  sion_apidesc->bcastr_cb(&sion_gendata->numfiles, sion_gendata->comm_data_global, _SION_INT32, 1, 0);

  if ((sion_gendata->grank == 0) && (sion_gendata->numfiles > 1)) {
    if (mapping_size != sion_gendata->gsize) {
      return _sion_errorprint(SION_NOT_SUCCESS, _SION_ERROR_RETURN,
        "_sion_generic_get_and_distribute_info_from_file: Incorrect sum of ntasks of files %d <> %d\n", mapping_size,
        sion_gendata->gsize);
    }
  }

  if (sion_gendata->numfiles < 0) {
    return _sion_errorprint(SION_NOT_SUCCESS, _SION_ERROR_RETURN,
      "_sion_generic_get_and_distribute_info_from_file: could not get numfiles from sion file\n");
  }

  if (sion_gendata->numfiles > 1) {
    DPRINTFP((1, "_sion_generic_get_and_distribute_info_from_file", sion_gendata->grank, "before scatter\n"));
    if (sion_gendata->grank == 0) {
      for (t = 0; t < mapping_size; t++) {
        DPRINTFP((1, "_sion_generic_get_and_distribute_info_from_file", sion_gendata->grank, "  %d -> (%d,%d)\n", t,
          mapping[t * 2], mapping[t * 2 + 1]));
      }
    }

    sion_apidesc->scatterr_cb(mapping, lpos, sion_gendata->comm_data_global, _SION_INT32, 2, 0);
    sion_gendata->filenumber = lpos[0];
    sion_gendata->lrank = lpos[1];
    DPRINTFP((1, "_sion_generic_get_and_distribute_info_from_file", sion_gendata->grank, "after scatter filenum+lrank (%d,%d)\n",
      sion_gendata->filenumber, sion_gendata->lrank));

    if (sion_gendata->grank == 0) {
      tasksinfile = malloc(sion_gendata->numfiles * sizeof(int));
      if (tasksinfile == NULL) {
        return _sion_errorprint(SION_ID_NOT_VALID, _SION_ERROR_RETURN,
          "_sion_generic_get_and_distribute_info_from_file: Cannot allocate memory for tasksinfile counter vector");
      }
      for (t = 0; t < sion_gendata->numfiles; t++) {
        tasksinfile[t] = 0; /* init counter */
      }
      for (t = 0; t < mapping_size; t++) {
        tasksinfile[mapping[t * 2]]++; /* count tasks in file  */
      }
      for (t = 0; t < mapping_size; t++) {
        mapping[t * 2 + 1] = tasksinfile[mapping[t * 2]]; /* set 2nd value of mapping to lsize  */
      }
    }
    sion_apidesc->scatterr_cb(mapping, lpos, sion_gendata->comm_data_global, _SION_INT32, 2, 0);
    sion_gendata->lsize = lpos[1];
    DPRINTFP((1, "_sion_generic_get_and_distribute_info_from_file", sion_gendata->grank, "after scatter lsize (%d, %d of %d)\n",
      sion_gendata->filenumber, sion_gendata->lrank, sion_gendata->lsize));

    if (sion_gendata->grank == 0) {
      free(tasksinfile);
    }
    /* WARNING: mapping file of sion file is now destroyed and should not be used until close  */

  } else {
    sion_gendata->filenumber = 0;
    sion_gendata->lrank = sion_gendata->grank;
    sion_gendata->lsize = sion_gendata->gsize;
    DPRINTFP((1, "_sion_generic_get_and_distribute_info_from_file", sion_gendata->grank,
      "only one file -> filenumber=%d lRank=%d\n", sion_gendata->filenumber, sion_gendata->lrank));
  }

  if (sion_gendata->grank == 0) {
    /* frees also mapping vector */
    if (sid >= 0) {
      _sion_close_sid(sid);
    }
  }

  return rc;
}

int sion_generic_parreinit(int sid, int64_t chunksize)
{
  int grank, gsize, rc;
  _sion_filedesc *sion_filedesc;
  _sion_generic_gendata *sion_gendata;

  if ((sid < 0) || (_sion_vcdtype(sid) != SION_FILEDESCRIPTOR) || !(sion_filedesc = _sion_vcdtovcon(sid))) {
    return _sion_errorprint(SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_parreinit: invalid sion_filedesc %d", sid);
  }
  sion_gendata = sion_filedesc->dataptr;

  grank = sion_gendata->grank;
  gsize = sion_gendata->gsize;

  DPRINTFP((1, "sion_generic_parreinit", grank, "enter parallel reinit of sid %d\n", sid));

  rc = _sion_parreinit_generic(sid, chunksize, grank, gsize, sion_gendata);

  DPRINTFP((1, "sion_generic_parreinit", grank, "leave parallel reinit of sid %d\n", sid));

  return rc;
}