sion_generic.c

/****************************************************************************
**  SIONLIB     http://www.fz-juelich.de/jsc/sionlib                       **
*****************************************************************************
**  Copyright (c) 2008-2019                                                **
**  Forschungszentrum Juelich, Juelich Supercomputing Centre               **
**                                                                         **
**  See the file COPYRIGHT in the package base directory for details       **
****************************************************************************/

#define _XOPEN_SOURCE 700

#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "sion_common.h"
#include "sion_const.h"
#include "sion_debug.h"
#include "sion_error_handler.h"
#include "sion_fd.h"
#include "sion_filedesc.h"
#include "sion_flags.h"
#include "sion_generic.h"
#include "sion_generic_apidesc.h"
#include "sion_generic_buddy.h"
#include "sion_generic_internal.h"
#include "sion_generic_mapped.h"
#include "sion_internal.h"

int sion_generic_create_api(char *name)
{
  int apiid = SION_ID_UNDEF;
  _sion_generic_apidesc *sion_apidesc;

  sion_apidesc = _sion_generic_alloc_apidesc();
  if (!sion_apidesc) {
    return _sion_errorprint(
      SION_ID_NOT_VALID, _SION_ERROR_RETURN, "_sion_generic_create_api: problems to alloc apidesc, aborting ...\n");
  }
  _sion_generic_init_apidesc(sion_apidesc);

  if (name == NULL) {
    return _sion_errorprint(
      SION_ID_NOT_VALID, _SION_ERROR_RETURN, "_sion_generic_create_api: name for API not given, aborting ...\n");
  }

  sion_apidesc->name = strdup(name);

  apiid = _sion_newvcd(sion_apidesc, SION_APIDESCRIPTOR);
  sion_apidesc->aid = apiid;
  sion_apidesc->level = SION_GENERIC_API_LEVEL_NONE;

  DPRINTFP((2, "_sion_generic_create_api", _SION_DEFAULT_RANK, "API created with name %s apiid=%d\n", sion_apidesc->name, apiid));

  return apiid;
}

int sion_generic_free_api(int aid)
{
  _sion_generic_apidesc *sion_apidesc;
  int rc = SION_SUCCESS;

  if ((aid < 0) || (_sion_vcdtype(aid) != SION_APIDESCRIPTOR) || !(sion_apidesc = _sion_vcdtovcon(aid))) {
    return _sion_errorprint(SION_NOT_SUCCESS, _SION_ERROR_RETURN, "sion_free_api: invalid sion_apidesc %d", aid);
  }
  _sion_freevcd(aid);
  rc = _sion_generic_free_apidesc(sion_apidesc);

  return rc;
}

int sion_generic_register_create_local_commgroup_cb(int aid, int create_lcg_cb(void **, void *, int, int, int, int, int, int))
{
  _sion_generic_apidesc *sion_apidesc;
  int rc = SION_SUCCESS;

  if ((aid < 0) || (_sion_vcdtype(aid) != SION_APIDESCRIPTOR) || !(sion_apidesc = _sion_vcdtovcon(aid))) {
    return _sion_errorprint(
      SION_NOT_SUCCESS, _SION_ERROR_RETURN, "sion_generic_register_create_local_commgroup_cb: invalid sion_apidesc %d", aid);
  }

  if (sion_apidesc->create_lcg_cb != NULL) {
    return _sion_errorprint(SION_NOT_SUCCESS, _SION_ERROR_RETURN,
      "sion_generic_register_create_local_commgroup_cb: update_cb already registered for apiid=%d", aid);
  }
  sion_apidesc->create_lcg_cb = create_lcg_cb;

  _sion_generic_update_api_level(sion_apidesc);

  DPRINTFP((2, "sion_generic_register_create_local_commgroup_cb", _SION_DEFAULT_RANK,
    "create_lcg_cb registered for apiid=%s (%d)\n", sion_apidesc->name, aid));

  return rc;
}

int sion_generic_register_free_local_commgroup_cb(int aid, int free_lcg_cb(void *))
{
  _sion_generic_apidesc *sion_apidesc;
  int rc = SION_SUCCESS;

  if ((aid < 0) || (_sion_vcdtype(aid) != SION_APIDESCRIPTOR) || !(sion_apidesc = _sion_vcdtovcon(aid))) {
    return _sion_errorprint(
      SION_NOT_SUCCESS, _SION_ERROR_RETURN, "sion_generic_register_free_local_commgroup_cb: invalid sion_apidesc %d", aid);
  }

  if (sion_apidesc->free_lcg_cb != NULL) {
    return _sion_errorprint(SION_NOT_SUCCESS, _SION_ERROR_RETURN,
      "sion_generic_register_free_local_commgroup_cb: update_cb already registered for apiid=%d", aid);
  }
  sion_apidesc->free_lcg_cb = free_lcg_cb;

  _sion_generic_update_api_level(sion_apidesc);

  DPRINTFP((2, "sion_generic_register_free_local_commgroup_cb", _SION_DEFAULT_RANK, "free_lcg_cb registered for apiid=%s (%d)\n",
    sion_apidesc->name, aid));

  return rc;
}

int sion_generic_register_barrier_cb(int aid, int barrier_cb(void *))
{
  _sion_generic_apidesc *sion_apidesc;
  int rc = SION_SUCCESS;

  if ((aid < 0) || (_sion_vcdtype(aid) != SION_APIDESCRIPTOR) || !(sion_apidesc = _sion_vcdtovcon(aid))) {
    return _sion_errorprint(
      SION_NOT_SUCCESS, _SION_ERROR_RETURN, "sion_generic_register_barrier_cb: invalid sion_apidesc %d", aid);
  }

  if (sion_apidesc->barrier_cb != NULL) {
    return _sion_errorprint(
      SION_NOT_SUCCESS, _SION_ERROR_RETURN, "sion_generic_register_barrier_cb: barrier_cb already registered for apiid=%d", aid);
  }
  sion_apidesc->barrier_cb = barrier_cb;

  _sion_generic_update_api_level(sion_apidesc);

  DPRINTFP((2, "sion_generic_register_barrier_cb", _SION_DEFAULT_RANK, "barrier_cb registered for apiid=%s (%d)\n",
    sion_apidesc->name, aid));

  return rc;
}

int sion_generic_register_bcastr_cb(int aid, int bcastr_cb(void *, void *, int, int, int))
{
  _sion_generic_apidesc *sion_apidesc;
  int rc = SION_SUCCESS;

  if ((aid < 0) || (_sion_vcdtype(aid) != SION_APIDESCRIPTOR) || !(sion_apidesc = _sion_vcdtovcon(aid))) {
    return _sion_errorprint(
      SION_NOT_SUCCESS, _SION_ERROR_RETURN, "sion_generic_register_bcastr_cb: invalid sion_apidesc %d", aid);
  }

  if (sion_apidesc->bcastr_cb != NULL) {
    return _sion_errorprint(
      SION_NOT_SUCCESS, _SION_ERROR_RETURN, "sion_generic_register_bcastr_cb: bcastr_cb already registered for apiid=%d", aid);
  }
  sion_apidesc->bcastr_cb = bcastr_cb;

  _sion_generic_update_api_level(sion_apidesc);

  DPRINTFP((2, "sion_generic_register_bcastr_cb", _SION_DEFAULT_RANK, "bcastr_cb registered for apiid=%s (%d)\n",
    sion_apidesc->name, aid));

  return rc;
}

int sion_generic_register_gatherr_cb(int aid, int gatherr_cb(void *, void *, void *, int, int, int))
{
  _sion_generic_apidesc *sion_apidesc;
  int rc = SION_SUCCESS;

  if ((aid < 0) || (_sion_vcdtype(aid) != SION_APIDESCRIPTOR) || !(sion_apidesc = _sion_vcdtovcon(aid))) {
    return _sion_errorprint(
      SION_NOT_SUCCESS, _SION_ERROR_RETURN, "sion_generic_register_gatherr_cb: invalid sion_apidesc %d", aid);
  }

  if (sion_apidesc->gatherr_cb != NULL) {
    return _sion_errorprint(
      SION_NOT_SUCCESS, _SION_ERROR_RETURN, "sion_generic_register_gatherr_cb: gatherr_cb already registered for apiid=%d", aid);
  }
  sion_apidesc->gatherr_cb = gatherr_cb;

  _sion_generic_update_api_level(sion_apidesc);

  DPRINTFP((2, "sion_generic_register_gatherr_cb", _SION_DEFAULT_RANK, "gatherr_cb registered for apiid=%s (%d)\n",
    sion_apidesc->name, aid));

  return rc;
}

int sion_generic_register_gathervr_cb(int aid, int gathervr_cb(void *, void *, void *, int, int *, int, int))
{
  _sion_generic_apidesc *sion_apidesc;
  int rc = SION_SUCCESS;

  if ((aid < 0) || (_sion_vcdtype(aid) != SION_APIDESCRIPTOR) || !(sion_apidesc = _sion_vcdtovcon(aid))) {
    return _sion_errorprint(
      SION_NOT_SUCCESS, _SION_ERROR_RETURN, "sion_generic_register_gathervr_cb: invalid sion_apidesc %d", aid);
  }

  if (sion_apidesc->gathervr_cb != NULL) {
    return _sion_errorprint(SION_NOT_SUCCESS, _SION_ERROR_RETURN,
      "sion_generic_register_gathervr_cb: gathervr_cb already registered for apiid=%d", aid);
  }
  sion_apidesc->gathervr_cb = gathervr_cb;

  _sion_generic_update_api_level(sion_apidesc);

  DPRINTFP((2, "sion_generic_register_gathervr_cb", _SION_DEFAULT_RANK, "gathervr_cb registered for apiid=%s (%d)\n",
    sion_apidesc->name, aid));

  return rc;
}

int sion_generic_register_scatterr_cb(int aid, int scatterr_cb(void *, void *, void *, int, int, int))
{
  _sion_generic_apidesc *sion_apidesc;
  int rc = SION_SUCCESS;

  if ((aid < 0) || (_sion_vcdtype(aid) != SION_APIDESCRIPTOR) || !(sion_apidesc = _sion_vcdtovcon(aid))) {
    return _sion_errorprint(
      SION_NOT_SUCCESS, _SION_ERROR_RETURN, "sion_generic_register_scatterr_cb: invalid sion_apidesc %d", aid);
  }

  if (sion_apidesc->scatterr_cb != NULL) {
    return _sion_errorprint(SION_NOT_SUCCESS, _SION_ERROR_RETURN,
      "sion_generic_register_scatterr_cb: scatterr_cb already registered for apiid=%d", aid);
  }
  sion_apidesc->scatterr_cb = scatterr_cb;

  _sion_generic_update_api_level(sion_apidesc);

  DPRINTFP((2, "sion_generic_register_scatterr_cb", _SION_DEFAULT_RANK, "scatterr_cb registered for apiid=%s (%d)\n",
    sion_apidesc->name, aid));

  return rc;
}

int sion_generic_register_scattervr_cb(int aid, int scattervr_cb(void *, void *, void *, int, int *, int, int))
{
  _sion_generic_apidesc *sion_apidesc;
  int rc = SION_SUCCESS;

  if ((aid < 0) || (_sion_vcdtype(aid) != SION_APIDESCRIPTOR) || !(sion_apidesc = _sion_vcdtovcon(aid))) {
    return _sion_errorprint(
      SION_NOT_SUCCESS, _SION_ERROR_RETURN, "sion_generic_register_scattervr_cb: invalid sion_apidesc %d", aid);
  }

  if (sion_apidesc->scattervr_cb != NULL) {
    return _sion_errorprint(SION_NOT_SUCCESS, _SION_ERROR_RETURN,
      "sion_generic_register_scattervr_cb: scattervr_cb already registered for apiid=%d", aid);
  }
  sion_apidesc->scattervr_cb = scattervr_cb;

  _sion_generic_update_api_level(sion_apidesc);

  DPRINTFP((2, "sion_generic_register_scattervr_cb", _SION_DEFAULT_RANK, "scattervr_cb registered for apiid=%s (%d)\n",
    sion_apidesc->name, aid));

  return rc;
}

int sion_generic_register_gather_and_execute_cb(int aid,
  int64_t gather_execute_cb(const void *, int64_t, void *, int, int, int, int, int64_t process_cb(const void *, int64_t *, int),
    void *, bool spec_iterator_cb(void *, int64_t *)))
{
  _sion_generic_apidesc *sion_apidesc;
  int rc = SION_SUCCESS;

  if ((aid < 0) || (_sion_vcdtype(aid) != SION_APIDESCRIPTOR) || !(sion_apidesc = _sion_vcdtovcon(aid))) {
    return _sion_errorprint(
      SION_NOT_SUCCESS, _SION_ERROR_RETURN, "sion_generic_register_gather_and_execute_cb: invalid sion_apidesc %d", aid);
  }

  if (sion_apidesc->gather_execute_cb != NULL) {
    return _sion_errorprint(SION_NOT_SUCCESS, _SION_ERROR_RETURN,
      "sion_generic_register_gather_and_execute_cb: scattervr_cb already registered for apiid=%d", aid);
  }
  sion_apidesc->gather_execute_cb = gather_execute_cb;

  _sion_generic_update_api_level(sion_apidesc);

  DPRINTFP((2, "sion_generic_register_gather_and_execute_cb", _SION_DEFAULT_RANK,
    "gather_execute_cb registered for apiid=%s (%d)\n", sion_apidesc->name, aid));

  return rc;
}

int sion_generic_register_execute_and_scatter_cb(int aid,
  int64_t execute_scatter_cb(void *, int64_t, void *, int, int, int, int, int64_t process_cb(void *, int64_t *, int), void *,
    bool spec_iterator_cb(void *, int64_t *)))
{
  _sion_generic_apidesc *sion_apidesc;
  int rc = SION_SUCCESS;

  if ((aid < 0) || (_sion_vcdtype(aid) != SION_APIDESCRIPTOR) || !(sion_apidesc = _sion_vcdtovcon(aid))) {
    return _sion_errorprint(
      SION_NOT_SUCCESS, _SION_ERROR_RETURN, "sion_generic_register_execute_and_scatter_cb: invalid sion_apidesc %d", aid);
  }

  if (sion_apidesc->execute_scatter_cb != NULL) {
    return _sion_errorprint(SION_NOT_SUCCESS, _SION_ERROR_RETURN,
      "sion_generic_register_execute_and_scatter_cb: scattervr_cb already registered for apiid=%d", aid);
  }
  sion_apidesc->execute_scatter_cb = execute_scatter_cb;

  DPRINTFP((2, "sion_generic_register_execute_and_scatter_cb", _SION_DEFAULT_RANK,
    "gather_execute_cb registered for apiid=%s (%d)\n", sion_apidesc->name, aid));

  return rc;
}

int sion_generic_register_get_multi_filename_cb(int aid, char *get_multi_filename_cb(const char *, int))
{
  _sion_generic_apidesc *sion_apidesc;
  int rc = 0;

  if ((aid < 0) || (_sion_vcdtype(aid) != SION_APIDESCRIPTOR) || !(sion_apidesc = _sion_vcdtovcon(aid))) {
    return _sion_errorprint(-1, _SION_ERROR_RETURN, "sion_generic_register_get_multi_filename_cb: invalid sion_apidesc %d", aid);
  }

  if (sion_apidesc->get_multi_filename_cb != NULL) {
    return _sion_errorprint(-1, _SION_ERROR_RETURN,
      "sion_generic_register_get_multi_filename_cb: get_multi_filename_cb already registered for apiid=%d", aid);
  }
  sion_apidesc->get_multi_filename_cb = get_multi_filename_cb;

  _sion_generic_update_api_level(sion_apidesc);

  DPRINTFP((2, "sion_generic_register_get_multi_filename_cb", -1, "get_multi_filename_cb registered for apiid=%s (%d)\n",
    sion_apidesc->name, aid));

  return rc;
}

int sion_generic_register_get_capability_cb(int aid, int get_capability_cb(void *))
{
  _sion_generic_apidesc *sion_apidesc;
  int rc = SION_SUCCESS;

  if ((aid < 0) || (_sion_vcdtype(aid) != SION_APIDESCRIPTOR) || !(sion_apidesc = _sion_vcdtovcon(aid))) {
    return _sion_errorprint(
      SION_NOT_SUCCESS, _SION_ERROR_RETURN, "sion_generic_register_get_capability_cb: invalid sion_apidesc %d", aid);
  }

  if (sion_apidesc->get_capability_cb != NULL) {
    return _sion_errorprint(SION_NOT_SUCCESS, _SION_ERROR_RETURN,
      "sion_generic_register_get_capability_cb: get_capability_cb already registered for apiid=%d", aid);
  }
  sion_apidesc->get_capability_cb = get_capability_cb;

  _sion_generic_update_api_level(sion_apidesc);

  DPRINTFP((2, "sion_generic_register_get_capability_cb", _SION_DEFAULT_RANK, "get_capability_cb registered for apiid=%s (%d)\n",
    sion_apidesc->name, aid));

  return rc;
}

int sion_generic_parclose(int sid)
{
  ONLY_DEBUG(int grank);
  int rc = SION_SUCCESS;
  _sion_filedesc *sion_filedesc;
  _sion_generic_gendata *sion_gendata;
  _sion_generic_apidesc *sion_apidesc;

  DPRINTFTS(-1, "enter sion_generic_parclose");

  if ((sid < 0) || (_sion_vcdtype(sid) != SION_FILEDESCRIPTOR) || !(sion_filedesc = _sion_vcdtovcon(sid))) {
    return _sion_errorprint(SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_parclose: invalid sion_filedesc %d", sid);
  }
  sion_gendata = sion_filedesc->dataptr;
  sion_apidesc = sion_gendata->apidesc;

  ONLY_DEBUG(grank = sion_gendata->grank);

  DPRINTFP((1, "sion_generic_parclose", grank, "enter parallel close of sid %d (%d file)\n", sid, sion_filedesc->nfiles));

  if (!sion_filedesc->usebuddy) {
    rc = _sion_parclose_generic(sid, sion_filedesc->rank, sion_filedesc->ntasks, _SION_INTERNAL_FLAG_NORMAL, sion_gendata, NULL);
  } else {
    rc = _sion_parclose_generic_buddy(sid, sion_filedesc->rank, sion_filedesc->ntasks, sion_gendata);
  }

  if (sion_apidesc->free_lcg_cb && sion_gendata->comm_data_local) {
    sion_apidesc->free_lcg_cb(sion_gendata->comm_data_local);
  }

  _sion_freevcd(sid);

  sion_apidesc->barrier_cb(sion_gendata->comm_data_global);

  free(sion_gendata);

  DPRINTFP((1, "sion_generic_parclose", grank, "leave parallel close of sid %d\n", sid));
  DPRINTFTS(-1, "leave sion_generic_parclose");

  return rc;
}

int sion_generic_paropen_mapped(int aid, char *fname, const char *file_mode, int *numFiles, void *gcommgroup, int grank,
  int gsize, int *nlocaltasks, int **globalranks, int64_t **chunksizes, int **mapping_filenrs, int **mapping_lranks,
  int32_t *fsblksize, FILE **fileptr)
{
  int sid;
  _sion_flags_store *flags_store = _sion_parse_flags(file_mode);
  if (!flags_store) {
    return _sion_errorprint(SION_ID_NOT_VALID, _SION_ERROR_RETURN,
      "sion_generic_paropen_mapped: could not parse file mode in %s, aborting ...\n", file_mode);
  }
  sid = _sion_generic_paropen_mapped(aid, fname, flags_store, numFiles, gcommgroup, grank, gsize, nlocaltasks, globalranks,
    chunksizes, mapping_filenrs, mapping_lranks, fsblksize, fileptr);
  _sion_flags_destroy_store(&flags_store);
  return sid;
}

int _sion_generic_paropen_mapped(int aid, char *fname, const _sion_flags_store *flags_store, int *numFiles, void *gcommgroup,
  int grank, int gsize, int *nlocaltasks, int **globalranks, int64_t **chunksizes, int **mapping_filenrs, int **mapping_lranks,
  int32_t *fsblksize, FILE **fileptr)
{
  int rc, sid = SION_ID_UNDEF;
  _sion_filedesc *sion_filedesc;
  _sion_generic_apidesc *sion_apidesc;
  _sion_generic_gendata *sion_gendata;

  DPRINTFTS(grank, "enter sion_generic_paropen_mapped");
  DPRINTFP((1, "sion_generic_paropen_mapped", grank,
    "enter generic parallel mapped open of file %s (global: %d of %d, nlocaltasks %d)\n", fname, grank, gsize, *nlocaltasks));

  if ((aid < 0) || (_sion_vcdtype(aid) != SION_APIDESCRIPTOR) || !(sion_apidesc = _sion_vcdtovcon(aid))) {
    return _sion_errorprint(
      SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_register_scattervr_cb: invalid sion_apidesc %d", aid);
  }

  if ((!sion_apidesc->create_lcg_cb) || (!sion_apidesc->free_lcg_cb) || (!sion_apidesc->barrier_cb) || (!sion_apidesc->bcastr_cb)
    || (!sion_apidesc->gatherr_cb) || (!sion_apidesc->gathervr_cb) || (!sion_apidesc->scatterr_cb)
    || (!sion_apidesc->scattervr_cb)) {
    return _sion_errorprint(SION_ID_NOT_VALID, _SION_ERROR_RETURN,
      "sion_generic_paropen_mapped: API %s not correctly initalized, aborting", sion_apidesc->name);
  }

  /* check parameters */
  if ((grank < 0) || (grank >= gsize)) {
    return _sion_errorprint(
      SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_paropen_mapped: global rank %d not valid (0..%d)", grank, gsize);
  }

  /* create data container */
  sion_gendata = _sion_generic_alloc_gendata();
  if (!sion_gendata) {
    return _sion_errorprint(
      SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_paropen_mapped: problems to alloc gendata, aborting ...\n");
  }
  _sion_generic_init_gendata(sion_gendata);

  sion_gendata->aid = aid;
  sion_gendata->apidesc = sion_apidesc;
  sion_gendata->comm_data_global = gcommgroup;
  sion_gendata->grank = grank;
  sion_gendata->gsize = gsize;
  sion_gendata->lrank = -1;
  sion_gendata->lsize = -1;
  sion_gendata->filenumber = -1;
  sion_gendata->numfiles = -1;
  sion_gendata->comm_data_local = NULL;

  if (!(flags_store->mask & (_SION_FMODE_WRITE | _SION_FMODE_READ))) {
    return _sion_errorprint(SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_paropen_mapped: unknown file mode");
  }

  sid = _sion_newvcd(NULL, SION_FILEDESCRIPTOR);

  // FIXME: re-enable debug output of file mode
  // DPRINTFP((1, "sion_generic_paropen_mapped", grank, "enter parallel mapped open  in %s mode\n", file_mode));
  DPRINTFP((1, "sion_generic_paropen_mapped", grank, "enter parallel mapped open\n"));
  rc = _sion_paropen_mapped_generic(sid, fname, flags_store->mask, numFiles, nlocaltasks, globalranks, chunksizes,
    mapping_filenrs, mapping_lranks, fsblksize, grank, gsize, _SION_INTERNAL_FLAG_NORMAL, fileptr, sion_gendata);
  sion_gendata->numfiles = *numFiles;
  // FIXME: re-enable debug output of file mode
  // DPRINTFP((1, "sion_generic_paropen_mapped", sion_gendata->grank, "leave parallel open of %d files in %s mode #tasks=%d sid=%d globalrank=%d\n", sion_gendata->numfiles,
  //                               file_mode, sion_gendata->lsize, sid, sion_gendata->grank));
  DPRINTFP(
    (1, "sion_generic_paropen_mapped", sion_gendata->grank, "leave parallel open of %d files #tasks=%d sid=%d globalrank=%d\n",
      sion_gendata->numfiles, sion_gendata->lsize, sid, sion_gendata->grank));

  /* test sid again and get internal data structure */
  if ((sid < 0) || (_sion_vcdtype(sid) != SION_FILEDESCRIPTOR) || !(sion_filedesc = _sion_vcdtovcon(sid))) {
    return _sion_errorprint(SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_paropen_mapped: invalid sion_filedesc %d", sid);
  }

  /* store additional data */
  sion_filedesc->dataptr = sion_gendata;

  /* Bug? */

  DPRINTFP((1, "sion_generic_paropen_mapped", grank, "leave parallel open of file %s sid=%d\n", fname, sid));
  DPRINTFTS(-1, "leave sion_generic_paropen_mapped");

  return rc;
}

int sion_generic_parclose_mapped(int sid)
{
  int grank, gsize, rc;
  _sion_filedesc *sion_filedesc;
  _sion_generic_gendata *sion_gendata;

  DPRINTFTS(-1, "enter sion_generic_parclose_mapped");
  if ((sid < 0) || (_sion_vcdtype(sid) != SION_FILEDESCRIPTOR) || !(sion_filedesc = _sion_vcdtovcon(sid))) {
    return _sion_errorprint(SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_parclose_mapped: invalid sion_filedesc %d", sid);
  }
  sion_gendata = sion_filedesc->dataptr;

  grank = sion_gendata->grank;
  gsize = sion_gendata->gsize;

  DPRINTFP((1, "sion_generic_parclose_mapped", grank, "enter parallel close of sid %d\n", sid));

  DPRINTFP((1, "sion_generic_parclose_mapped", grank, "closing %d file(s)\n", sion_filedesc->nfiles));

  rc = _sion_parclose_mapped_generic(sid, grank, gsize, sion_gendata);

  _sion_freevcd(sid);

  _sion_generic_free_gendata(sion_gendata);

  DPRINTFP((1, "sion_generic_parclose_mapped", grank, "leave parallel close of sid %d\n", sid));
  DPRINTFTS(-1, "leave sion_generic_parclose_mapped");

  return rc;
}

int _sion_generic_get_and_distribute_info_from_file(_sion_generic_gendata *sion_gendata, const char *fname)
{
  int sid = -1, t = 0;
  int rc = SION_SUCCESS;
  int *tasksinfile;
  int mapping_size = -1;
  int32_t *mapping = NULL;
  int32_t lpos[2];
  _sion_generic_apidesc *sion_apidesc;

  sion_apidesc = sion_gendata->apidesc;

  if (sion_gendata->grank == 0) {
    /* open and get mapping of sion file */
    DPRINTFP((1, "_sion_generic_get_and_distribute_info_from_file", sion_gendata->grank, "before open\n"));
    sid = _sion_open_read(fname, _SION_READ_MASTER_ONLY_OF_MULTI_FILES, &SION_OPTIONS_INIT);
    if (sid >= 0) {
      DPRINTFP((1, "_sion_generic_get_and_distribute_info_from_file", sion_gendata->grank, "after open\n"));
      rc = sion_get_mapping(sid, &mapping_size, &mapping, &sion_gendata->numfiles);
      DPRINTFP((1, "_sion_generic_get_and_distribute_info_from_file", sion_gendata->grank, "sion file %d files\n",
        sion_gendata->numfiles));
    } else {
      sion_gendata->numfiles = -1;
    }
  }

  /* each task has to know if more than file was used in sion file */
  sion_apidesc->bcastr_cb(&sion_gendata->numfiles, sion_gendata->comm_data_global, _SION_INT32, 1, 0);

  if ((sion_gendata->grank == 0) && (sion_gendata->numfiles > 1)) {
    if (mapping_size != sion_gendata->gsize) {
      return _sion_errorprint(SION_NOT_SUCCESS, _SION_ERROR_RETURN,
        "_sion_generic_get_and_distribute_info_from_file: Incorrect sum of ntasks of files %d <> %d\n", mapping_size,
        sion_gendata->gsize);
    }
  }

  if (sion_gendata->numfiles < 0) {
    return _sion_errorprint(SION_NOT_SUCCESS, _SION_ERROR_RETURN,
      "_sion_generic_get_and_distribute_info_from_file: could not get numfiles from sion file\n");
  }

  if (sion_gendata->numfiles > 1) {
    DPRINTFP((1, "_sion_generic_get_and_distribute_info_from_file", sion_gendata->grank, "before scatter\n"));
    if (sion_gendata->grank == 0) {
      for (t = 0; t < mapping_size; t++) {
        DPRINTFP((1, "_sion_generic_get_and_distribute_info_from_file", sion_gendata->grank, "  %d -> (%d,%d)\n", t,
          mapping[t * 2], mapping[t * 2 + 1]));
      }
    }

    sion_apidesc->scatterr_cb(mapping, lpos, sion_gendata->comm_data_global, _SION_INT32, 2, 0);
    sion_gendata->filenumber = lpos[0];
    sion_gendata->lrank = lpos[1];
    DPRINTFP((1, "_sion_generic_get_and_distribute_info_from_file", sion_gendata->grank, "after scatter filenum+lrank (%d,%d)\n",
      sion_gendata->filenumber, sion_gendata->lrank));

    if (sion_gendata->grank == 0) {
      tasksinfile = malloc(sion_gendata->numfiles * sizeof(int));
      if (tasksinfile == NULL) {
        return _sion_errorprint(SION_ID_NOT_VALID, _SION_ERROR_RETURN,
          "_sion_generic_get_and_distribute_info_from_file: Cannot allocate memory for tasksinfile counter vector");
      }
      for (t = 0; t < sion_gendata->numfiles; t++) {
        tasksinfile[t] = 0; /* init counter */
      }
      for (t = 0; t < mapping_size; t++) {
        tasksinfile[mapping[t * 2]]++; /* count tasks in file  */
      }
      for (t = 0; t < mapping_size; t++) {
        mapping[t * 2 + 1] = tasksinfile[mapping[t * 2]]; /* set 2nd value of mapping to lsize  */
      }
    }
    sion_apidesc->scatterr_cb(mapping, lpos, sion_gendata->comm_data_global, _SION_INT32, 2, 0);
    sion_gendata->lsize = lpos[1];
    DPRINTFP((1, "_sion_generic_get_and_distribute_info_from_file", sion_gendata->grank, "after scatter lsize (%d, %d of %d)\n",
      sion_gendata->filenumber, sion_gendata->lrank, sion_gendata->lsize));

    if (sion_gendata->grank == 0) {
      free(tasksinfile);
    }
    /* WARNING: mapping file of sion file is now destroyed and should not be used until close  */

  } else {
    sion_gendata->filenumber = 0;
    sion_gendata->lrank = sion_gendata->grank;
    sion_gendata->lsize = sion_gendata->gsize;
    DPRINTFP((1, "_sion_generic_get_and_distribute_info_from_file", sion_gendata->grank,
      "only one file -> filenumber=%d lRank=%d\n", sion_gendata->filenumber, sion_gendata->lrank));
  }

  if (sion_gendata->grank == 0) {
    /* frees also mapping vector */
    if (sid >= 0) {
      _sion_close_sid(sid);
    }
  }

  return rc;
}

int sion_generic_parreinit(int sid, int64_t chunksize)
{
  int grank, gsize, rc;
  _sion_filedesc *sion_filedesc;
  _sion_generic_gendata *sion_gendata;

  if ((sid < 0) || (_sion_vcdtype(sid) != SION_FILEDESCRIPTOR) || !(sion_filedesc = _sion_vcdtovcon(sid))) {
    return _sion_errorprint(SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_parreinit: invalid sion_filedesc %d", sid);
  }
  sion_gendata = sion_filedesc->dataptr;

  grank = sion_gendata->grank;
  gsize = sion_gendata->gsize;

  DPRINTFP((1, "sion_generic_parreinit", grank, "enter parallel reinit of sid %d\n", sid));

  rc = _sion_parreinit_generic(sid, chunksize, grank, gsize, sion_gendata);

  DPRINTFP((1, "sion_generic_parreinit", grank, "leave parallel reinit of sid %d\n", sid));

  return rc;
}

sion_generic_options *sion_generic_options_new()
{
  sion_generic_options *options = malloc(sizeof(sion_generic_options));
  if (options) {
    *options = SION_GENERIC_OPTIONS_INIT;
  }
  return options;
}

void sion_generic_options_delete(sion_generic_options *options)
{
  free(options);
}

void sion_generic_options_set_chunksize(sion_generic_options *options, int64_t chunksize)
{
  options->chunksize = chunksize;
}

void sion_generic_options_set_fsblksize(sion_generic_options *options, int32_t fsblksize)
{
  options->fsblksize = fsblksize;
}

void sion_generic_options_set_keyval_mode(sion_generic_options *options, sion_keyval_mode keyval_mode)
{
  options->keyval_mode = keyval_mode;
}

void sion_generic_options_set_buddy(sion_generic_options *options)
{
  sion_generic_options_set_buddylevel(options, 1);
}

void sion_generic_options_set_buddylevel(sion_generic_options *options, int32_t buddylevel)
{
  options->buddylevel = buddylevel;
}

void sion_generic_options_set_collective(sion_generic_options *options)
{
  options->collsize = -1;
}

void sion_generic_options_set_collective_size(sion_generic_options *options, int32_t size)
{
  options->collsize = size;
}

void sion_generic_options_set_collective_merge(sion_generic_options *options)
{
  if (options->collsize == 0) {
    options->collsize = -1;
  }
  options->collective_merge = true;
}

void sion_generic_options_set_lowlevel_api(sion_generic_options *options, sion_lowlevel_api lowlevel_api)
{
  options->lowlevel_api = lowlevel_api;
}

void sion_generic_options_set_endianness(sion_generic_options *options, sion_endianness endianness)
{
  options->endianness = endianness;
}

int sion_generic_paropen(int aid, const char *fname, sion_open_mode mode, void *gcommgroup, int grank, int gsize, int filenumber,
  int numfiles, int lrank, int lsize, const sion_generic_options *options_)
{
  sion_generic_options options = (options_) ? *options_ : SION_GENERIC_OPTIONS_INIT;

  DPRINTFTS(grank, "enter sion_generic_paropen");
  DPRINTFP((1, "sion_generic_paropen", grank,
    "enter generic parallel open of FILE %s (global: %d of %d, local: %d of %d, file: %d of %d )\n", fname, grank, gsize, lrank,
    lsize, filenumber, numfiles));

  _sion_generic_apidesc *apidesc;
  if ((aid < 0) || (_sion_vcdtype(aid) != SION_APIDESCRIPTOR) || !(apidesc = _sion_vcdtovcon(aid))) {
    return _sion_errorprint(SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_register_scattervr_cb: invalid apidesc %d", aid);
  }

  if ((apidesc->level != SION_GENERIC_API_LEVEL_STD) && (apidesc->level != SION_GENERIC_API_LEVEL_FULL)) {
    return _sion_errorprint(
      SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_paropen: API %s not correctly initalized, aborting", apidesc->name);
  }

  /* check parameters */
  if ((grank < 0) || (grank >= gsize)) {
    return _sion_errorprint(
      SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_paropen: global rank %d not valid (0..%d)", grank, gsize);
  }

#ifdef HACK_SCALASCA
  /* hack for scalasca */
  if (strstr(fname, ".def") != NULL) {
    options.fsblksize = 65536;
    options.chunksize = 65536;
  }
#endif
  DPRINTFP((1, "sion_generic_paropen", grank, "setting fsblksize = %d for file %s\n", options.fsblksize, fname));

  /* create data container */
  _sion_generic_gendata *gendata = _sion_generic_alloc_gendata();
  if (!gendata) {
    return _sion_errorprint(
      SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_paropen: problems to alloc gendata, aborting ...\n");
  }
  _sion_generic_init_gendata(gendata);

  gendata->aid = aid;
  gendata->apidesc = apidesc;
  gendata->comm_data_global = gcommgroup;
  gendata->comm_data_local = NULL;
  gendata->grank = grank;
  gendata->gsize = gsize;

  if (mode == SION_OPEN_WRITE) {
    if ((lrank < 0) || (lrank >= lsize)) {
      return _sion_errorprint(
        SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_paropen: local rank %d not valid (0..%d)", lrank, lsize);
    }
    gendata->lrank = lrank;
    gendata->lsize = lsize;

    if ((filenumber < 0) || (filenumber >= numfiles)) {
      return _sion_errorprint(
        SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_paropen: file number %d not valid (0..%d)", filenumber, numfiles);
    }
    gendata->filenumber = filenumber;
    gendata->numfiles = numfiles;

  } else if (mode == SION_OPEN_READ) {
    /* file mode READ */

    DPRINTFP((1, "sion_generic_paropen", grank, "READ: buddylevel=%d\n", options.buddylevel));

    /* get number of files and filenumber */
    if (!options.buddylevel) {
      int rc = _sion_generic_get_and_distribute_info_from_file(gendata, fname);
      if (rc != SION_SUCCESS) {
        return _sion_errorprint(SION_ID_NOT_VALID, _SION_ERROR_RETURN,
          "sion_generic_paropen: error in _sion_generic_get_and_distribute_info_from_file");
      }
    } else {
      /* buddy will determine file layout later, but need local commgroup to optimize file operations */

      gendata->lrank = lrank;
      gendata->lsize = lsize;

      if ((filenumber < 0) || (filenumber >= numfiles)) {
        return _sion_errorprint(
          SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_paropen: file number %d not valid (0..%d)", filenumber, numfiles);
      }
      gendata->filenumber = filenumber;
      gendata->numfiles = numfiles;
    }

  } else {
    return _sion_errorprint(SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_paropen: unknown file mode");
  }

  /* generate local commgroup according to information from file (read) or parameter (write) */
  int rc = apidesc->create_lcg_cb(&gendata->comm_data_local, gendata->comm_data_global, gendata->grank, gendata->gsize,
    gendata->lrank, gendata->lsize, gendata->filenumber, gendata->numfiles);
  if (rc) {
    return _sion_errorprint(
      SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_paropen_generic: error in create_local_commgroup callback function");
  }

  /* set up parameters of call to generic open  */
  char *nfname =
    (apidesc->get_multi_filename_cb ? apidesc->get_multi_filename_cb : _sion_get_multi_filename)(fname, gendata->filenumber);

  int sid = _sion_newvcd(NULL, SION_FILEDESCRIPTOR);

  if (!options.buddylevel) {
    /* STANDARD  */
    /* FIXME: add debug output for flags_store
    DPRINTFP((1, "sion_generic_paropen", grank, "call parallel open of %d files (current name %s) in %s mode\n", gendata->numfiles, nfname, file_mode)); */
    DPRINTFP((1, "sion_generic_paropen", grank, "call parallel open of %d files (current name %s)\n", gendata->numfiles, nfname));
    rc = _sion_paropen_generic_one_file(sid, nfname, mode, gendata->numfiles, gendata->filenumber, gendata->lrank, gendata->lsize,
      gendata->grank, _SION_INTERNAL_FLAG_NORMAL, gendata, NULL, &options);
    /* FIXME: add debug output for flags_store
    DPRINTFP((1, "sion_generic_paropen", gendata->grank, "leave parallel open of %d files in %s mode #tasks=%d sid=%d globalrank=%d\n", gendata->numfiles,
        file_mode, gendata->lsize, sid, gendata->grank)); */
    DPRINTFP((1, "sion_generic_paropen", gendata->grank, "leave parallel open of %d files #tasks=%d sid=%d globalrank=%d\n",
      gendata->numfiles, gendata->lsize, sid, gendata->grank));

  } else {
    /* branch BUDDY Checkpointing  */
    /* FIXME: add debug output for flags_store
    DPRINTFP((1, "sion_generic_paropen", grank, "call parallel buddy open of %d files (current name %s) in %s mode\n", gendata->numfiles, nfname, file_mode)); */
    DPRINTFP(
      (1, "sion_generic_paropen", grank, "call parallel buddy open of %d files (current name %s)\n", gendata->numfiles, nfname));
    rc = _sion_paropen_generic_buddy(sid, fname, mode, gendata->numfiles, gendata->filenumber, gendata->lrank, gendata->lsize,
      gendata->grank, gendata, &options);
    /* FIXME: add debug output for flags_store
    DPRINTFP((1, "sion_generic_paropen", gendata->grank, "leave parallel buddy open of %d files in %s mode #tasks=%d sid=%d globalrank=%d rc=%d\n", gendata->numfiles,
        file_mode, gendata->lsize, sid, gendata->grank, rc)); */
    DPRINTFP(
      (1, "sion_generic_paropen", gendata->grank, "leave parallel buddy open of %d files #tasks=%d sid=%d globalrank=%d rc=%d\n",
        gendata->numfiles, gendata->lsize, sid, gendata->grank, rc));
  }

  /* test sid and get internal data structure */
  _sion_filedesc *filedesc;
  if ((rc < 0) || (sid < 0) || (_sion_vcdtype(sid) != SION_FILEDESCRIPTOR) || !(filedesc = _sion_vcdtovcon(sid))) {
    DPRINTFP((1, "sion_generic_paropen", gendata->grank, "invalid rc %d or filedesc %d", rc, sid));
    return _sion_errorprint(SION_ID_NOT_VALID, _SION_ERROR_RETURN, "sion_generic_paropen: invalid rc %d or filedesc %d", rc, sid);
  }

  free(nfname);

  /* Bug? */

  DPRINTFP((1, "sion_generic_paropen", grank, "leave parallel open of file %s sid=%d\n", fname, sid));
  DPRINTFTS(-1, "leave sion_generic_paropen");

  return sid;
}