/****************************************************************************
**  SIONLIB     http://www.fz-juelich.de/jsc/sionlib                       **
*****************************************************************************
**  Copyright (c) 2008-2018                                                **
**  Forschungszentrum Juelich, Juelich Supercomputing Centre               **
**                                                                         **
**  See the file COPYRIGHT in the package base directory for details       **
****************************************************************************/

#ifndef SION_SION_GENERIC_H
#define SION_SION_GENERIC_H

/*!
 * \defgroup generic Generic API
 * @{
 *
 * \brief an abstraction of communication between parallel tasks
 *
 * Introduction
 * ------------
 *
 * The generic API of SIONlib is designed to implement a user-defined
 * SIONlib parallel API. This new feature of SIONlib enables also SIONlib
 * for those application, not using one of the SIONlib standard
 * communication layers like MPI, OpenMP, or hybrid (MPI+OpenMP).
 *
 * SIONlib requires only a few communication functions to collect and
 * distributed meta-data information during open- and close-call among
 * the parallel tasks. Most of these communication functions are
 * collective operation like bcast, gather, or scatter.
 *
 *
 * Communications Groups
 * ---------------------
 *
 * According to the existing SIONlib standard communication layer,
 * SIONlib uses two different groups of tasks:
 *
 * - Global
 *   - This communication group includes all tasks which will be part of
 *     the SIONlib file operation (e.g. `MPI_COMM_WORLD`).
 * - Local
 *   - SIONlib uses this communination group to collect those tasks which
 *     accesses same physical file.
 *
 * A data structure has to be defined and allocated for the global
 * commgroup before lsion_open call and a pointer to the data structure
 * has to be passed to lsion_open as a parameter.
 *
 * The local commgroup has to be created during sion file
 * open. Therefore, a callback function fore create and free such a
 * commgroup has to be registered.  The callback functions for
 * communication will get a pointer to one of these commgroup data
 * structures afterwards as a parameter.
 *
 * SIONlib uses both commgroups for communication: the global commgroup
 * for example to collect and distribute task-to-file mapping
 * information. The local commgroup will be used to collect and
 * distribute file meta information (e.g. chunksizes).  Callback
 * functions are therefore required to be able to perform the implemented
 * communication operation on both commgroup.
 *
 * The second, local communication group will be defined by SIONlib
 * according to the parameters filenumber, numfiles, lrank and lsize
 * given by each task to the lsion_generic_paropen call. For the
 * definition of the commgroup SIONlib also uses a callback function
 * internally, which has to create the commgroup and store the data into
 * the corresponding data structure.
 *
 * Remarks:
 * - The communination groups are not dynamic. This means, the
 *   communication groups must not be changed after a file is opened.
 *
 * Usage
 * -----
 *
 * For using the generic SIONlib API the following steps are required:
 *
 * ### Definition phase
 *
 * - Definition of callback functions and commgroups
 * - Registration of callback functions and commgroups
 *
 * ### I/O-operation phase
 *
 * - Parallel open of SIONlib file
 * - I/O-operations (read/write, ..)
 * - Parallel close of SIONlib file
 *
 * Example
 * -------
 *
 * An example can be found in the parallel test section of the SIONlib
 * distribution:
 *
 * - \<topdir>/test/serial/test_genapi_1.c (generic API, only create and link test)
 * - \<topdir>/test/parallel/test_genapi_1.c (generic API)
 * - \<topdir>/test/parallel/test_genapi_2.c (generic mapped API)
 *
 * The example defines a SIONlib generic API and performs some I/O tests with
 * the new API. Internally the API uses MPI to implement the communication
 * operations.
 *
 * \if sion_internal
 *
 * ToDos & Open Issues
 * -------------------
 *
 * - Testing ...
 * - The default parallel interface for MPI implements some special
 *   collective write/read operation which requires internally some
 *   point-to-point operation.  To use these collective write operation
 *   from within the generic interface of SIONlib, two new callback
 *   functions have to be defined and registered (send/recv).  The
 *   definition of these callbacks will be optional. If not defined, the
 *   generic collective read and write functions will then use the
 *   non-collective SIONlib write/read operations internally.
 * - Implementation of the generic versions of sion_paropen_mapped ...
 *
 * \endif
 */

#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>

#ifdef __cplusplus
extern "C" {
#endif

/// \brief creates data structures for local communication group
///
/// - only `local_commgroup` is output parameter: pointer to new data structure
/// - will be called as soon as all information is available (e.g. after read of mapping during paropen)
typedef int (*sion_create_lcg_cb)(
  void **local_commgroup, void *global_commgroup, int grank, int gsize, int lrank, int lsize, int filenumber, int numfiles);

/// \brief free data structure which local_commgroup points on
typedef int (*sion_free_lcg_cb)(void *local_commgroup);

/// \brief performs a barrier on all tasks described with `commgroup` (local or global)
///
/// Blocks until all processes have reached this routine.
typedef int (*sion_barrier_cb)(void *commgroup);

/// \brief performs a broadcast operation from task `root` to all other tasks described with `commgroup` (local or global)
///
/// Replicates data on `root` at memory position `data` of size `nelem` to memory position `data` of all other tasks.
typedef int (*sion_bcastr_cb)(void *data, void *commgroup, int datatype, int nelem, int root);

/// \brief performs a gather operation on all tasks described with `commgroup` (local or global)
///
/// - collects `nelem` of data type `datatype` from each task from memory at `indata` and stores data of tasks on task `root` in
///   memory at position `outdata`
/// - `nelem` has same the value on each task, `outdata` is the of size `nelem` * `ntasks`
/// - on `root` `indata` and `outdata` are not overlapping
typedef int (*sion_gatherr_cb)(void *indata, void *outdata, void *commgroup, int datatype, int nelem, int root);

/// \brief performs a gather operation on all tasks described with `commdata` (local or global)
///
/// - collects `nelem` of data type `datatype` from each tasks from memory at `indata` and stores data of tasks on task `root` in
///   memory at position `outdata`
/// - `nelem` can be different among tasks (or zero), size of `outdata` is sum of `nelem` on each task
/// - `count` is an array containing the number of elems per task, this array will only to be provided on `root`, otherwise `NULL`
/// - on `root` `indata` and `outdata` are not overlapping
typedef int (*sion_gathervr_cb)(void *indata, void *outdata, void *commdata, int datatype, int *count, int nelem, int root);

/// \brief performs a scatter operation on all tasks described with `commgroup` (local or global)
///
/// - distributes `nelem` of data type `datatype` to each tasks from memory at `outdata + offset` on task `root` and stores data
///   in memory at position `indata`
/// - `nelem` has same value on each task, `outdata` is of size `nelem` * `ntasks`
/// - on `root` `indata` and `outdata` are not overlapping
typedef int (*sion_scatterr_cb)(void *indata, void *outdata, void *commgroup, int datatype, int nelem, int root);

/// \brief performs a scatter operation on all tasks described with `commgroup` (local or global)
///
/// - distributes `nelem` of data type `datatype` to each tasks from memory at `outdata + offset` on task `root` and stores data
///   in memory at position `indata`
/// - `count` is an array containing the number of elems per task, this array must only be provided on `root`, otherwise `NULL`
/// - `nelem` can be different among tasks (or zero), size of `outdata` is sum of `nelem` on each task
/// - on `root` `indata` and `outdata` are not overlapping
typedef int (*sion_scattervr_cb)(void *indata, void *outdata, void *commgroup, int datatype, int *count, int nelem, int root);

typedef char *(*sion_get_multi_filename_cb)(const char *, int);

typedef int64_t (*sion_gather_execute_cb)(const void *, int64_t, void *, int, int, int, int,
  int64_t process_cb(const void *, int64_t *, int), void *, bool spec_iterator_cb(void *, int64_t *));

typedef int64_t (*sion_execute_scatter_cb)(void *, int64_t, void *, int, int, int, int,
  int64_t process_cb(void *, int64_t *, int), void *, bool spec_iterator_cb(void *, int64_t *));

typedef int (*sion_get_capability_cb)(void *);

/// \brief Create new api
///
/// \param name a name for that API which will be used in debugging messages for example
///
/// \returns an integer value containing an API descriptor
int lsion_generic_create_api(char *name);

/// \brief free new api
///
/// Frees internal data structures for this API.
int lsion_generic_free_api(int aip);

/// \brief register callback to create local communication group
int lsion_generic_register_create_local_commgroup_cb(int aid, sion_create_lcg_cb cb);

/// \brief register callback to free local communication group
int lsion_generic_register_free_local_commgroup_cb(int aid, sion_free_lcg_cb cb);

/// \brief register callback for communication
int lsion_generic_register_barrier_cb(int aid, sion_barrier_cb cb);

/// \brief register callback for communication
int lsion_generic_register_bcastr_cb(int aid, sion_bcastr_cb cb);

/// \brief register callback for communication
int lsion_generic_register_gatherr_cb(int aid, sion_gatherr_cb cb);

/// \brief register callback for communication
int lsion_generic_register_gathervr_cb(int aid, sion_gathervr_cb cb);

/// \brief register callback for communication
int lsion_generic_register_scatterr_cb(int aid, sion_scatterr_cb cb);

/// \brief register callback for communication
int lsion_generic_register_scattervr_cb(int aid, sion_scattervr_cb cb);

int lsion_generic_register_get_multi_filename_cb(int aid, sion_get_multi_filename_cb cb);

/// \brief register callback for communication
int lsion_generic_register_gather_and_execute_cb(int aid, sion_gather_execute_cb cb);

/// \brief register callback for communication
int lsion_generic_register_execute_and_scatter_cb(int aid, sion_execute_scatter_cb cb);

int lsion_generic_register_get_capability_cb(int aid, sion_get_capability_cb cb);

/*!
 * @brief Open a sion file a generic interface.
 *
 * For a description please see the \ref sion_paropen_mpi_description
 * "description" of lsion_paropen_mpi.
 *
 * - WRITE-Mode
 *   - The parameters `gcommgroup`, `grank`, `gsize`, `filenumber`, `numfiles`, `lrank`, `lsize` are input parameters.
 *     However these have to be defined as variables and initialized before call.
 * - READ-Mode
 *   - `grank` and `gsize` are input parameters
 *   - `filenumber`, `numfiles`, `lrank`, `lsize` are output parameters and will be read from SIONlib file meta data
 *   - The data container of the SIONlib file (chunks) from file will be mapped to tasks according to the order given by the
 *     global commgroup.
 *
 * @param[in]       aid          sion API id
 * @param[in]       fname        name of file, should be equal on all tasks
 * @param[in]       file_mode    like the type parameter of fopen (currently recognized options: "rb", "wb")
 * @param[in,out]   chunksize    maximum size to be written with single write call
 * @param[in,out]   fsblksize    file system block size (-1 for automatic)
 * @param[in]       gcommgroup   global communicator
 * @param[in]       grank        global rank of process
 * @param[in]       gsize        size of global communicator
 * @param[in,out]   filenumber   file number (for multiple file mode)
 * @param[in,out]   numfiles     number of files to use (-1 for automatic choosing from local communicator)
 * @param[in]       lrank        local rank of process
 * @param[in]       lsize        size of local communicator
 * @param[in,out]   fileptr      file pointer (NULL for not using an external file pointer)
 * @param[out]      newfname     return value for actual file name if using multiple files
 *
 * @retval          sid          sion file handle or -1 if error occured
 */
int lsion_generic_paropen(int aid, const char *fname, const char *file_mode, int64_t *chunksize, int32_t *fsblksize,
  void *gcommgroup, int grank, int gsize, int *filenumber, int *numfiles, const int *lrank, const int *lsize, FILE **fileptr,
  char **newfname);

int lsion_generic_parclose(int sid);

/// each task specifies a set of globalranks which should opened on this task
///
/// \param aid SIONlib API id
/// \param fname name of file
/// \param file_mode like the type parameter of fopen
/// \param numFiles number of files to use
/// \param gcommgroup global communicator
/// \param grank global rank of calling task
/// \param gsize size of global communicator
/// \param nlocaltasks gives the number of `globalranks` to be opened by this task
/// \param globalranks an array containing these globalranks to be opened
/// \param chunksizes an array containing the chunksize for each globalrank which is locally opened
/// \param mapping_filenrs an array containing the filenumber for each globalrank which is locally opened
/// \param mapping_lranks an array containing the local rank in the file for each globalrank which is locally opened
/// \param fsblksize file system block size
/// \param fileptr file pointer
///
/// \returns a SIONlib file handle or -1 if an error occurred
///
/// - WRITE-Mode
///   - The parameters `gcommgroup`, `grank`, `gsize`, `numFiles`, `nlocaltasks`, `globalranks`, `chunksizes`, `mapping_filenrs`,
///     `mapping_lranks`, `fsblksize` are input parameters. However these have to be defined as variables and initialized before
///     call.
/// - READ-Mode
///   - The parameters `gcommgroup`, `grank`, `gsize`, `numfiles`, `nlocaltasks`, `globalranks` are input parameter.
int lsion_generic_paropen_mapped(int aid, char *fname, const char *file_mode, int *numFiles, void *gcommgroup, int grank,
  int gsize, int *nlocaltasks, int **globalranks, int64_t **chunksizes, int **mapping_filenrs, int **mapping_lranks,
  int32_t *fsblksize, FILE **fileptr);

int lsion_generic_parclose_mapped(int sid);

int lsion_generic_parreinit(int sid, int64_t chunksize);

/*!
 * \defgroup collective Collective API
 * @{
 *
 * \brief I/O operations that involve all tasks that have opened a file
 *
 * Introduction
 * ============
 *
 * Collective I/O calls use an I/O schema similar to MPI-I/O two-stage
 * I/O: Tasks will be divided into collectors and sender. Each collector
 * collects and writes data of its own tasks and a number of following
 * tasks to the SION-file. Each sender will only send its own data to the
 * corresponding collector task.
 *
 * Advantages of this methods are the reduced number of writer tasks,
 * which could be lead e.g. to a better efficiency of the I/O nodes and a
 * more densely packing of the chunks in the SION file. The reason for
 * this is that the alignment to file system blocks boundaries is only
 * needed per collector.
 *
 * The number of collectors depends on the chunksize requested by each
 * task and can also be set by environment variables (see below). The
 * main goal is to use enough senders per collector so that the collector
 * can write a full file system block.
 *
 * Usage
 * =====
 *
 * In order to use collective I/O in SIONlib two changes to normal
 * SIONlib output are needed.
 *
 * - The write call lsion_fwrite() has to be replaced by its collective
 *   counterpart lsion_coll_fwrite()
 * - Either of the following
 *   - The environment variable `SION_COLLSIZE` or `SION_COLLNUM` (see
 *     below) needs to be set, or
 *   - the file needs to be opend with `...,collective,collsize=NUM,...`
 *     added to the file_mode, where `NUM` has the same meaning as the
 *     value of the environment variable `SION_COLLSIZE`.
 *
 * If both, the environment variable `SION_COLLSIZE` and the open flag
 * are set, the environment variable overwrites the value chosen in the
 * flags.
 *
 * Parameters
 * ==========
 *
 * Environment
 * - `SION_COLLDEBUG`
 * - `SION_COLLSIZE`
 *   - `-1`: collector is computed by SIONlib and depends on chunksizes
 *     and file system blocksize
 *   - `0`: collective is not used
 *   - `> 0`: number of tasks to be collected by one (master) task
 * - `SION_COLLNUM`
 *   - `> 0`: roughly the number of collectors to use
 *
 * Procedure to determine collectors and senders
 * =============================================
 * - If `SION_COLLSIZE` set: -> `sion_filedesc->collsize`
 * - If not check if `SION_COLLNUM` set: ->
 *   \code
 *   numcoll = max(SION_COLLNUM, ntasks);
 *   sion_filedesc->collsize = sion_filedesc->ntasks / numcoll;
 *   \endcode
 * - in calculate_startpointer: not less than one fsblksize per collector
 *   \code
 *   max_num_collectors  = (int) (gsize / sion_filedesc->fsblksize);
 *   \endcode
 * - `sion_filedesc->collsize > 0`: number of collectors or sender /
 *   collector given by user
 *   \code
 *   num_collectors = min(sion_filedesc->ntasks /
 *   sion_filedesc->collsize, max_num_collectors);
 *   \endcode
 * - `sion_filedesc->collsize < 0`: SIONlib computes number of collectors
 *   \code
 *   num_collectors = min(max_num_collectors, ntasks); # given by data
 *   size and fsblksize
 *   \endcode
 * - limit number of collectors according table:
 *   \code
 *   if ((sion_filedesc->ntasks >= 512) && (num_collectors > 32))
 *   num_collectors = 32;
 *   \endcode
 *
 * | sion_filedesc->ntasks >= | num_collectors > | -> num_collectors = |
 * |--------------------------|------------------|---------------------|
 * |                      256 |               16 |                  16 |
 * |                      128 |                8 |                   8 |
 * |                       64 |                8 |                   8 |
 * |                       32 |                8 |                   8 |
 * |                       16 |                8 |                   4 |
 *
 * Collective Modes
 * ================
 *
 * Normal write mode (for comparison)
 * -----------------------------------
 *
 * \code
 *           +-----+               +-----+               +-----+               +-----+
 * tasks     | t_0 |               | t_1 |               | t_2 |               | t_3 |
 *           +-----+               +-----+               +-----+               +-----+
 *              |                     |                     |                     |
 *              V                     V                     V                     V
 *           +---------------------+---------------------+---------------------+---------------------+
 * FS blocks | d_0 |               | d_1 |               | d_2 |               | d_3 |               |
 *           +---------------------+---------------------+---------------------+---------------------+
 * \endcode
 *
 * Each chunk is extended to at least one file system block. This
 * prevents contention between different tasks trying to write to the
 * same file system block, but also is a potential wast of space if the
 * data actually written is very small.
 *
 * Standard collective write mode
 * ------------------------------
 *
 * \code
 *           +-----+               +-----+               +-----+               +-----+
 * tasks     | t_0 |<------+       | t_1 |               | t_2 |               | t_3 |
 *           +-----+       |       +-----+               +-----+               +-----+
 *              |          |          |                     |                     |
 *              |          |          V                     V                     |
 *              |          +----------+----------<----------+                     |
 *              |                                                                 |
 *              |                     +---------------------<---------------------+
 *              |                     |
 *              V                     V
 *           +---------------------+---------------------+---------------------+---------------------+
 * FS blocks | d_0 | d_1 | d_2 |   | d_3 |               |                     |                     |
 *           +---------------------+---------------------+---------------------+---------------------+
 * \endcode
 *
 * The standard collective write mode tries to collect data from
 * different tasks for a better usage of the available space and a
 * reduction of the processes actually writing to the file system (see
 * introduction).
 *
 * \anchor collective_merge_description
 * Merge collective write mode
 * ---------------------------
 *
 * \code
 *           +-----+               +-----+               +-----+               +-----+
 * tasks     | t_0 |<------+       | t_1 |               | t_2 |               | t_3 |
 *           +-----+       |       +-----+               +-----+               +-----+
 *              |          |          |                     |                     |
 *              |          |          V                     V                     |
 *              |          +----------+----------<----------+                     |
 *              |                                                                 |
 *              |                     +---------------------<---------------------+
 *              |                     |
 *              V                     V
 *           +---------------------+---------------------+---------------------+---------------------+
 * FS blocks | ----- d_0 ----- |   | d_3 |               |                     |                     |
 *           +---------------------+---------------------+---------------------+---------------------+
 * \endcode
 *
 * Merge collective write behaves like the standard collective write but
 * represents all the collected data as it would belong to the collector
 * and leaves the collected chunks empty.
 *
 * It is enabled using `"...,collectivemerge,..."` or `"...,cmerge,..."` in
 * file mode.
 *
 * Sources
 * =======
 * - `_lsion_coll_check_env()` in `sion_internal_par.c`
 * - `_lsion_calculate_startpointers_collective()` in  `sion_internal_startptr.c`
 */

size_t lsion_coll_fwrite(const void *data, size_t size, size_t nitems, int sid);
size_t lsion_coll_fread(void *data, size_t size, size_t nitems, int sid);

/// @}

#ifdef __cplusplus
}
#endif

/// @}

#endif

