/****************************************************************************
**  SIONLIB     http://www.fz-juelich.de/jsc/sionlib                       **
*****************************************************************************
**  Copyright (c) 2008-2018                                                **
**  Forschungszentrum Juelich, Juelich Supercomputing Centre               **
**                                                                         **
**  See the file COPYRIGHT in the package base directory for details       **
****************************************************************************/

#ifndef SION_SION_MPI_H
#define SION_SION_MPI_H

#ifdef SION_MPI

/*!
 * \defgroup mpi MPI Parallel API
 * @{
 * \brief Open (and close) SIONlib files from multiple MPI processes in parallel
 *
 * \see \ref overview
 */

#include <mpi.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>

#include "sion_const.h"
#include "sion_enums.h"

#include "sion_datatypes_mpi.h"

#ifdef __cplusplus
extern "C" {
#endif

/*!
 * @brief Open a sion file using MPI.
 *
 * This function opens a sion file using MPI. It processes the MPI
 * specific parts and then passes its arguments on to
 * lsion_generic_paropen().
 *
 * For more description please see the \ref sion_paropen_mpi_description
 * "description" of lsion_paropen_mpi.
 *
 * @param[in]       fname        name of file, should be equal on all tasks
 * @param[in]       file_mode    like the type parameter of fopen. See
 *                               \ref file_mode_description "file mode description" for details
 * @param[in,out]   numFiles     number of multi files to use (-1 for automatic choosing from local communicator)
 * @param[in]       gComm        global MPI communicator (typically MPI_COMM_WORLD)
 * @param[in]       lComm        local MPI communicator (= gComm if no adaption to I/O nodes is needed)
 *                               lComm should be a partition of gComm, i.e. a disjoint cover
 * @param[in,out]   chunksize    maximum size to be written with single write call
 * @param[in,out]   fsblksize    file system block size. Must be equal on all processes (-1 for automatic)
 * @param[in,out]   globalrank   global rank of process
 *                               any globally unique id for current task. It
 *                               will be stored in sion file. Useful if comm is
 *                               not MPI_COMM_WORLD (typically:
 *                               globalrank = rank in MPI_COMM_WORLD)
 * @param[out]      fileptr      file pointer (NULL for not using an external
 *                               which allows for more optimisation)
 * @param[out]      newfname     return value for actual file name if using
 *                               multi files. NULL does not return the name. If
 *                               passed non NULL pointer the memory needs to be
 *                               freed by the user.
 *
 * @retval          sid          sion file handle or -1 if error occurred
 */
int lsion_paropen_mpi(const char *fname, const char *file_mode, int *numFiles, MPI_Comm gComm, const MPI_Comm *lComm,
  int64_t *chunksize, int32_t *fsblksize, int *globalrank, FILE **fileptr, char **newfname);

/*!
 * @brief Close a sion file using MPI.
 *
 * For more description please see the \ref sion_parclose_mpi_description
 * "description" of lsion_parclose_mpi.
 *
 * @param[in]   sid   sion file handle
 *
 * @retval   success value (SION_SUCCESS or SION_NOT_SUCCESS)
 */
int lsion_parclose_mpi(int sid);

int lsion_parreinit_mpi(int sid, int64_t chunksize);

/* new mapped functions */

/* open function:
   write-mode:
     if numFiles > 1  --> mapping_filenrs, mapping_lranks contains local mappping
    */

int lsion_paropen_mapped_mpi(char *fname, const char *file_mode, int *numFiles, MPI_Comm gComm, int *nlocaltasks,
  int **globalranks, int64_t **chunksizes, int **mapping_filenrs, int **mapping_lranks, int32_t *fsblksize, FILE **fileptr);

int lsion_parclose_mapped_mpi(int sid);

/* Serialization */
int lsion_startof_transaction_mpi(int sid);
int lsion_endof_transaction_mpi(int sid);
size_t lsion_coll_fwrite_mpi(const void *data, size_t size, size_t nitems, int sid);
size_t lsion_coll_fread_mpi(void *data, size_t size, size_t nitems, int sid);

/* for backward compatibility */
int lsion_paropen_comms_mpi(char *fname, const char *file_mode, int *numFiles, MPI_Comm gComm, MPI_Comm lComm, int64_t *chunksize,
  int32_t *fsblksize, int *globalrank, FILE **fileptr, char *newfname);
int lsion_paropen_multi_mpi(char *fname, const char *file_mode, int *numFiles, MPI_Comm gComm, MPI_Comm *lComm, int64_t *chunksize,
  int32_t *fsblksize, int *globalrank, FILE **fileptr, char *newfname);

/* for file check */
typedef struct _sion_file_check_par_args_mpi_struct sion_file_check_par_args_mpi;
sion_file_check_par_args_mpi *lsion_file_check_par_args_init_mpi(
  const char *file_mode, MPI_Comm gComm, int numfiles, MPI_Comm lComm);
int lsion_file_check_par_args_free_mpi(sion_file_check_par_args_mpi *args);
int lsion_file_check_par_cb_mpi(char *fname, void *args);

typedef enum { _SION_MULTIFILE_SINGLE, _SION_MULTIFILE_NUMBER, _SION_MULTIFILE_COMMUNICATOR } _sion_mpi_multifile_mode;

typedef union {
  int number;
  MPI_Comm communicator;
} _sion_mpi_multifile_data;

typedef struct {
  int64_t chunksize; // default: fsblksize
  int32_t fsblksize; // default: auto-detect

  _sion_mpi_multifile_mode multifile_mode; // default: _SION_MULTI_FILE_SINGLE
  _sion_mpi_multifile_data multifile;

  sion_keyval_mode keyval_mode; // default: SION_KEYVAL_MODE_NONE

  int32_t buddylevel; // default: 0, no buddy checkpointing

  int32_t collsize; // default: 0, do not use collective mode
  bool collective_merge; // default: false
} sion_mpi_options;

sion_mpi_options lsion_mpi_options_new();

void lsion_mpi_options_set_chunksize(sion_mpi_options *options, int64_t chunksize);
void lsion_mpi_options_set_fsblksize(sion_mpi_options *options, int32_t fsblksize);

void lsion_mpi_options_set_multifile_number(sion_mpi_options *options, int multifile_number);
void lsion_mpi_options_set_multifile_communicator(sion_mpi_options *options, MPI_Comm multifile_communicator);

void lsion_mpi_options_set_keyval_mode(sion_mpi_options *options, sion_keyval_mode keyval_mode);

void lsion_mpi_options_set_buddy(sion_mpi_options *options);
void lsion_mpi_options_set_buddylevel(sion_mpi_options *options, int32_t buddylevel);

void lsion_mpi_options_set_collective(sion_mpi_options *options);
void lsion_mpi_options_set_collective_size(sion_mpi_options *options, int32_t size);
void lsion_mpi_options_set_collective_merge(sion_mpi_options *options);

int lsion_paropen_mpi_with_options(
  const char *filename, sion_open_mode mode, MPI_Comm communicator, const sion_mpi_options *options);

#ifdef __cplusplus
}
#endif

/// @}

#endif

#endif
