/****************************************************************************
**  SIONLIB     http://www.fz-juelich.de/jsc/sionlib                       **
*****************************************************************************
**  Copyright (c) 2008-2019                                                **
**  Forschungszentrum Juelich, Juelich Supercomputing Centre               **
**                                                                         **
**  See the file COPYRIGHT in the package base directory for details       **
****************************************************************************/

#ifndef SION_SION_MPI_H
#define SION_SION_MPI_H

#ifdef SION_MPI

/// \defgroup mpi MPI Parallel API
/// @{
/// \brief Open (and close) SIONlib files from multiple MPI processes in parallel

#include <mpi.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>

#include "sion_const.h"
#include "sion_enums.h"

#include "sion_datatypes_mpi.h"

#ifdef __cplusplus
extern "C" {
#endif

/// \brief Holds non-essential arguments for `sion_paropen_mpi()`.
///
/// These are:
/// - the sizes of chunks for the logical file opened by this task,
/// - the size of a file system block to assume,
/// - whether and in what mode to use SIONlib's key-value mechanism,
/// - whether to use the buddy redundancy mechanism,
/// - whether to use collective I/O,
/// - what low-level API to use for file system access,
/// - the endianness of user data in the file,
/// - how many files to use.
typedef struct sion_mpi_options sion_mpi_options;

/// \brief Open a SIONlib file from multiple MPI processes.
///
/// This function opens a SIONlib file using MPI. It processes the MPI
/// specific parts and then passes its arguments on to
/// `sion_generic_paropen()`.
///
/// \param[in] filename name of the file (must match on all processes)
/// \param[in] mode what mode to open the file in
/// \param[in] communicator a communicator containing all processes that call this function
/// \param[in] options additional options or `NULL`
///
/// \returns SIONlib file handle or -1 if an error occurred
int sion_paropen_mpi(const char *filename, sion_open_mode mode, MPI_Comm communicator, const sion_mpi_options *options);

/// \brief Close a SIONlib file using MPI.
///
/// Uses `sion_generic_parclose()` internally.
///
/// \param[in] sid SIONlib file handle
///
/// \returns success value (SION_SUCCESS or SION_NOT_SUCCESS)
int sion_parclose_mpi(int sid);

int sion_parreinit_mpi(int sid, int64_t chunksize);

/* new mapped functions */

/* open function:
   write-mode:
     if numFiles > 1  --> mapping_filenrs, mapping_lranks contains local mappping
    */

int sion_paropen_mapped_mpi(char *fname, const char *file_mode, int *numFiles, MPI_Comm gComm, int *nlocaltasks,
  int **globalranks, int64_t **chunksizes, int **mapping_filenrs, int **mapping_lranks, int32_t *fsblksize, FILE **fileptr);

int sion_parclose_mapped_mpi(int sid);

/* for file check */
typedef struct _sion_file_check_par_args_mpi_struct sion_file_check_par_args_mpi;
sion_file_check_par_args_mpi *sion_file_check_par_args_init_mpi(
  sion_open_mode mode, MPI_Comm communicator, const sion_mpi_options *options);
void sion_file_check_par_args_free_mpi(sion_file_check_par_args_mpi *args);
int sion_file_check_par_cb_mpi(char *fname, void *args);

/// \brief Allocates and initializes an instance of `sion_mpi_options`
///
/// \returns a pointer to the new instance or `NULL` if allocation failed
sion_mpi_options *sion_mpi_options_new();

/// \brief Delete an instance of `sion_mpi_options`
///
/// \param[in] options a pointer to the instance to delete
void sion_mpi_options_delete(sion_mpi_options *options);

/// \brief Set the chunk size of a logical file in the container
///
/// \param[in,out] options an options object
/// \param[in] chunksize chunk sizes for the logical file opened by this process
void sion_mpi_options_set_chunksize(sion_mpi_options *options, int64_t chunksize);

/// \brief Set the file system block size to assume.
///
/// \param[in,out] options an options object
/// \param[in] fsblksize the file system block size
void sion_mpi_options_set_fsblksize(sion_mpi_options *options, int32_t fsblksize);

/// \brief Set the number of physical files to use
///
/// \param[in,out] options an options object
/// \param[in] multifile_number how many files to use
void sion_mpi_options_set_multifile_number(sion_mpi_options *options, int multifile_number);

/// \brief Create multiple physical files based on disjunct communicators
///
/// A separate physical file will be created for every distinct communicator.
/// The union of all the different communicators should be the communicator
/// passed to `sion_paropen_mpi()`.
///
/// \param[in,out] options an options object
/// \param[in] multifile_communicator
void sion_mpi_options_set_multifile_communicator(sion_mpi_options *options, MPI_Comm multifile_communicator);

/// \brief Set the key-value mode to use for a container.
///
/// \param[in,out] options an options object
/// \param[in] keyval_mode the key-value mode to use
void sion_mpi_options_set_keyval_mode(sion_mpi_options *options, sion_keyval_mode keyval_mode);

/// \brief Enable buddy checkpointing mechanism
///
/// Has the same effect as `sion_generic_options_set_buddylevel(options, 1)`.
///
/// \see resiliency
///
/// \param[in] options an options object
void sion_mpi_options_set_buddy(sion_mpi_options *options);

/// \brief Enable buddy checkpointing mechanism
///
/// Enables a specific level of buddy redundancy.
///
/// \see resiliency
///
/// \param[in] options an options object
/// \param[in] buddylevel the level of redundancy to use
void sion_mpi_options_set_buddylevel(sion_mpi_options *options, int32_t buddylevel);

/// \brief Enable collective I/O
///
/// The size of collector groups will be determined via heuristics.
///
/// \see collective
///
/// \param[in] options an options object
void sion_mpi_options_set_collective(sion_mpi_options *options);

/// \brief Enable collective I/O
///
/// \see collective
///
/// \param[in] options an options object
/// \param[in] size size of the collector groups
void sion_mpi_options_set_collective_size(sion_mpi_options *options, int32_t size);

/// \brief Use collective merging
///
/// \see collective
///
/// \param[in] options an options object
void sion_mpi_options_set_collective_merge(sion_mpi_options *options);

/// \brief Set the low-level API to use for opening a container.
///
/// \param[in,out] options an options object
/// \param[in] lowlevel_api the low-level API to use
void sion_mpi_options_set_lowlevel_api(sion_mpi_options *options, sion_lowlevel_api lowlevel_api);

/// \brief Set the endianness for the contents of a container
///
/// \param[in,out] options an options object
/// \param[in] endianness the endianness of the contents
void sion_mpi_options_set_endianness(sion_mpi_options *options, sion_endianness endianness);

#ifdef __cplusplus
}
#endif

/// @}

#endif

#endif
