/****************************************************************************
**  SIONLIB     http://www.fz-juelich.de/jsc/sionlib                       **
*****************************************************************************
**  Copyright (c) 2008-2019                                                **
**  Forschungszentrum Juelich, Juelich Supercomputing Centre               **
**                                                                         **
**  See the file COPYRIGHT in the package base directory for details       **
****************************************************************************/

#ifndef SION_SION_OMPI_H
#define SION_SION_OMPI_H

#ifdef SION_OMPI

/// \defgroup ompi MPI+OpenMP Parallel API
/// @{
/// \brief Open (and close) SIONlib files from multiple OpenMP threads on
///   multiple MPI processes in parallel

#include <stdint.h>
#include <stdio.h>

#include <mpi.h>

#include "sion_const.h"
#include "sion_datatypes_mpi.h"

#ifdef __cplusplus
extern "C" {
#endif

/// \brief Holds non-essential arguments for `sion_paropen_ompi()`.
///
/// These are:
/// - the sizes of chunks for the logical file opened by this task,
/// - the size of a file system block to assume,
/// - whether and in what mode to use SIONlib's key-value mechanism,
/// - whether to use the buddy redundancy mechanism,
/// - whether to use collective I/O,
/// - what low-level API to use for file system access,
/// - the endianness of user data in the file,
/// - how many files to use.
typedef struct sion_ompi_options sion_ompi_options;

/// \brief Open a SIONlib file from multiple OpenMP threads on
///   multiple MPI processes in parallel
///
/// This function opens a distinct logical file in a SIONlib container for each
/// OpenMP thread on each MPI process that calls it.
///
/// \param[in] filename name of file, should be equal on all tasks
/// \param[in] mode whether to open the file for reading or writing
/// \param[in] communicator a communicator containing all processes that participate in opening the container
/// \param[in] options additional options (or `NULL`)
///
/// \returns a SIONlib file handle or -1 if an error occured
int sion_paropen_ompi(const char *filename, sion_open_mode mode, MPI_Comm communicator, const sion_ompi_options *options);

/// \brief closes a SIONlib file previously opened in OpenMP/MPI mode
///
/// \param[in] sid SIONlib file id
int sion_parclose_ompi(int sid);

/// \brief Allocates and initializes an instance of `sion_ompi_options`
///
/// \returns a pointer to the new instance or `NULL` if allocation failed
sion_ompi_options *sion_ompi_options_new();

/// \brief Delete an instance of `sion_ompi_options`
///
/// \param[in] options a pointer to the instance to delete
void sion_ompi_options_delete(sion_ompi_options *options);

/// \brief Set the chunk size of a logical file in the container
///
/// \param[in,out] options an options object
/// \param[in] chunksize chunk sizes for the logical file opened by this process
void sion_ompi_options_set_chunksize(sion_ompi_options *options, int64_t chunksize);

/// \brief Set the file system block size to assume.
///
/// \param[in,out] options an options object
/// \param[in] fsblksize the file system block size
void sion_ompi_options_set_fsblksize(sion_ompi_options *options, int32_t fsblksize);

/// \brief Set the number of physical files to use
///
/// \param[in,out] options an options object
/// \param[in] multifile_number how many files to use
void sion_ompi_options_set_multifile_number(sion_ompi_options *options, int multifile_number);

/// \brief Create multiple physical files based on disjunct communicators
///
/// A separate physical file will be created for every distinct communicator.
/// The union of all the different communicators should be the communicator
/// passed to `sion_paropen_ompi()`.
///
/// \param[in,out] options an options object
/// \param[in] multifile_communicator
void sion_ompi_options_set_multifile_communicator(sion_ompi_options *options, MPI_Comm multifile_communicator);

/// \brief Set the key-value mode to use for a container.
///
/// \param[in,out] options an options object
/// \param[in] keyval_mode the key-value mode to use
void sion_ompi_options_set_keyval_mode(sion_ompi_options *options, sion_keyval_mode keyval_mode);

/// \brief Enable buddy checkpointing mechanism
///
/// Has the same effect as `sion_generic_options_set_buddylevel(options, 1)`.
///
/// \see resiliency
///
/// \param[in] options an options object
void sion_ompi_options_set_buddy(sion_ompi_options *options);

/// \brief Enable buddy checkpointing mechanism
///
/// Enables a specific level of buddy redundancy.
///
/// \see resiliency
///
/// \param[in] options an options object
/// \param[in] buddylevel the level of redundancy to use
void sion_ompi_options_set_buddylevel(sion_ompi_options *options, int32_t buddylevel);

/// \brief Enable collective I/O
///
/// The size of collector groups will be determined via heuristics.
///
/// \see collective
///
/// \param[in] options an options object
void sion_ompi_options_set_collective(sion_ompi_options *options);

/// \brief Enable collective I/O
///
/// \see collective
///
/// \param[in] options an options object
/// \param[in] size size of the collector groups
void sion_ompi_options_set_collective_size(sion_ompi_options *options, int32_t size);

/// \brief Use collective merging
///
/// \see collective
///
/// \param[in] options an options object
void sion_ompi_options_set_collective_merge(sion_ompi_options *options);

/// \brief Set the low-level API to use for opening a container.
///
/// \param[in,out] options an options object
/// \param[in] lowlevel_api the low-level API to use
void sion_ompi_options_set_lowlevel_api(sion_ompi_options *options, sion_lowlevel_api lowlevel_api);

/// \brief Set the endianness for the contents of a container
///
/// \param[in,out] options an options object
/// \param[in] endianness the endianness of the contents
void sion_ompi_options_set_endianness(sion_ompi_options *options, sion_endianness endianness);

#ifdef __cplusplus
}
#endif

/// @}

#endif

#endif
