/****************************************************************************
**  SIONLIB     http://www.fz-juelich.de/jsc/sionlib                       **
*****************************************************************************
**  Copyright (c) 2008-2018                                                **
**  Forschungszentrum Juelich, Juelich Supercomputing Centre               **
**                                                                         **
**  See the file COPYRIGHT in the package base directory for details       **
****************************************************************************/

#ifndef SION_SION_COMMON_H
#define SION_SION_COMMON_H

/*!
 * \defgroup common Common API
 * @{
 * \brief Common functions of the SIONlib API shared between serial and parallel modes
 */

#include <stdint.h>
#include <stdio.h>

#include "sion_enums.h"

struct _sion_io_stat_struct;

/* default mode */
#define SION_SERIAL_MASTER

#define SION_MAX_FSBLOCKSIZE 32 * 1024 * 1024
#define SION_DEFAULT_FSBLOCKSIZE 2 * 1024 * 1024

#define SION_FILENAME_LENGTH 1024

#define MAXCHUNKS 100 /*!< Initial maximum number of chunks in a sion file */

#define WFLARGEMETABLOCK_none /* for BENCHMARKING only */

#ifdef __cplusplus
extern "C" {
#endif

int _lsion_close_sid(int sid);

/* I/O */
/*!
 * @brief Write data to sion file.
 *
 * See \ref sion_fwrite_description
 *
 * @param[in]  data   pointer to data to be written
 * @param[in]  size   size of a single item
 * @param[in]  nitems number of items to be written
 * @param[in]  sid    sion file id to be written to
 *
 * @return     number of elements written
 */
size_t lsion_fwrite(const void *data, size_t size, size_t nitems, int sid);

/*!
 * @brief Read data from sion file.
 *
 * This is the basic function for reading data from the underlying
 * storage.
 *
 * @param[out] data   pointer to data buffer
 * @param[in]  size   size of a single item
 * @param[in]  nitems number of items to be written
 * @param[in]  sid    sion file id to be written to
 *
 * @return     number of elements read
 */
size_t lsion_fread(void *data, size_t size, size_t nitems, int sid);

/* helper functions */
/*!
 * @brief Function that indicates whether the end of file is reached for this task.
 *
 * See \ref sion_feof_description.
 *
 * @param[in]   sid     sion file handle (in)
 *
 * @return      SION_SUCCESS (1) if end of last chunk reached
 *              SION_NOT_SUCCESS (0) otherwise
 */
int lsion_feof(int sid);

/*!
 * @brief Funtion to ensure that enough space is available for writing.
 *
 * See \ref sion_ensure_free_space_description
 *
 * @param[in]   sid     sion file handle
 * @param[in]   bytes   number of bytes requested for the next write operation
 *
 * @return     SION_SUCCESS if ok
 */
int lsion_ensure_free_space(int sid, int64_t bytes);

/*!
 * @brief Flushed sion file.
 *
 * @param[in]   sid             sion file handle
 *
 * @return     SION_SUCCESS if ok
 */
int lsion_flush(int sid);

/* get size of internal data structure */
/*!
 * @brief Function returns size of internal data structure for sid
 *
 * @param[in]    sid        sion file handle
 * @param[out]   numbytes   number of bytes used
 * @param[out]   numfds     number of file descriptors used
 *
 * @return     SION_SUCCESS if ok
 */
int lsion_get_sizeof(int sid, int *numbytes, int *numfds);

/* duplicate fd (only parallel read-mode) */
/*!
 * @brief Function which duplicates a sion file descriptor
 *
 * Restrictions:
 *  - only in read mode
 *  - only in parallel mode
 *  - collective operation are not callable on the new sid
 *
 * @param[in]   sid             sion file handle
 * @param[in]   mode            one of SION_DUP_ALL, SION_DUP_RANK, SION_DUP_RANK_KEY
 * @param[in]   rank            dup only those part of the internal data struture which are needed to read rank _rank_ from file
 * @param[in]   key             dup only those part of the internal data struture which are needed to read on the rank _rank_ data for key _key_ from file
 *
 * @return     new sid if ok, SION_ID_NOT_VALID in other cases
 */
int lsion_dup(int sid, int mode, int rank, uint64_t key);

/*!
 * @brief Function which destroy a duplicated sion file descriptor
 *
*
 * @param[in]   sid             sion file handle
 *
 * @return     SION_SUCCESS okay
 */
int lsion_dedup(int sid);

/* special helper functions, for used with szlib */
int lsion_optimize_fp_buffer(int sid);

/* get information (with sion datatypes) */
/*!
 * @brief Returns edianness of data in file sid
 *
 * @param[in] sid sion file handle
 *
 * @return    1-> big endian, 0-> little endian
 */
int lsion_get_file_endianness(int sid);

/*!
 * @brief Returns whether or not byte swapping is needed for sid
 *
 * @param[in] sid sion file handle
 *
 * @return    1-> swap needed, 0-> no swap needed
 */
int lsion_endianness_swap_needed(int sid);
int64_t lsion_get_bytes_written(int sid);
int64_t lsion_get_bytes_read(int sid);

/*!
 * @brief Return the number of bytes available in the current chunk.
 *
 * See \ref sion_bytes_avail_in_block_description.
 *
 * @param[in]   sid             sion file handle
 *
 * @return      rc>0  number of bytes
 *              rc<=0 file position is after end of block
 */
int64_t lsion_bytes_avail_in_block(int sid);

/*!
 * @brief Function that returns the number of bytes available in the current chunk.
 *
 * This function returns the number of bytes remaining in the current chunk (bytes not read).
 * It is a local function, which can be called independently from other MPI tasks.
 *
 * @param[in]   sid             sion file handle
 *
 * @return      rc>0  number of bytes
 *              rc<=0 file position is after end of block
 */
int64_t lsion_bytes_avail_in_chunk(int sid);

/*!
 * @brief Function that returns the current file position
 *
 * This function returns an int64_t containing the current file possition.
 *
 * @param[in]   sid     sion file handle (in)
 *
 * @return      current file position
 *
 */
int64_t lsion_get_position(int sid);

/*!
 * @brief Returns pointers to the internal field mapping.
 *
 * This function is only needed if the sion file was opened for reading in
 * serial mode.
 *
 * @param[in]   sid          sion file handle
 * @param[out]  mapping_size size of mapping vector
 * @param[out]  mapping      pointer to mapping vector
 * @param[out]  numfiles     pointer to mapping vector
 *
 * @return      SION_SUCCESS if ok
 */
int lsion_get_mapping(int sid, int *mapping_size, int32_t **mapping, int *numfiles);

/*!
 * @brief Returns pointers to internal fields.
 *
 * See \ref sion_get_locations_description .
 *
 * @param[in]   sid                 sion file handle
 * @param[out]  ntasks              number of tasks which wrote to the sion
 *                                  file
 * @param[out]  maxchunks           maximum number of blocks in the file
 * @param[out]  globalskip          distance in bytes between the first bytes
 *                                  of two subsequent chunks of a task
 * @param[out]  start_of_varheader  start position of the meta data block at
 *                                  the end of sion file
 * @param[out]  sion_chunksizes     field containing requested chunk size of
 *                                  each task
 *                                  access: sion_chunksizes[size*chunknr+rank]
 * @param[out]  sion_globalranks    field containing global unique id of each
 *                                  task
 * @param[out]  sion_blockcount     field containing number of blocks used by
 *                                  each task
 * @param[out]  sion_blocksizes     field containing for each task and block
 *                                  the number of bytes used in this chunk e.g.
 *                                  access:
 *                                  sion_blocksizes[size*blocknr+rank]
 * @return      SION_SUCCESS if ok
 */
int lsion_get_locations(int sid, int *ntasks, int *maxchunks, int64_t *globalskip, int64_t *start_of_varheader,
  int64_t **sion_chunksizes, int64_t **sion_globalranks, int64_t **sion_blockcount, int64_t **sion_blocksizes);

/* parallel mode */
/*!
 * @brief Returns current position in file and pointer fiels containing chunk
 * sizes.
 *
 * Returns current position in file and pointer fiels containing chunk
 * sizes. This function is only needed if the sion file was opened for reading
 * in parallel mode (sion_paropen..., lsion_open_rank).
 *
 * @param[in]   sid            sion file handle
 * @param[out]  currentchunknr number of current block in the file
 * @param[out]  currentpos     position in current block in the file
 * @param[out]  maxchunks      number of last block in the file
 * @param[out]  chunksizes     field containing chunk size of each task
 *
 * @return      SION_SUCCESS if ok
 */
int lsion_get_current_location(int sid, int *currentchunknr, int64_t *currentpos, int *maxchunks, int64_t **chunksizes);

/* serial mode */
/*!
 * @brief Returns current position in file and pointer fiels containing chunk
 * sizes.
 *
 * Returns current position in file and pointer fiels containing chunk
 * sizes. This function is only needed if the sion file was opened for reading
 * in parallel mode (sion_paropen..., lsion_open_rank).
 *
 * @param[in]   sid                 sion file handle
 * @param[out]  ntasks              number of tasks
 * @param[out]  sion_currentpos     current positions for all ranks
 * @param[out]  sion_currentblocknr current block number for all ranks
 *
 * @return      SION_SUCCESS if ok
 */
int lsion_get_current_locations(int sid, int *ntasks, int64_t **sion_currentpos, int64_t **sion_currentblocknr);
int lsion_get_number_of_files(int sid);
int lsion_get_filenumber(int sid);

/* 1 is opened with lsion_open or lsion_open_rank */
int lsion_is_serial_opened(int sid);

/* 1 if using GPFS or Linux hints */
int lsion_using_hints(int sid);

/*   return current endianness (1-> big endian, 0 ->little endian) */
/*!
 * \brief Return endianness
 *
 * @return 1-> big endian
 *         0 ->little endian
 */
int lsion_get_endianness(void);

/*   return if SIONlib supports PTHREADS (Thread-safe) 1 -> yes, 0 -> no  */
int lsion_is_thread_safe(void);

/*   return version numbers */
/*!
 * \brief Return version numbers
 *
 *  return version numbers
 */
int lsion_get_version(int *main_version, int *sub_version, int *patch_level, int *fileformat_version);

typedef struct _sion_io_stat_struct sion_io_stat_t;
struct _sion_io_stat_struct {
  int nfiles;
  const char **names;
  size_t *sizes;
  int *roles;
};

/* return info about accessed (local) files on this task  */
sion_io_stat_t *lsion_get_io_info(int sid);

/* return info about accessed (local) files on this task */
sion_io_stat_t *lsion_get_io_info_by_spec(int sid, int roles, int flag);
int lsion_free_io_info(sion_io_stat_t *info);

/* Seeking */
/// \brief moves the file pointer inside a logical file
///
/// This moves the file pointer inside a logical file.
/// The file pointer is moved by a relative `offset` that is interpreted according to the `whence` argument as follows:
/// - SION_SEEK_BEGIN: offset is relative to the beginning of the file
/// - SION_SEEK_CURRENT: offset is relative to the current position of the file pointer
/// - SION_SEEK_END: offset is relative to the end of the file
/// - SION_SEEK_CHUNK_BEGIN: offset is relative to the beginning of the current chunk
/// - SION_SEEK_CHUNK_BEGIN: offset is relative to the end of the current chunk
///
/// The file pointer must not be moved before the beginning of the file.
/// In read mode, the file pointer must also not be moved beyond the end of the file.
/// When seeking relative to the bounds of the current chunk (SION_SEEK_CHUNK_BEGIN and SION_SEEK_CHUNK_END), seeking outside of
/// the current chunk is not allowed.
///
/// \see lsion_seek_chunk lsion_switch_logical_file
///
/// \param[in,out] sid SIONlib file handle
/// \param[in] offset offset in positions by which to move the file pointer
/// \param[in] whence determines how `offset` should be interpreted
///
/// \returns SION_SUCCESS if file pointer can be moved to new position
int lsion_seek(int sid, int64_t offset, sion_seek_mode whence);

/// \brief moves the file pointer by increments of entire chunks inside the logical file
///
/// Moves the file pointer to the beginning of a chunk that is identified by a relative offset, according to `whence`:
/// - SION_SEEK_BEGIN: seeks from the beginning of the file to the chunk with number `offset`
/// - SION_SEEK_CURRENT: offset is a number of chunks relative to the current chunk
/// - SION_SEEK_END: offset is relative to the end of the file
//
/// The file pointer must not be moved before the beginning of the file.
/// In read mode, the file pointer must also not be moved beyond the end of the file.
///
/// \see lsion_seek lsion_switch_logical_file
///
/// \param[in,out] sid SIONlib file handle
/// \param[in] offset offset in chunks by which to move the file pointer
/// \param[in] whence determines how `offset` should be interpreted
///
/// \returns SION_SUCCESS if file pointer can be moved to new position
int lsion_seek_chunk(int sid, int32_t offset, sion_seek_mode whence);

/// \brief moves the file pointer to a different logical file
///
/// Switches from one logical file in a SIONlib container to another.
/// When opening a SIONlib cointainer, the individual per logical file file pointers start out at the first position of each
/// logical file.
/// However when switching from one logical file to another, the position of the file pointer is persisted and reloaded when
/// switching back.
///
/// \see lsion_seek lsion_seek_chunk
///
/// \param[in,out] sid SIONlib file handle
/// \param[in] file_number the number of the logical file to switch to
///
/// \returns SION_SUCCESS if file pointer can be moved to new position
int lsion_switch_logical_file(int sid, int file_number);

/* register user defined lock and unlock routines, which are needed to access shared data structures internally  */
/*!
 * @brief Function which registers callback funtions for lock and unlock internal access to shared data structures
 *
 *
 * @param[in]   lock()             lock function
 * @param[in]   unlock()           unlock function
 * @param[in]   lock_data()           unlock function
 *
 * @return     SION_SUCCESS okay
 */
int lsion_lock_register_lock_callbacks(int lock(void *), int unlock(void *), void *lock_data);

/*!
 * @brief Function which return flag, if user callback for locking are registered
 *
 * @return     SION_SUCCESS (yes) of SION_NOT_SUCCESS (no)
 */
int lsion_lock_user_callbacks_defined(void);

/* Utilities */
/**
 * @brief Perform byte-order swapping for arrays
 *
 * `n` elements of `size` bytes each are swapped if `do_swap` is `true`.
 * In-place swapping (`target == source`) is allowed.
 * If `target != source`, the buffers must not overlap.
 *
 * @param[out] target the byte-swapped data is written starting at this address
 * @param[in] source the data to be byte-swapped is read starting at this address
 * @param[in] size the size (in bytes) of a single element
 * @param[in] n the number of elements to be byte-swapped
 * @param[in] do_swap byte-swapping is only performed if this argument is `true`
 */
void lsion_swap(void *target, void *source, int size, int n, int do_swap);

char *lsion_get_fname(int sid);
int64_t lsion_get_chunksize(int sid);
int32_t lsion_get_globalrank(int sid);

/// \defgroup keyval Key-Value API
/// @{
///
/// \brief I/O of key-value pairs to SIONlib files
///
/// Introduction
/// ============
///
/// The use of hybrid applications (MPI+OpenMP) suggests a new structural
/// layer to be available in SIONlib. Key-value pairs allow convenient
/// saving and especially loading of multiple pieces of data within one
/// single rank.
///
/// Implementations
/// ===============
///
/// In general the keys are of data type `uint64_t`, while the data is of
/// data type `void *`. All Implementations need to store the amount of
/// data actually written for the specific key. During reads the current
/// position inside a key's data needs to be stored.
///
/// In order to prevent additional global communication keys are local for
/// each.
///
/// Enabling Key Value Mode
/// -------------------------
///
/// Enabling a SIONlib file for usage with the key-value is done by
/// passing the keyword _keyval_ to the open function with the value
/// choosing the method, e.g. `keyval=inline` for the inline
/// implementation.
///
/// Opening a SIONlib file for read mode is possible using the keyval mode
/// "keyval=unknown". Subsequent calls of `lsion_get_keyval_mode(int sid,
/// int foo)`
///
/// ~~~{.c}
/// int lsion_get_keyval_mode(int sid);
/// ~~~
///
/// return the mode which can be compared to constants like
/// SION_KEYVAL_INLINE for inline mode.
///
/// Using an open call without keyval argument fails on a file which is
/// written using keyval.
///
/// General Interface
/// -----------------
///
/// The general interface is
///
/// ~~~{.c}
/// size_t lsion_fwrite_key (void* data, uint64_t key, size_t size, size_t nitems, int sid);
/// size_t lsion_fread_key(void* data, uint64_t key, size_t size, size_t nitems, int sid);
/// ~~~
///
/// Key Information Management
/// --------------------------
///
/// In order to prevent a full search through all previous data for every
/// key read from a SIONlib file and to keep track of the remaining bytes
/// for a key, some caching and management is needed. Every matching or
/// non-matching key which is read from file is added to the "key manager"
/// which hold data of
///
/// - the key itself
/// - the offset of the key
/// - the length of corresponding data
/// - the number of bytes left unread from the key
///
/// Besides managing the pieces of information it also offers an interface
/// for iterating over all available keys which is especially useful for
/// dumping data of all keys.
///
/// The key manager saves information about keys in the "key value
/// table". This hash table consists of two columns of types
///
/// - `uint64_t`
/// - `void *`
///
/// which hold the key and the according bucket, respectively. The buckets
/// use linked lists in order to store the actual data.
///
/// The key manager resides inside the corresponding SIONlib file
/// descriptor and is pointed by `siondesc->keyvalptr`.
///
/// Inline
/// ------
///
/// ### Implementation Details
///
/// From the output point of view the most straight forward implementation
/// is writing the (key, length, data) inside a chunk.
///
/// ~~~
/// +---------------------+---------------------+---------------------+
/// | k_0 | l_0 | d_0 ... | k_1 | l_1 | d_1 ... | k_2 | l_2 | d_2 ... |
/// +-----------------------------------------------------------------+ ...
/// |                             c_0                                 |
/// +-----------------------------------------------------------------+
/// ~~~
///
/// symbol | description
/// ------ | -----------
/// k_i    | key i
/// d_i    | data associated to k_i
/// l_i    | length of d_i
/// c_i    | chunk i
///
/// ### Advantages
///
/// - Simple output implementation
/// - Additional data structures are contained inside the existing ones
///
/// ### Disadvantages
///
/// - Yields a performance penalty if keys are small and read / written
///   consecutively
/// - Since the key information is directly attached to the corresponding
///   data, this format efficiently yields larger data sizes, which is not
///   obvious for the user and might result in large unused disk space
/// - Key manager is needed and might grow significantly large if many
///   keys are used
///
/// Dense (not implemented yet)
/// ---------------------------
///
/// ### Implementation Details
///
/// For consecutive read and write operations separating the data and the
/// key information is preferable. Caching key information makes it
/// possible to write a compact block of information filling up the gaps
/// that are left during output of the associated data.
///
/// The chunks are not shown in proportional size.
///
/// ~~~
/// +-----+-----------------------------------------------+
/// | o_0 | d_0   d_1   d_2 | (k_0 l_0 o_1) (k_1 l_1 o_2) |
/// +-----------------------------------------------------+...
///       |                      c_0                      |
///       +-----------------------------------------------+
///
///    +-----------+-------------------------------------------+
///    | d_3   d_4 | (k_2 l_2 o_3) (k_3 l_3 o_4) (k_4 l_4 o_5) |
/// ...+-----------+-------------------------------------------+...
///    |    c_1    |                    c_2                    |
///    +-----------+-------------------------------------------+
/// ~~~
///
/// symbol | description
/// ------ | -----------
/// k_i    | key i
/// o_i    | offset of key i
/// d_i    | data associated to k_i
/// l_i    | length of d_i
/// c_i    | chunk i
///
/// The rules for writing the key information are
/// - data never follows key information
/// - key information is written at the first possible position
///
/// ### Advantages
///
/// - Allows efficient read / write of consecutive keys.
///
/// ### Disadvantages
///
/// - Needs key manager
/// - Needs more bookkeeping than inline implementation

struct _sion_key_stat_struct;

/*!
 * @brief Writes data for key.
 *
 * Writes data to sion file associating it to key. This function uses the
 * diffenent implementations of which one is chosen with the open call.
 *
 * @param[in]  data    data to be written
 * @param[in]  key     key associated to the data
 * @param[in]  size    size of single item
 * @param[in]  nitems  number of items
 * @param[in]  sid     sion file handle
 *
 * @return  number of successfully written items
 */
size_t lsion_fwrite_key(const void *data, uint64_t key, size_t size, size_t nitems, int sid);

/*!
 * @brief Read data for key.
 *
 * Reads data from sion file associated to key. This function uses the
 * diffenent implementations of which one is chosen with the open call.
 * SIONlib will maintain a file position per key so that subsequent reads
 * are starting at last position of current record.
 *
 * @param[out]  data    buffer to be read into
 * @param[in]   key     key associated to the data
 * @param[in]   size    size of single item
 * @param[in]   nitems  number of items
 * @param[in]   sid     sion file handle
 *
 * @return  number of successfully read items
 */
size_t lsion_fread_key(void *data, uint64_t key, size_t size, size_t nitems, int sid);

/* iterator for key-value pairs */
/*!
 * @brief Resets key iterator.
 *
 * @param[in]   sid     sion file handle
 *
 * @return  SION_SUCCESS on success
 */
int lsion_fread_key_iterator_reset(int sid);

/*!
 * @brief Forward to next key.
 *
 * Get next record for current task, data itself has to be read with
 * lsion_fread_key; this function will forward to next record although if data
 * of last record is not or only read by part
 *
 * @param[in]   sid      sion file handle
 * @param[out]  key   filled with next key
 * @param[out]  size  filled with next size
 *
 * @return  SION_SUCCESS on success
 */
int lsion_fread_key_iterator_next(int sid, uint64_t *key, size_t *size);

/*!
 * @brief Seek to position in key.
 *
 * This function searches for a key and on success the file position will
 * be move to the searched key and position, so that the data can be read with
 * lsion_fread_key.
 *
 * @param[in]  sid         sion file handle
 * @param[in]  key         key to seek in
 * @param[in]  entrynum    nth occurrence of key or SION_ABSOLUTE_POS for
 *                         absolute position
 * @param[in]  posinentry  byte-position in entry, or absolute byte-position in
 *                         all entries in write-order (SION_ABSOLUTE_POS)
 *
 * @return  SION_SUCCESS on success
 */
int lsion_seek_key(int sid, uint64_t key, int entrynum, int64_t posinentry);

/*!
 * @brief Performs a full scan of all meta data in current file.
 *
 * @param[in]   sid     sion file handle
 *
 * @return  SION_SUCCESS on success
 */
int lsion_key_full_scan(int sid);

/*!
 * @brief Resets key iterator.
 *
 * @param[in]   sid     sion file handle
 *
 * @return  SION_SUCCESS on success
 */
int lsion_key_list_iterator_reset(int sid);

/*!
 * @brief Forward to next key.
 *
 * @param[in]   sid      sion file handle
 * @param[out]  keyptr   filled with next key
 *
 * @return  SION_SUCCESS on success
 */
int lsion_key_list_iterator_next(int sid, uint64_t *keyptr);

typedef struct _sion_key_stat_struct sion_key_stat_t;
struct _sion_key_stat_struct {
  uint64_t key;
  int64_t num_blocks;
  int64_t total_size;
};

/*!
 * @brief get statistics about key
 *
 * @param[in]   sid         sion file handle
 * @param[in]   key         key
 * @param[out]  stat  data structure which will be filled
 *
 * @return  SION_SUCCESS on success
 */
int lsion_key_get_stat(int sid, uint64_t key, sion_key_stat_t *stat);

/*!
 * @brief Return selected mode for key value.
 *
 * @param[in]  sid  sion file handle
 *
 * @return  keyval mode, e.g. SION_KEYVAL_INLINE
 */
int lsion_get_keyval_mode(int sid);

/*!
 * @brief Returns key value mode as string.
 *
 * @param[in]  type  mode to be converted
 *
 * @return  string representation of mode
 */
char *lsion_keyval_type_to_str(int type);

/// @}

#ifdef __cplusplus
}
#endif

/* Debugging  */
#ifndef DPRINTF
#define DPRINTF(A)
#define DPRINTFP(A)
#define DPRINTTS(A, B)
#define DPRINTFTS2(A, B)
#endif

/// @}

#endif
