/****************************************************************************
**  SIONLIB     http://www.fz-juelich.de/jsc/sionlib                       **
*****************************************************************************
**  Copyright (c) 2008-2009                                                **
**  Forschungszentrum Juelich, Juelich Supercomputing Centre               **
**                                                                         **
**  See the file COPYRIGHT in the package base directory for details       **
****************************************************************************/
#ifndef SION_SION_H
#define SION_SION_H

#define SION_MAIN_VERSION 1

#define SION_SUB_VERSION 1
#define SION_VERSION_PATCHLEVEL 007
#define SION_FILEFORMAT_VERSION 3

#ifdef SION_DEBUG
#define DPRINTF(A) sion_dprintf A
#define DPRINTFP(A) sion_dprintfp A
#define DPRINTFTS(rank,desc) sion_dprintfp(128,"TIMING",rank,"                                                                           step=%-20s timestamp=%20s",desc,_sion_get_time_asc())
#define DPRINTFTS2(rank,desc) sion_dprintfp(256,"TIMING",rank,"                                                                           step=%-20s timestamp=%20s",desc,_sion_get_time_asc())
#else
#define DPRINTF(A)
#define DPRINTFP(A)
#define DPRINTFTS(A,B)
#define DPRINTFTS2(A,B)
#endif

#ifdef __cplusplus
extern "C" {
#endif


/* Blue Gene/L */
#if defined(__blrts__)

#define _SION_BGL
#if !defined(_BGL)
#define _BGL
#endif

/* Blue Gene/P */
#elif defined(__bgp__)

#if !defined(_SION_BGP)
#define _SION_BGP
#endif
#if !defined(_BGP)
#define _BGP
#endif

/* AIX */
#elif defined(_AIX)
#if !defined(_SION_AIX)
#define _SION_AIX
#endif

/* Cray XT (catamount) */
#elif defined(__LIBCATAMOUNT__)
#define _SION_CATAMOUNT

/* Sun Solaris systems */
#elif defined(__sun) && defined(__SVR4)
#define _SION_SUNSOL

#elif defined(__linux__)
#if !defined(_SION_LINUX)
#define _SION_LINUX
#endif

#endif


#if defined(_SION_LINUX)
#define sion_int32 int
#define sion_uint32 unsigned int
#define sion_int64 long long
#elif defined(_SION_AIX)
#define sion_int32 int
#define sion_uint32 unsigned int
#define sion_int64 long long
#elif defined(_SION_BGL)
#define sion_int32 int
#define sion_uint32 unsigned int
#define sion_int64 long long
#elif defined(_SION_BGP)
#define sion_int32 int
#define sion_uint32 unsigned int
#define sion_int64 long long
#elif defined(_SION_XT)
#define sion_int32 int
#define sion_uint32 unsigned int
#define sion_int64 long
#endif

#define SION_MAX_FSBLOCKSIZE 32*1024*1024
#define SION_DEFAULT_FSBLOCKSIZE 2*1024*1024


#define SION_FILENAME_LENGTH 1024

#define SION_DIFFSIZE  1        /*!< Each rank has a different chunk size (default) */
#define SION_DUMMY     2

#define MAXBLOCKS 100   /*!< Initial maximum number of blocks in a sion file */

#ifdef SION_MPI
#include "mpi.h"

#if defined(_SION_LINUX)
#define SION_MPI_INT32 MPI_INT
#define SION_MPI_INT64 MPI_LONG_LONG
#elif defined(_SION_AIX)
#define SION_MPI_INT32 MPI_INT
#define SION_MPI_INT64 MPI_LONG_LONG
#elif defined(_SION_BGL)
#define SION_MPI_INT32 MPI_INTEGER4
#define SION_MPI_INT64 MPI_INTEGER8
#elif defined(_SION_BGP)
#define SION_MPI_INT32 MPI_INTEGER4
#define SION_MPI_INT64 MPI_INTEGER8
#elif defined(_SION_XT)
#define SION_MPI_INT32 MPI_INTEGER4
#define SION_MPI_INT64 MPI_INTEGER8
#endif

  /* Generic MPI Wrapper function */
  int sion_paropen_mpi(    char          *fname,
			   char          *file_mode,
			   int           *numFiles,
			   MPI_Comm       gComm,
			   MPI_Comm      *lComm,
			   sion_int64    *chunksize,
			   sion_int32    *fsblksize,
			   int           *globalrank,
			   FILE         **fileptr,
			   char         **newfname);

  int sion_paropen_multi_mpi(char *fname,
                           char *file_mode,
                           int  *numFiles,
                           MPI_Comm gComm,
                           MPI_Comm *lComm,
                           sion_int64 *chunksize,
                           sion_int32 *fsblksize,
                           int  *globalrank,
                           FILE **fileptr,
                           char *newfname);

  int sion_parclose_mpi(   int      sid  );

  int sion_paropen_comms_mpi(char *fname,
                           char *file_mode,
                           int *numFiles,
                           MPI_Comm gComm,
                           MPI_Comm lComm,
                           sion_int64 *chunksize,
                           sion_int32 *fsblksize,
                           int  *globalrank,
                           FILE **fileptr,
                           char *newfname);

  /* Help functions */
  int _sion_split_comm_mpi( int       numComm,
			    MPI_Comm  gComm,
			    MPI_Comm  *lComm,
			    int       *color);
  
  int _sion_get_color_from_splitted_comm_mpi( MPI_Comm gComm,
					      MPI_Comm lComm,
					      int *numComm,
					      int *color);
  
  int _sion_get_numfiles_from_file_mpi( char *fname);

#define SION_OLD_VERSION_GET_FILENUMno
#ifdef SION_OLD_VERSION_GET_FILENUM
  int _sion_get_filenumber_from_files_mpi_old( char *fname,
					   MPI_Comm gComm,
					   int numfiles);
#else
  int _sion_get_filenumber_from_files_mpi( char    *fname,
					   MPI_Comm gComm,
					   int     *numfiles,
					   int     *filenumber,
					   int     *lRank);
#endif


  int _gather_cb_mpi(void *indata, void *outdata, void *commvp, int dtype);
  int _scatter_cb_mpi(void *indata, void *outdata, void *commvp, int dtype);
  int _bcast_cb_mpi(void *data, void *commvp, int dtype);
  int _barrier_cb_mpi(void *commvp, void *dummy);

  int _sion_errorprint_mpi(int rc, int level, const char *format, ...);

#endif

  /*!
   * @brief Sion File Descriptor Alias
   */
  typedef struct _sion_filedesc_struct _sion_filedesc;

  int sion_open(    char *fname,
                    char* file_mode,
                    int  *ntasks,
                    int  *nfiles,
		    sion_int64 **chunksizes,
                    sion_int32  *fsblksize,
                    int  **globalranks,
                    FILE **fileptr);

  int sion_open_rank(    char  *fname,
			 char  *file_mode,
			 sion_int64  *chunksize,
			 sion_int32  *fsblksize,
			 int   *rank,
			 FILE **fileptr);

  size_t sion_fwrite(const void *data, size_t size, size_t nitems, int sid);
  size_t sion_fread( void *ptr, size_t size, size_t nmemb, int sid);

  int sion_feof(int sid);

  int sion_flush(int sid);

  int sion_close(int sid);

  int sion_set_fp_closed(int sid);

  /* needed for files opened again with fdopen(dup(...)) */
  /* TODO: _fp or _fd  */
  int sion_set_second_fp(int sid, FILE *fp);
  int sion_unset_second_fp(int sid);

  FILE * sion_get_fp(int sid);

  int sion_ensure_free_space(int sid, sion_int64 bytes);

  int sion_get_file_endianness(int sid);

  /* FILE INFO with sion datatypes */
  int sion_get_locations(    int    sid,
			     int   *ntasks,
			     int   *maxblocks,
			     sion_int64 *globalskip,
			     sion_int64 *start_of_varheader,
			     sion_int64 **sion_localsizes,
			     sion_int64 **sion_globalranks,
			     sion_int64 **sion_blockcount,
			     sion_int64 **sion_blocksizes);

  /* parallel mode */
  int sion_get_current_location(int sid, 
				int *currentblock, 
				sion_int64 *currentpos,  
				int *numchunks, 
				sion_int64 **chunksizes);

  /* serial mode */
  int sion_get_current_locations(    int sid,
				     int *ntasks,
				     sion_int64 **sion_currentpos,
				     sion_int64 **sion_currentblocknr );

  int sion_get_number_of_files(   int sid );
  int sion_get_filenumber(  int sid );

  sion_int64 sion_get_bytes_written(int sid);
  sion_int64 sion_get_bytes_read(int sid);


#define SION_CURRENT_RANK        -101  /*!< Alias for the current rank */
#define SION_CURRENT_BLK         -102  /*!< Alias for the current block */
#define SION_CURRENT_CHUNK       -102  /*!< Alias for the current block */
#define SION_CURRENT_POS         -103  /*!< Alias for the current position in the current block */
#define SION_ABSOLUTE_POS        -104  /*!< Alias for the flag to specify in sion_seek an absolute position in bytestream */

  int sion_seek(  int  sid,
		  int  rank,
		  int  currentblocknr,
		  sion_int64 posinblk );

  int sion_seek_fp(  int  sid,
		     int  rank,
		     int  currentblocknr,
		     sion_int64 posinblk,
		     FILE **fileptr
		     );

  /* get number of bytes not read in current block */
  sion_int64 sion_bytes_avail_in_block( int sid );
  sion_int64 sion_bytes_avail_in_chunk( int sid );

/* defined in sion_debug.c */
  int sion_dprintf(int mask, const char *format, ...);
  int sion_dprintfp(int mask, const char *callfunction, int rank, const char *format, ...);
  int sion_isdebug(void);
  void sion_debug_on(int mask, const char *filename);
  void sion_debug_off(void);
  FILE *sion_get_dfile(void);
  void sion_dclose(void);

  int sion_feof(int sid);

  double _sion_get_time();
  char  *_sion_get_time_asc();

  /*   return current endianness (1-> big endian, 0 ->little endian) */
  int sion_get_endianness();


/*  perform byte-order swapping for arrays n units of size byte are
    swapped if and only if aflag==0 in-place swapping (target==source)
    is allowed if target != source, the buffers must not overlap */
  void sion_swap(void *target, void *source, int size, int n, int aflag);


/*! \internal 
  INTERNAL defines */
#define SIONFP ((sion_filedesc->secondfd==NULL) ? sion_filedesc->fd : sion_filedesc->secondfd)
#define SION_ID "sion"  /*!< Sion identification string (offset: 0x00) */

/* Datatypes used for the collective operations */
#define _SION_INT32     10
#define _SION_INT64     11
#define _SION_CHAR      12

  int _sion_paropen_multi_generic(char  *fname,
                                  char  *file_mode,
                                  char  *prefix,
                                  int   *numFiles,
				  int   *filenumber, 
                                  sion_int64  *chunksize,
                                  sion_int32  *fsblksize,
                                  int    rank,
                                  int    ntasks,
                                  int   *globalrank,
                                  int    flag,
                                  FILE **fileptr,
                                  int    gather_cb(void * , void *, void *, int),
                                  int    scatter_cb(void *, void *, void *, int),
                                  int    bcast_cb(void *, void *, int),
                                  int    barrier_cb(void *, void *),
                                  void  *cb_parm1,
                                  void  *cb_parm2 );


  int _sion_parclose_generic(   int    sid,
                                int    rank,
                                int    ntasks,
				int    mapping_size,
				sion_int32 *mapping,
                                int    gather_cb(void * , void *, void *, int),
                                int    scatter_cb(void *, void *, void *, int),
                                int    bcast_cb(void *, void *, int),
                                int    barrier_cb(void *, void *),
                                void  *cb_parm1,
                                void  *cb_parm2 );


  int _sion_calculate_startpointers(    int   ntasks,
                                        sion_int64 *chunksizes,
                                        sion_int32  fsblksize,
                                        sion_int64 *startpointers,
                                        sion_int64 *globalskip);

  FILE * _sion_open_file_for_read( char *fname);
  FILE * _sion_create_and_open_file_for_write( char *fname);
  FILE * _sion_open_existing_file_for_write( char *fname);

  int _sion_stat_file( char *fname);
  long _sion_get_opt_blksize( FILE *fileptr);

  int _sion_write_header(    FILE *fileptr,
                             sion_int32  fsblksize,
                             int  ntasks,
                             sion_int64 *globalranks,
                             sion_int64 *chunksizes,
                             int  nfiles,
			     int  filenumber,
			     sion_int64  flag1,
			     sion_int64  flag2,
                             char *prefix,
			     int  endianness);

  int _sion_read_header_fixed_part(    FILE *fileptr,
                                       int  *endianness,
				       int  *swapbytes,
                                       sion_int32 *fsblksize,
                                       int *ntasks,
                                       int *nfiles,
                                       int *filenumber,
				       sion_int64  *flag1,
				       sion_int64  *flag2,
                                       char *prefix,
                                       sion_int32 *sion_version,
                                       sion_int32 *sion_version_patchlevel,
                                       sion_int32 *sion_fileformat_version);
  
  int _sion_read_header_var_part(    FILE *fileptr,
                                     int  ntasks,
                                     sion_int64 *globalranks,
                                     sion_int64 *chunksizes,
                                     sion_int32  *maxblocks,
                                     sion_int64 *start_of_varheader,
				     int swapbytes);

  int _sion_read_header_var_part_locations(    FILE *fileptr,
                                               int  ntasks,
                                               sion_int32  maxblocks,
                                               sion_int64  start_of_varheader,
                                               sion_int64 *sion_blockcount,
                                               sion_int64 *sion_blocksizes,
					       int swapbytes);

  int _sion_read_header_var_part_locations_rank(    FILE *fileptr,
                                                    int   rank,
                                                    int   ntasks,
                                                    sion_int64  start_of_varheader,
                                                    sion_int64 *numblocks,
                                                    sion_int64 *blocksizes,
                                                    _sion_filedesc *sion_filedesc, 
						    int swapbytes);

int _sion_read_header_var_part_mapping(FILE  *fileptr,
				       int         ntasks,
				       sion_int32  maxblocks,
				       sion_int64  start_of_varheader,
				       int         *mapping_size,
				       sion_int32 **mapping,
				       int          swapbytes);

int _sion_read_header_var_part_mapping_rank(FILE        *fileptr,
					    int          rank,
					    int          ntasks,
					    sion_int32   maxblocks,
					    sion_int64   start_of_varheader,
					    int         *mapping_filenr,
					    int         *mapping_lrank,
					    int          swapbytes);

int sion_get_mapping( int          sid,
		      int         *mapping_size,
		      sion_int32 **mapping,
		      int         *numfiles );
  
  sion_int64 sion_get_position( int sid );

  int _sion_set_position(    FILE *fileptr,
			     sion_int64  startpointer);

  sion_int64 _sion_get_position(    FILE *fileptr );

  int _sion_flush_file( int sid );
  int _sion_flush_file_fp(FILE *fileptr );
  int _sion_purge_file_fp(FILE *fileptr );

  /* Error types */
#define _SION_ERROR_RETURN         -10001
#define _SION_ERROR_ABORT          -10002
#define _SION_ERROR_WARN           -10003
#define _SION_ERROR_UNKNOWN        -10020

  int __sion_error(int rc, int level, char * errmsg);
  int _sion_errorprint(int rc, int level, const char *format, ...);


  int _sion_allocate_sion_arrays(    int   ntasks,
                                     sion_int64 **sion_chunksizes,
                                     sion_int64 **sion_startpointers,
                                     sion_int64 **sion_globalranks);

  int _sion_free_sion_arrays(sion_int64 **sion_chunksizes, 
			     sion_int64 **sion_startpointers, 
			     sion_int64 **sion_globalranks);


  int _sion_allocate_sion_arrays_locations(    int   ntasks,
                                               sion_int32   maxblocks,
                                               sion_int64 **sion_blockcount,
                                               sion_int64 **sion_currentpos,
                                               sion_int64 **sion_currentblocknr,
                                               sion_int64 **sion_blocksizes);

  int _sion_deallocate_sion_arrays(sion_int64 **sion_chunksizes,
                                   sion_int64 **sion_startpointers,
                                   sion_int64 **sion_globalranks);

  int _sion_deallocate_sion_arrays_locations(    int   ntasks,
                                                 sion_int32   maxblocks,
                                                 sion_int64 **sion_blockcount,
                                                 sion_int64 **sion_currentpos,
                                                 sion_int64 **sion_currentblocknr,
                                                 sion_int64 **sion_blocksizes);

  int _sion_create_new_block( int  sid );
  int _sion_flush_block( int  sid );
  int _sion_init_filedesc(  _sion_filedesc *sion_filedesc) ;
  int _sion_print_filedesc( _sion_filedesc *sion_filedesc, int level, char *desc, int all);
  int _sion_free_filedesc(  _sion_filedesc *sion_filedesc);

  char * _sion_get_multi_filename(char *fname, int filenumber);


#define SION_FILESTATE_UNKNOWN       -1         /*!< Unknown file state(error) */
#define SION_FILESTATE_PAROPEN       0          /*!< File opened in parallel */
#define SION_FILESTATE_SEROPEN       1          /*!< File opened in serial */
#define SION_FILESTATE_SEROPENRANK   2          /*!< File opened in serial for a specific rank */
#define SION_FILESTATE_CLOSE        10          /*!< File closed */
#define SION_FILEMODE_UNKNOWN       -20         /*!< File mode unknown (error) */
#define SION_FILEMODE_READ          20          /*!< File opened for reading */
#define SION_FILEMODE_WRITE         30          /*!< File opened for writing */

#define SION_FILEDESCRIPTOR 11  /*!< int id -> data structure mapping (from visit library) */


#define _SION_FLAG1_NONE      0                 /*!< Flag1 not used */
#define _SION_FLAG2_NONE      0                 /*!< Flag2 not used */


/*!
 * \struct _sion_filedesc_struct
 * \brief Sion File Descriptor Structure
 *
 * The sion file descriptor structure stores all the properties for the sion file.
 * This properties are the META data of a sion file
 */
  struct _sion_filedesc_struct {
    FILE *fd;                   /*!< Pointer to the current sion file */
    FILE *secondfd;             /*!< Pointer to a second sion file (Scalasca) */
    char *fname;                /*!< Name of the file */
    int   rank;                 /*!< Rank of the task writting the file */
    int   ntasks;               /*!< Number of tasks using the sion file */
    int   state;                /*!< INTERNAL STATE */
    int   mode;                 /*!< Mode used to open the file(rb/wb) */
    int   endianness;           /*!< Endianness used */
    int   swapbytes;            /*!< swapping bytes is needed for this sion file  */
    int   fileversion;          /*!< Version number of sion file format */
    int   fsblksize;		/*!< Filesystem block size */
    int   numblocks;		/*!< Current number of the block (0..n) */
    int   maxblocks;            /*!< Max number of chunks per task */
    sion_int64 flag1;           /*!< placeholder for future flags */            
    sion_int64 flag2;           /*!< placeholder for future flags */            
    sion_int64 startpos;        /*!< Start position for the current task */
    sion_int64 chunksize;       /*!< Default chunksize for the current task(From Meta block 1) */
    sion_int64 *chunksizes;     /*!< Array of the all chunksizes for the current task */

    sion_int64 globalskip;      /*!< number of bytes to skip to next block of same processor */
    sion_int64 end_of_header;   /*!< End of META block 1 (beginning of the sion file) */
    sion_int64 start_of_varheader;      /*!< Beginning of the META block 2 (end of the sion file) */
    sion_int64 start_of_data;   /*!< Start of the data in the sion file */

    sion_int64 *all_chunksizes;        /*!< only used if sion is opened from a single task for read  */
    sion_int64 *all_globalranks;       /*!< only used if sion is opened from a single task for read  */
    sion_int64 *all_startpointers;     /*!< only used if sion is opened from a single task for read  */

    sion_int64 *all_currentpos;        /*!< only used if sion is opened from a single task for read  */
    sion_int64 *all_currentblocknr;    /*!< only used if sion is opened from a single task for read  */

    sion_int64 *all_blockcount;        /*!< size, only used if sion is opened from a single task for read  */
    sion_int64 *all_blocksizes;        /*!< size*maxblocks, only used if sion is opened from a single task for read  */

    sion_int64 currentpos;	       /*!< only used if opened  for one rank, reading */
    int currentblocknr;                /*!< only used if opened  for one rank, reading */
    int debug;                         /*!< 0 or 1 for debugging */

    int   nfiles;               /*!< Number of files in the set */
    int   filenumber;           /*!< file number in the set */
    int    mapping_size;        /*!< size of mapping table, -1 if not available */
    sion_int32 *mapping;        /*!< pointer to mapping table if reading multi file */

    char *prefix;               /*!< Filenames of all files in the set */
    char *newfname;             /*!< Filenames of current files (multi-file) */

#ifdef SION_MPI
    MPI_Comm   lComm;           /*!< Local communicator */
    MPI_Comm   gComm;           /*!< Global communicator */
#endif

  };




  int _sion_realloc_blocklist(  _sion_filedesc *sion_filedesc,
                                sion_int32 maxblocks);

#ifdef SION_PTHREADS
#define SION_FDDATA_INITIALIZER {0,0,-1,NULL,PTHREAD_MUTEX_INITIALIZER}
#else
#define SION_FDDATA_INITIALIZER {0,0,-1,NULL}
#endif


#ifdef __cplusplus
}
#endif

/* Examples */
/*! \example parallel_write.example
 *
 * This is an example of how to use sionlib for parallel writing
 * to a sion file
 *
 * @remarks	the call to sion_ensure_free_space can be omitted, if it is guaranteed
 * that not more bytes as specified in chunksize are written to the file.
 *
 * @remarks	chunksize can be different on different tasks
 *
 * @remarks	if the data does not fit in the current chunk, sion_ensure_free_space
 * would assign a new chunk in the file to the task and also advance
 * the file pointer to the new position. This operation is non-collective;
 * all information about the locally used chunks and the number of
 * bytes written to the chunks are buffered in memory and stored in the
 * sion file in the collective sion_parclose_mpi.
 *
 * @remarks	one parameter of the open function is an MPI communicator,
 * which allows opening a sion file from a subset of all MPI tasks
 */

/*! \example parallel_read.example
 *
 * This is an example of how to use sionlib for parallel reading
 * from a sion file
 *
 * @remarks		it must be ensured that the number of bytes read from a chunk is
 * not larger as the number of bytes written to it. For this the function
 * sion_bytes_avail_in_block provides this number for the current chunk.
 * @remarks		if all bytes of a chunk are already read and there are more chunks
 * available for this task, sion_feof will advance the filepointer to
 * the start position of the next chunk in the sion file
 * @remarks		sion_paropen_mpi is collective. The meta data will read only by one
 * task and broadcasted to all other tasks. It is also possible to
 * open/close the sion file on each with serial sion functions
 * without collective operations, see 3).
 */

/*! \example parallel_read_nocol.example
 *
 * This is an example of how to use sionlib for parallel reading
 * from a sion file (without collectives)
 *
 * @remarks the meta-information of the sion file will be read from each task.
 * This is a parallel access to same filesystem blocks, however this
 * are only read accesses which should not be lock the filesystem block.
 */

/*! \example serial_write.example
 *
 * This is an example of how to use sionlib for writing to
 * a sion file in serial.
 *
 * @remarks access to all ranks is possible (sion_seek)
 * @remarks the call to sion_ensure_free_space can be omitted, if it is guaranteed
 * that not more bytes as specified in chunksize are written to the file.
 */

/*! \example serial_read.example
 *
 * This is an example of how to use sionlib for reading to
 * a sion file in serial.
 *
 * @remarks	access to all ranks and chunks is possible (sion_seek)
 * @remarks	sion_get_locations returns pointers to internal fields, containing the number
 * of chunks written by each task (sion_chunkcount) and their sizes (sion_chunksizes)
 */
#endif
