SIONlib  1.7.7
Scalable I/O library for parallel access to task-local files
sion_common.h
Go to the documentation of this file.
1 /****************************************************************************
2 ** SIONLIB http://www.fz-juelich.de/jsc/sionlib **
3 *****************************************************************************
4 ** Copyright (c) 2008-2019 **
5 ** Forschungszentrum Juelich, Juelich Supercomputing Centre **
6 ** **
7 ** See the file COPYRIGHT in the package base directory for details **
8 ****************************************************************************/
9 
16 #ifndef SION_SION_COMMON_H
17 #define SION_SION_COMMON_H
18 
19 /* SIONlib PUBLIC Interface: common */
20 
21 #include <stdio.h>
22 #include <stdint.h>
23 
24 #include "sion_const.h"
25 #include "sion_datatypes.h"
26 
27 /* default mode */
28 #define SION_SERIAL_MASTER
29 
30 #define SION_MAX_FSBLOCKSIZE 32*1024*1024
31 #define SION_DEFAULT_FSBLOCKSIZE 2*1024*1024
32 
33 #define SION_FILENAME_LENGTH 1024
34 
35 #define MAXCHUNKS 100
37 #define WFLARGEMETABLOCK_none /* for BENCHMARKING only */
38 
39 #ifdef __cplusplus
40 extern "C" {
41 #endif
42 
43 
44  int _sion_close_sid(int sid);
45 
46  /* I/O */
47  size_t sion_fwrite(const void *data, size_t size, size_t nitems, int sid);
48  size_t sion_fread( void *data, size_t size, size_t nitems, int sid);
49 
50  /* I/O: key-value */
51  size_t sion_fwrite_key(const void *data, uint64_t key, size_t size, size_t nitems, int sid);
52  size_t sion_fread_key(void *data, uint64_t key, size_t size, size_t nitems, int sid);
53 
54  /* iterator for key-value pairs */
55  int sion_fread_key_iterator_reset(int sid);
56  int sion_fread_key_iterator_next(int sid, uint64_t *key, size_t *size);
57 
58  /* seek for key-value */
59  int sion_seek_key(int sid, uint64_t key, int entrynum, sion_int64 posinentry );
60 
61  /* scans input file for all information about all keys of this rank */
62  int sion_key_full_scan(int sid);
63 
64  /* iterator for list of keys */
65  int sion_key_list_iterator_reset(int sid);
66  int sion_key_list_iterator_next(int sid, uint64_t *keyptr);
67 
68  typedef struct _sion_key_stat_struct sion_key_stat_t;
70  uint64_t key;
71  sion_int64 num_blocks;
72  sion_int64 total_size;
73  };
74 
75  /* fills a stat-data structure containing information about the key */
76  int sion_key_get_stat(int sid, uint64_t key, sion_key_stat_t *stat);
77 
78  /* helper functions */
79  int sion_feof(int sid);
80  int sion_ensure_free_space(int sid, sion_int64 bytes);
81 
82  int sion_flush(int sid);
83 
84  /* get size of internal data structure */
85  int sion_get_sizeof(int sid, int *numbytes, int *numfds);
86 
87  /* duplicate fd (only parallel read-mode) */
88  int sion_dup(int sid, int mode, int rank, uint64_t key);
89  int sion_dedup(int sid);
90 
91  /* special helper functions, for used with szlib */
92  int sion_set_fp_closed(int sid);
93  int sion_set_second_fp(int sid, FILE *fp);
94  int sion_unset_second_fp(int sid);
95  FILE * sion_get_fp(int sid);
96  int sion_get_fd(int sid);
97  int sion_optimize_fp_buffer(int sid);
98 
99 
100  /* get information (with sion datatypes) */
101  int sion_get_file_endianness(int sid);
102  int sion_endianness_swap_needed(int sid);
103  sion_int64 sion_get_bytes_written(int sid);
104  sion_int64 sion_get_bytes_read(int sid);
105  sion_int64 sion_bytes_avail_in_block( int sid );
106  sion_int64 sion_bytes_avail_in_chunk( int sid );
107  sion_int64 sion_get_position( int sid );
108 
109  int sion_get_mapping( int sid,
110  int *mapping_size,
111  sion_int32 **mapping,
112  int *numfiles );
113 
114  int sion_get_locations( int sid,
115  int *ntasks,
116  int *maxblocks,
117  sion_int64 *globalskip,
118  sion_int64 *start_of_varheader,
119  sion_int64 **sion_chunksizes,
120  sion_int64 **sion_globalranks,
121  sion_int64 **sion_blockcount,
122  sion_int64 **sion_blocksizes);
123  /* parallel mode */
124  int sion_get_current_location( int sid,
125  int *currentblock,
126  sion_int64 *currentpos,
127  int *maxchunks,
128  sion_int64 **chunksizes );
129  /* serial mode */
130  int sion_get_current_locations( int sid,
131  int *ntasks,
132  sion_int64 **sion_currentpos,
133  sion_int64 **sion_currentblocknr );
134  int sion_get_number_of_files( int sid );
135  int sion_get_filenumber( int sid );
136 
137  /* 1 is opened with sion_open or sion_open_rank */
138  int sion_is_serial_opened( int sid );
139 
140  /* 1 if using GPFS or Linux hints */
141  int sion_using_hints(int sid);
142 
143  /* return current endianness (1-> big endian, 0 ->little endian) */
144  int sion_get_endianness(void);
145 
146  /* return if SIONlib supports PTHREADS (Thread-safe) 1 -> yes, 0 -> no */
147  int sion_is_thread_safe(void);
148 
149  /* return keyvalue mode (constants described in sion_const.h) */
150  int sion_get_keyval_mode(int sid);
151  char* sion_keyval_type_to_str(int type); /* converts keyval mode to character string */
152 
153 
154  /* return version numbers */
155  int sion_get_version(int *main_version, int *sub_version, int *patch_level, int *fileformat_version);
156 
157  typedef struct _sion_io_stat_struct sion_io_stat_t;
159  int nfiles;
160  const char **names;
161  size_t *sizes;
162  int *roles;
163  };
164 
165  /* return info about accessed (local) files on this task */
166  sion_io_stat_t* sion_get_io_info(int sid);
167 
168  /* return info about accessed (local) files on this task */
169  sion_io_stat_t* sion_get_io_info_by_spec(int sid, int roles, int flag);
170  int sion_free_io_info(sion_io_stat_t *info);
171 
172 
173  /* Seeking */
174  int sion_seek( int sid,
175  int rank,
176  int currentblocknr,
177  sion_int64 posinblk );
178 
179  int sion_seek_fp( int sid,
180  int rank,
181  int currentblocknr,
182  sion_int64 posinblk,
183  FILE **fileptr
184  );
185 
186  /* register user defined lock and unlock routines, which are needed to access shared data structures internally */
187  int sion_lock_register_lock_callbacks(int lock(void *), int unlock(void *), void *lock_data);
189 
190 
191  /* Utilities */
192  void sion_swap(void *target, void *source, int size, int n, int do_swap);
193 
194  /* for backward compatibility */
195 #define _sion_stat_file _sion_file_stat_file
196  int _sion_file_stat_file(const char *fname);
197 
198 #ifdef __cplusplus
199 }
200 #endif
201 
202  /* Debugging */
203 #ifndef DPRINTF
204 #define DPRINTF(A)
205 #define DPRINTFP(A)
206 #define DPRINTTS(A,B)
207 #define DPRINTFTS2(A,B)
208 #endif
209 
210 
211 #endif
char * sion_keyval_type_to_str(int type)
Returns key value mode as string.
size_t sion_fwrite_key(const void *data, uint64_t key, size_t size, size_t nitems, int sid)
Writes data for key.
int sion_seek_key(int sid, uint64_t key, int entrynum, sion_int64 posinentry)
Seek to position in key.
int sion_get_keyval_mode(int sid)
Return selected mode for key value.
int sion_key_list_iterator_reset(int sid)
Resets key iterator.
int sion_key_full_scan(int sid)
Performs a full scan of all meta data in current file.
size_t sion_fread_key(void *data, uint64_t key, size_t size, size_t nitems, int sid)
Read data for key.
int sion_fread_key_iterator_reset(int sid)
Resets key iterator.
int sion_key_list_iterator_next(int sid, uint64_t *keyptr)
Forward to next key.
int sion_key_get_stat(int sid, uint64_t key, sion_key_stat_t *stat)
get statistics about key
int sion_fread_key_iterator_next(int sid, uint64_t *key, size_t *size)
Forward to next key.
int sion_get_current_location(int sid, int *currentblock, sion_int64 *currentpos, int *maxchunks, sion_int64 **chunksizes)
Returns current position in file and pointer fiels containing chunk sizes.
Definition: sion_common.c:185
int sion_get_current_locations(int sid, int *ntasks, sion_int64 **sion_currentpos, sion_int64 **sion_currentblocknr)
Returns current position in file and pointer fiels containing chunk sizes.
Definition: sion_common.c:307
size_t sion_fread(void *data, size_t size, size_t nitems, int sid)
Read data from sion file.
Definition: sion_common.c:609
int sion_get_file_endianness(int sid)
Returns edianness of data in file sid.
Definition: sion_common.c:253
int sion_endianness_swap_needed(int sid)
Returns whether or not byte swapping is needed for sid.
Definition: sion_common.c:277
int sion_get_locations(int sid, int *ntasks, int *maxblocks, sion_int64 *globalskip, sion_int64 *start_of_varheader, sion_int64 **sion_chunksizes, sion_int64 **sion_globalranks, sion_int64 **sion_blockcount, sion_int64 **sion_blocksizes)
Returns pointers to internal fields.
Definition: sion_common.c:84
int sion_get_mapping(int sid, int *mapping_size, sion_int32 **mapping, int *numfiles)
Returns pointers to the internal field mapping.
Definition: sion_common.c:219
size_t sion_fwrite(const void *data, size_t size, size_t nitems, int sid)
Write data to sion file.
Definition: sion_common.c:470
sion_int64 sion_get_position(int sid)
Function that returns the current file position.
Definition: sion_common.c:930
int sion_seek_fp(int sid, int rank, int currentblocknr, sion_int64 posinblk, FILE **fileptr)
Deprecated. Use sion_seek() instead.
Definition: sion_common.c:721
sion_int64 sion_bytes_avail_in_chunk(int sid)
Function that returns the number of bytes available in the current chunk.
Definition: sion_common.c:894
int sion_flush(int sid)
Flushed sion file.
Definition: sion_common.c:1030
int sion_ensure_free_space(int sid, sion_int64 bytes)
Funtion to ensure that enough space is available for writing.
Definition: sion_common.c:1053
int sion_feof(int sid)
Function that indicates whether the end of file is reached for this task.
Definition: sion_common.c:809
int sion_get_version(int *main_version, int *sub_version, int *patch_level, int *fileformat_version)
Return version numbers.
Definition: sion_tools.c:51
int sion_get_endianness(void)
Return endianness.
Definition: sion_tools.c:32
sion_int64 sion_bytes_avail_in_block(int sid)
Return the number of bytes available in the current chunk.
Definition: sion_common.c:879
int _sion_file_stat_file(const char *fname)
Check if file exists (LARGE_FILE support on BlueGene)
Definition: sion_file.c:318
void sion_swap(void *target, void *source, int size, int n, int do_swap)
Definition: sion_convert.c:44
int sion_seek(int sid, int rank, int currentblocknr, sion_int64 posinblk)
Function to set the file pointer to a new position.
Definition: sion_common.c:698
int sion_dup(int sid, int mode, int rank, uint64_t key)
Function which duplicates a sion file descriptor.
Definition: sion_common.c:1161
int sion_lock_user_callbacks_defined(void)
Function which return flag, if user callback for locking are registered.
Definition: sion_common.c:1239
int sion_get_sizeof(int sid, int *numbytes, int *numfds)
Function returns size of internal data structure for sid.
Definition: sion_common.c:1123
int sion_lock_register_lock_callbacks(int lock(void *), int unlock(void *), void *lock_data)
Function which registers callback funtions for lock and unlock internal access to shared data structu...
Definition: sion_common.c:1220
int sion_dedup(int sid)
Function which destroy a duplicated sion file descriptor.
Definition: sion_common.c:1195