Common functions of the SIONlib API shared between serial and parallel modes. More...

Collaboration diagram for Common API:

Modules
	Key-Value API
	I/O of key-value pairs to SIONlib files.

	Datatypes
	Datatypes for use throughout the SIONlib API (deprecated)

	Constants
	Constants for use throughout the SIONlib API.

Data Structures
struct	_sion_io_stat_struct

Macros
#define	SION_SERIAL_MASTER

#define	SION_MAX_FSBLOCKSIZE 32 * 1024 * 1024

#define	SION_DEFAULT_FSBLOCKSIZE 2 * 1024 * 1024

#define	SION_FILENAME_LENGTH 1024

#define	MAXCHUNKS 100

#define	WFLARGEMETABLOCK_none /* for BENCHMARKING only */

#define	DPRINTF(A)

#define	DPRINTFP(A)

#define	DPRINTTS(A, B)

#define	DPRINTFTS2(A, B)

Typedefs
typedef struct _sion_io_stat_struct	sion_io_stat_t

Functions
size_t	sion_write (const void *data, size_t size, size_t nitems, int sid)
	Write data to a SIONlib file. More...

size_t	sion_fwrite (const void *data, size_t size, size_t nitems, int sid)
	Write data to a SIONlib file. More...

size_t	sion_read (void *data, size_t size, size_t nitems, int sid)
	Read data from SIONlib file. More...

size_t	sion_fread (void *data, size_t size, size_t nitems, int sid)
	Read data from a SIONlib file. More...

int	sion_eof (int sid)
	Function that indicates whether the end of this logical file has been reached. More...

int	sion_feof (int sid)
	Function that indicates whether the end of this logical file has been reached. More...

int	sion_ensure_free_space (int sid, int64_t bytes)
	Function to ensure that enough space is available for writing. More...

int	sion_flush (int sid)
	Flushed sion file. More...

int	sion_get_sizeof (int sid, int numbytes, int numfds)
	Function returns size of internal data structure for sid. More...

int	sion_dup (int sid, int mode, int rank, uint64_t key)
	Function which duplicates a sion file descriptor. More...

int	sion_dedup (int sid)
	Function which destroy a duplicated sion file descriptor. More...

int	sion_optimize_fp_buffer (int sid)

int	sion_get_file_endianness (int sid)
	Returns edianness of data in file sid. More...

int	sion_endianness_swap_needed (int sid)
	Returns whether or not byte swapping is needed for sid. More...

int64_t	sion_get_bytes_written (int sid)

int64_t	sion_get_bytes_read (int sid)

int64_t	sion_bytes_avail_in_block (int sid)
	Return the number of bytes available in the current chunk. More...

int64_t	sion_bytes_avail_in_chunk (int sid)
	Function that returns the number of bytes available in the current chunk. More...

int64_t	sion_get_position (int sid)
	Function that returns the current file position. More...

int	sion_get_mapping (int sid, int mapping_size, int32_t mapping, int numfiles)
	Returns pointers to the internal field mapping. More...

int	sion_get_locations (int sid, int ntasks, int maxchunks, int64_t globalskip, int64_t start_of_varheader, int64_t sion_chunksizes, int64_t sion_globalranks, int64_t sion_blockcount, int64_t sion_blocksizes)
	Returns pointers to internal fields. More...

int	sion_get_current_location (int sid, int currentchunknr, int64_t currentpos, int maxchunks, int64_t *chunksizes)
	Returns current position in file and pointer fiels containing chunk sizes. More...

int	sion_get_current_locations (int sid, int ntasks, int64_t sion_currentpos, int64_t *sion_currentblocknr)
	Returns current position in file and pointer fiels containing chunk sizes. More...

int	sion_get_number_of_files (int sid)

int	sion_get_filenumber (int sid)

int	sion_get_number_of_logical_files (int sid)

int64_t *	sion_get_chunksizes (int sid)

int32_t	sion_get_fsblksize (int sid)

int	sion_is_serial_opened (int sid)

int	sion_using_hints (int sid)

int	sion_get_endianness (void)
	Return endianness. More...

int	sion_is_thread_safe (void)

int	sion_get_version (int main_version, int sub_version, int patch_level, int fileformat_version)
	Return version numbers. More...

sion_io_stat_t *	sion_get_io_info (int sid)

sion_io_stat_t *	sion_get_io_info_by_spec (int sid, int roles, int flag)

int	sion_free_io_info (sion_io_stat_t *info)

int	sion_seek (int sid, int64_t offset, sion_seek_mode whence)
	moves the file pointer inside a logical file More...

int	sion_seek_chunk (int sid, int32_t offset, sion_seek_mode whence)
	moves the file pointer by increments of entire chunks inside the logical file More...

int	sion_switch_logical_file (int sid, int file_number)
	moves the file pointer to a different logical file More...

int64_t	sion_tell (int sid)
	Inspect position of file pointer in logical file. More...

int	sion_lock_register_lock_callbacks (int lock(void ), int unlock(void ), void *lock_data)
	Function which registers callback funtions for lock and unlock internal access to shared data structures. More...

int	sion_lock_user_callbacks_defined (void)
	Function which return flag, if user callback for locking are registered. More...

void	sion_swap (void target, void source, int size, int n, int do_swap)
	Perform byte-order swapping for arrays. More...

char *	sion_get_fname (int sid)

int64_t	sion_get_chunksize (int sid)

int32_t	sion_get_globalrank (int sid)

Detailed Description

Common functions of the SIONlib API shared between serial and parallel modes.

Macro Definition Documentation

◆ MAXCHUNKS

#define MAXCHUNKS 100

Initial maximum number of chunks in a sion file

Definition at line 34 of file sion_common.h.

Function Documentation

◆ sion_bytes_avail_in_block()

int64_t sion_bytes_avail_in_block ( int sid )

Return the number of bytes available in the current chunk.

This function returns the number of bytes currently available in the current chunk (bytes not read). This function is a local function, which can be called independently from other MPI tasks.

Parameters

[in] sid sion file handle

Returns: rc>0 number of bytes rc<=0 file position is after end of block

Definition at line 560 of file sion_common.c.

References sion_bytes_avail_in_chunk().

◆ sion_bytes_avail_in_chunk()

int64_t sion_bytes_avail_in_chunk ( int sid )

Function that returns the number of bytes available in the current chunk.

This function returns the number of bytes remaining in the current chunk (bytes not read). It is a local function, which can be called independently from other MPI tasks.

Parameters

[in] sid sion file handle

Returns: rc > 0 number of bytes rc <= 0 file position is after end of block

Definition at line 565 of file sion_common.c.

Referenced by sion_bytes_avail_in_block().

Here is the caller graph for this function:

◆ sion_dedup()

int sion_dedup ( int sid )

Function which destroy a duplicated sion file descriptor.

Parameters

[in] sid sion file handle

Returns: SION_SUCCESS okay

Definition at line 729 of file sion_common.c.

◆ sion_dup()

int sion_dup	(	int	sid,
		int	mode,
		int	rank,
		uint64_t	key
	)

Function which duplicates a sion file descriptor.

Restrictions:

only in read mode
only in parallel mode
collective operation are not callable on the new sid

Parameters

[in]	sid	sion file handle
[in]	mode	one of SION_DUP_ALL, SION_DUP_RANK, SION_DUP_RANK_KEY
[in]	rank	dup only those part of the internal data struture which are needed to read rank rank from file
[in]	key	dup only those part of the internal data struture which are needed to read on the rank rank data for key key from file

Returns: new sid if ok, SION_ID_NOT_VALID in other cases

Definition at line 704 of file sion_common.c.

◆ sion_endianness_swap_needed()

int sion_endianness_swap_needed ( int sid )

Returns whether or not byte swapping is needed for sid.

Parameters

[in] sid sion file handle

Returns: 1-> swap needed, 0-> no swap needed

Definition at line 147 of file sion_common.c.

◆ sion_ensure_free_space()

int sion_ensure_free_space	(	int	sid,
		int64_t	bytes
	)

Function to ensure that enough space is available for writing.

Prior to version 2.0.0 of SIONlib, this function was used to make sure that the file, especially the file pointer, was in a state where the next call to sion_write() could successfully write bytes many bytes to the file. This potentially involved flushing the current chunk and moving the file pointer to the beginning of the next chunk.

Deprecated:: This function is still provided for backwards compatibility. The function arguments are still checked for some constraints, returning SION_NOT_SUCCESS if they are not met, like before. However, SION_SUCCESS is returned, even if bytes is largen than the chunk size and the file pointer is never repositioned as if the file consists of a single infinitely sized chunk. Moving the file pointer to the beginning of the next block can be achieved with sion_seek_chunk().

Parameters

[in]	sid	SIONlib file handle
[in]	bytes	number of bytes requested for the next write operation

Returns: SION_SUCCESS if ok

Definition at line 634 of file sion_common.c.

◆ sion_eof()

int sion_eof ( int sid )

Function that indicates whether the end of this logical file has been reached.

This function indicates whether the end of this logical file has been reached or not. This means the file pointer points behind the last byte of the last chunk of this task. The function is a task local function, which can be called independently from other tasks and does not invoke any communication.

Parameters

[in] sid SIONlib file handle

Returns: SION_SUCCESS (1) if end of last chunk reached SION_NOT_SUCCESS (0) otherwise

Definition at line 534 of file sion_common.c.

Referenced by sion_feof().

Here is the caller graph for this function:

◆ sion_feof()

int sion_feof ( int sid )

Function that indicates whether the end of this logical file has been reached.

Deprecated:: Use sion_eof() instead.

Definition at line 555 of file sion_common.c.

References sion_eof().

◆ sion_flush()

int sion_flush ( int sid )

Flushed sion file.

Parameters

[in] sid sion file handle

Returns: SION_SUCCESS if ok

Definition at line 624 of file sion_common.c.

◆ sion_fread()

size_t sion_fread	(	void *	data,
		size_t	size,
		size_t	nitems,
		int	sid
	)

Read data from a SIONlib file.

Deprecated:: Use sion_read() instead.

Definition at line 384 of file sion_common.c.

References sion_read().

◆ sion_fwrite()

size_t sion_fwrite	(	const void *	data,
		size_t	size,
		size_t	nitems,
		int	sid
	)

Write data to a SIONlib file.

Deprecated:: Use sion_write() instead.

Definition at line 348 of file sion_common.c.

References sion_write().

◆ sion_get_current_location()

int sion_get_current_location	(	int	sid,
		int *	currentchunknr,
		int64_t *	currentpos,
		int *	maxchunks,
		int64_t **	chunksizes
	)

Returns current position in file and pointer fiels containing chunk sizes.

Returns current position in file and pointer fiels containing chunk sizes. This function is only needed if the sion file was opened for reading in parallel mode (sion_paropen..., sion_open_rank).

Deprecated:: This function exposes internal fields that are considered an implementation detail. It will probably be removed prior to the final release of SIONlib 2.0.0.

Parameters

[in]	sid	sion file handle
[out]	currentchunknr	number of current block in the file
[out]	currentpos	position in current block in the file
[out]	maxchunks	number of last block in the file
[out]	chunksizes	field containing chunk size of each task

Returns: SION_SUCCESS if ok

Definition at line 90 of file sion_common.c.

◆ sion_get_current_locations()

int sion_get_current_locations	(	int	sid,
		int *	ntasks,
		int64_t **	sion_currentpos,
		int64_t **	sion_currentblocknr
	)

Returns current position in file and pointer fiels containing chunk sizes.

Returns current position in file and pointer fiels containing chunk sizes. This function is only needed if the sion file was opened for reading in parallel mode (sion_paropen..., sion_open_rank).

Deprecated:: This function exposes internal fields that are considered an implementation detail. It will probably be removed prior to the final release of SIONlib 2.0.0.

Parameters

[in]	sid	sion file handle
[out]	ntasks	number of tasks
[out]	sion_currentpos	current positions for all ranks
[out]	sion_currentblocknr	current block number for all ranks

Returns: SION_SUCCESS if ok

Definition at line 160 of file sion_common.c.

◆ sion_get_endianness()

int sion_get_endianness ( void )

Return endianness.

Returns: 1-> big endian 0 ->little endian

Definition at line 812 of file sion_common.c.

◆ sion_get_file_endianness()

int sion_get_file_endianness ( int sid )

Returns edianness of data in file sid.

Parameters

[in] sid sion file handle

Returns: 1-> big endian, 0-> little endian

Definition at line 132 of file sion_common.c.

◆ sion_get_locations()

int sion_get_locations	(	int	sid,
		int *	ntasks,
		int *	maxchunks,
		int64_t *	globalskip,
		int64_t *	start_of_varheader,
		int64_t **	sion_chunksizes,
		int64_t **	sion_globalranks,
		int64_t **	sion_blockcount,
		int64_t **	sion_blocksizes
	)

Returns pointers to internal fields.

Returns pointers to internal fields, containing the number of chunks written by each task (sion_chunkcount) and their sizes (sion_chunksizes). This function is only needed if the sion file was opened for reading in serial mode.

Deprecated:: This function exposes internal fields that are considered an implementation detail. It will probably be removed prior to the final release of SIONlib 2.0.0.

Parameters

[in]	sid	sion file handle
[out]	ntasks	number of tasks which wrote to the sion file
[out]	maxchunks	maximum number of blocks in the file
[out]	globalskip	distance in bytes between the first bytes of two subsequent chunks of a task
[out]	start_of_varheader	start position of the meta data block at the end of sion file
[out]	sion_chunksizes	field containing requested chunk size of each task access: sion_chunksizes[size*chunknr+rank]
[out]	sion_globalranks	field containing global unique id of each task
[out]	sion_blockcount	field containing number of blocks used by each task
[out]	sion_blocksizes	field containing for each task and block the number of bytes used in this chunk e.g. access: sion_blocksizes[size*blocknr+rank]

Returns: SION_SUCCESS if ok

Definition at line 35 of file sion_common.c.

◆ sion_get_mapping()

int sion_get_mapping	(	int	sid,
		int *	mapping_size,
		int32_t **	mapping,
		int *	numfiles
	)

Returns pointers to the internal field mapping.

This function is only needed if the sion file was opened for reading in serial mode.

Deprecated:: This function exposes internal fields that are considered an implementation detail. It will probably be removed prior to the final release of SIONlib 2.0.0.

Parameters

[in]	sid	sion file handle
[out]	mapping_size	size of mapping vector
[out]	mapping	pointer to mapping vector
[out]	numfiles	pointer to mapping vector

Returns: SION_SUCCESS if ok

Definition at line 109 of file sion_common.c.

◆ sion_get_position()

int64_t sion_get_position ( int sid )

Function that returns the current file position.

This function returns an int64_t containing the current file possition.

Deprecated:: This function is badly named and documented. It will probably be renamed sion_tell_physical() before the final release of SIONlib 2.0.0.

Parameters

[in] sid sion file handle (in)

Returns: current file position

Definition at line 590 of file sion_common.c.

◆ sion_get_sizeof()

int sion_get_sizeof	(	int	sid,
		int *	numbytes,
		int *	numfds
	)

Function returns size of internal data structure for sid.

Parameters

[in]	sid	sion file handle
[out]	numbytes	number of bytes used
[out]	numfds	number of file descriptors used

Returns: SION_SUCCESS if ok

Definition at line 682 of file sion_common.c.

◆ sion_get_version()

int sion_get_version	(	int *	main_version,
		int *	sub_version,
		int *	patch_level,
		int *	fileformat_version
	)

Return version numbers.

return version numbers

Definition at line 824 of file sion_common.c.

References SION_VERSION_MAJOR, SION_VERSION_MINOR, and SION_VERSION_PATCH.

◆ sion_lock_register_lock_callbacks()

int sion_lock_register_lock_callbacks	(	int	lockvoid *,
		int	unlockvoid *,
		void *	lock_data
	)

Function which registers callback funtions for lock and unlock internal access to shared data structures.

Parameters

[in]	lock()	lock function
[in]	unlock()	unlock function
[in]	lock_data()	unlock function

Returns: SION_SUCCESS okay

Definition at line 743 of file sion_common.c.

Referenced by sion_paropen_omp(), and sion_paropen_ompi().

Here is the caller graph for this function:

◆ sion_lock_user_callbacks_defined()

int sion_lock_user_callbacks_defined ( void )

Function which return flag, if user callback for locking are registered.

Returns: SION_SUCCESS (yes) of SION_NOT_SUCCESS (no)

Definition at line 757 of file sion_common.c.

◆ sion_read()

size_t sion_read	(	void *	data,
		size_t	size,
		size_t	nitems,
		int	sid
	)

Read data from SIONlib file.

This function attempts toread nitems items, each of size size, from the file identified by sid starting at the current position of the file pointer and stores them in the memory buffer starting at data. The file pointer is advanced by the number of bytes read from the file.

sion_read() returns the number of items that were read from the file. A number less than nitems indicates that either the end of the logical file has been reached, which can be confirmed via sion_eof(), or that an error occurred.

Parameters

[out]	data	pointer to data buffer
[in]	size	size of a single item
[in]	nitems	number of items to be written
[in]	sid	sion file id to be written to

Returns: number of elements read

Definition at line 353 of file sion_common.c.

Referenced by sion_fread().

Here is the caller graph for this function:

◆ sion_seek()

int sion_seek	(	int	sid,
		int64_t	offset,
		sion_seek_mode	whence
	)

moves the file pointer inside a logical file

This moves the file pointer inside a logical file. The file pointer is moved by a relative offset that is interpreted according to the whence argument as follows:

SION_SEEK_BEGINNING offset is relative to the beginning of the file
SION_SEEK_CURRENT: offset is relative to the current position of the file pointer
SION_SEEK_END: offset is relative to the end of the file
SION_SEEK_CHUNK_BEGINNING offset is relative to the beginning of the current chunk
SION_SEEK_CHUNK_END: offset is relative to the end of the current chunk

The file pointer must not be moved before the beginning of the file. In read mode, the file pointer must also not be moved beyond the end of the file. When seeking relative to the bounds of the current chunk (SION_SEEK_CHUNK_BEGINNING and SION_SEEK_CHUNK_END), seeking outside of the current chunk is not allowed.

See also: sion_seek_chunk sion_switch_logical_file

Parameters

[in,out]	sid	SIONlib file handle
[in]	offset	offset in positions by which to move the file pointer
[in]	whence	determines how `offset` should be interpreted

Returns: SION_SUCCESS if file pointer can be moved to new position

Definition at line 389 of file sion_common.c.

◆ sion_seek_chunk()

int sion_seek_chunk	(	int	sid,
		int32_t	offset,
		sion_seek_mode	whence
	)

moves the file pointer by increments of entire chunks inside the logical file

Moves the file pointer to the beginning of a chunk that is identified by a relative offset, according to whence:

SION_SEEK_BEGINNING: seeks from the beginning of the file to the chunk with number offset
SION_SEEK_CURRENT: offset is a number of chunks relative to the current chunk
SION_SEEK_END: offset is relative to the end of the file The file pointer must not be moved before the beginning of the file. In read mode, the file pointer must also not be moved beyond the end of the file.

See also: sion_seek sion_switch_logical_file

Parameters

[in,out]	sid	SIONlib file handle
[in]	offset	offset in chunks by which to move the file pointer
[in]	whence	determines how `offset` should be interpreted

Returns: SION_SUCCESS if file pointer can be moved to new position

Definition at line 432 of file sion_common.c.

◆ sion_swap()

void sion_swap	(	void *	target,
		void *	source,
		int	size,
		int	n,
		int	do_swap
	)

Perform byte-order swapping for arrays.

n elements of size bytes each are swapped if do_swap is true. In-place swapping (target == source) is allowed. If target != source, the buffers must not overlap.

Parameters

[out]	target	the byte-swapped data is written starting at this address
[in]	source	the data to be byte-swapped is read starting at this address
[in]	size	the size (in bytes) of a single element
[in]	n	the number of elements to be byte-swapped
[in]	do_swap	byte-swapping is only performed if this argument is `true`

Definition at line 17 of file sion_convert.c.

◆ sion_switch_logical_file()

int sion_switch_logical_file	(	int	sid,
		int	file_number
	)

moves the file pointer to a different logical file

Switches from one logical file in a SIONlib container to another. When opening a SIONlib cointainer, the individual per logical file file pointers start out at the first position of each logical file. However when switching from one logical file to another, the position of the file pointer is persisted and reloaded when switching back.

See also: sion_seek sion_seek_chunk

Parameters

[in,out]	sid	SIONlib file handle
[in]	file_number	the number of the logical file to switch to

Returns: SION_SUCCESS if file pointer can be moved to new position

Definition at line 479 of file sion_common.c.

◆ sion_tell()

int64_t sion_tell ( int sid )

Inspect position of file pointer in logical file.

Parameters

[in] sid SIONlib file descriptor

Returns: the position of the file pointer in the logical file

Definition at line 524 of file sion_common.c.

◆ sion_write()

size_t sion_write	(	const void *	data,
		size_t	size,
		size_t	nitems,
		int	sid
	)

Write data to a SIONlib file.

This function takes nitems items, each of size size, from the memory buffer at data and writes them to the file identified by sid starting at the current position of the file pointer. The file pointer is advanced by the number of bytes written to the file.

sion_write() returns the number of items that were written to the file with numbers less than nitems indicating an error.

Parameters

[in]	data	pointer to data to be written
[in]	size	size of a single item
[in]	nitems	number of items to be written
[in]	sid	sion file id to be written to

Returns: number of elements written

Definition at line 292 of file sion_common.c.

Referenced by sion_fwrite().

Here is the caller graph for this function:

Modules

Data Structures

Macros

Typedefs

Functions

Detailed Description

Macro Definition Documentation

◆ MAXCHUNKS

Function Documentation

◆ sion_bytes_avail_in_block()

◆ sion_bytes_avail_in_chunk()

◆ sion_dedup()

◆ sion_dup()

◆ sion_endianness_swap_needed()

◆ sion_ensure_free_space()

◆ sion_eof()

◆ sion_feof()

◆ sion_flush()

◆ sion_fread()

◆ sion_fwrite()

◆ sion_get_current_location()

◆ sion_get_current_locations()

◆ sion_get_endianness()

◆ sion_get_file_endianness()

◆ sion_get_locations()

◆ sion_get_mapping()

◆ sion_get_position()

◆ sion_get_sizeof()

◆ sion_get_version()

◆ sion_lock_register_lock_callbacks()

◆ sion_lock_user_callbacks_defined()

◆ sion_read()

◆ sion_seek()

◆ sion_seek_chunk()

◆ sion_swap()

◆ sion_switch_logical_file()

◆ sion_tell()

◆ sion_write()