sc_scda.h File Reference

Routines for parallel I/O with the scda format. More...

#include <sc_containers.h>
#include <sc_random.h>

Include dependency graph for sc_scda.h:

Go to the source code of this file.

Data Structures
struct	sc_scda_ferror
	Error values for the scda functions. More...
struct	sc_scda_params
	A parameter struct for the functions sc_scda_fopen_write and sc_scda_fopen_read. More...

Macros
#define	SC_SCDA_HEADER_BYTES   128
	number of file header bytes
#define	SC_SCDA_USER_STRING_BYTES   58
	number of user string bytes
#define	SC_SCDA_INLINE_FIELD   32
	byte count of inline data

Typedefs
typedef struct sc_scda_fcontext	sc_scda_fcontext_t
	Opaque context for writing and reading a libsc data file, i.e. More...
typedef uint64_t	sc_scda_ulong
	Type for element counts and sizes.
typedef enum sc_scda_ret	sc_scda_ret_t
	Error values for scda-related errors. More...
typedef struct sc_scda_ferror	sc_scda_ferror_t
	Error values for the scda functions. More...
typedef struct sc_scda_params	sc_scda_params_t
	A parameter struct for the functions sc_scda_fopen_write and sc_scda_fopen_read. More...

Enumerations
enum	sc_scda_ret {   SC_SCDA_FERR_SUCCESS = 0 ,   SC_SCDA_FERR_FORMAT = 15000 ,   SC_SCDA_FERR_USAGE ,   SC_SCDA_FERR_DECODE ,   SC_SCDA_FERR_ARG ,   SC_SCDA_FERR_COUNT ,   SC_SCDA_FERR_MPI ,   SC_SCDA_FERR_LASTCODE }
	Error values for scda-related errors. More...

Functions
void	sc_scda_params_init (sc_scda_params_t *params)
	Initialize a scda parameter structure to the defaults. More...
sc_scda_fcontext_t *	sc_scda_fopen_write (sc_MPI_Comm mpicomm, const char *filename, const char *user_string, size_t *len, sc_scda_params_t *params, sc_scda_ferror_t *errcode)
	Open a file for writing and write the file header to the file. More...
sc_scda_fcontext_t *	sc_scda_fwrite_inline (sc_scda_fcontext_t *fc, const char *user_string, size_t *len, sc_array_t *inline_data, int root, sc_scda_ferror_t *errcode)
	Write an inline data section. More...
sc_scda_fcontext_t *	sc_scda_fwrite_block (sc_scda_fcontext_t *fc, const char *user_string, size_t *len, sc_array_t *block_data, size_t block_size, int root, int encode, sc_scda_ferror_t *errcode)
	Write a fixed-size block file section. More...
sc_scda_fcontext_t *	sc_scda_fwrite_array (sc_scda_fcontext_t *fc, const char *user_string, size_t *len, sc_array_t *array_data, sc_array_t *elem_counts, size_t elem_size, int indirect, int encode, sc_scda_ferror_t *errcode)
	Write a fixed-size array file section. More...
int	sc_scda_proc_sizes (sc_array_t *elem_sizes, sc_array_t *elem_counts, sc_array_t *proc_sizes, sc_scda_ferror_t *errcode)
	This is a collective function to determine the processor sizes. More...
sc_scda_fcontext_t *	sc_scda_fwrite_varray (sc_scda_fcontext_t *fc, const char *user_string, size_t *len, sc_array_t *array_data, sc_array_t *elem_counts, sc_array_t *elem_sizes, sc_array_t *proc_sizes, int indirect, int encode, sc_scda_ferror_t *errcode)
	Write a variable-size array file section. More...
sc_scda_fcontext_t *	sc_scda_fopen_read (sc_MPI_Comm mpicomm, const char *filename, char *user_string, size_t *len, sc_scda_params_t *params, sc_scda_ferror_t *errcode)
	Open a file for reading and read the file header from the file. More...
sc_scda_fcontext_t *	sc_scda_fread_section_header (sc_scda_fcontext_t *fc, char *user_string, size_t *len, char *type, size_t *elem_count, size_t *elem_size, int *decode, sc_scda_ferror_t *errcode)
	Read the next file section header. More...
sc_scda_fcontext_t *	sc_scda_fread_inline_data (sc_scda_fcontext_t *fc, sc_array_t *data, int root, sc_scda_ferror_t *errcode)
	Read the data of an inline data section. More...
sc_scda_fcontext_t *	sc_scda_fread_block_data (sc_scda_fcontext_t *fc, sc_array_t *block_data, size_t block_size, int root, sc_scda_ferror_t *errcode)
	Read the data of a block of given size. More...
sc_scda_fcontext_t *	sc_scda_fread_array_data (sc_scda_fcontext_t *fc, sc_array_t *array_data, sc_array_t *elem_counts, size_t elem_size, int indirect, sc_scda_ferror_t *errcode)
	Read the data of a fixed-size array. More...
sc_scda_fcontext_t *	sc_scda_fread_varray_sizes (sc_scda_fcontext_t *fc, sc_array_t *elem_sizes, sc_array_t *elem_counts, sc_scda_ferror_t *errcode)
	Read the element sizes of a variable-size array. More...
sc_scda_fcontext_t *	sc_scda_fread_varray_data (sc_scda_fcontext_t *fc, sc_array_t *array_data, sc_array_t *elem_counts, sc_array_t *elem_sizes, sc_array_t *proc_sizes, int indirect, sc_scda_ferror_t *errcode)
	Read the data of a variable-size array. More...
int	sc_scda_ferror_class (sc_scda_ferror_t errcode, sc_scda_ferror_t *errclass)
	Translate a sc_scda error code to an error class. More...
int	sc_scda_ferror_is_success (sc_scda_ferror_t errorcode)
	Check if a scda_errorcode_t encodes success. More...
int	sc_scda_ferror_string (sc_scda_ferror_t errcode, char *str, int *len)
	Translate a sc_scda error code/class to an error string. More...
int	sc_scda_fclose (sc_scda_fcontext_t *fc, sc_scda_ferror_t *errcode)
	Close a file opened for parallel write/read and the free the file context. More...

Detailed Description

Routines for parallel I/O with the scda format.

Warning

This API will change in the next libsc version!

Functionality to write and read in parallel using a prescribed serial-equivalent file format called scda.

General

The scda format is as in this preprint.

However, in contrast to the preprint the API in this file provides the two functions sc_scda_fopen_write and sc_scda_fopen_read instead of providing one opening function with a mode parameter to decide and writing and reading.

In addition, we add in this file the options structure sc_scda_params as parameter for opening files.

The file format includes metadata in ASCII and therefore enables the human eye to parse the file structure using a standard text editor.

The file format scda is in particular suitable for parallel I/O and is accompanied by a convention for element-wise compression.

The format is designed such that the the parallel partition and in particular the process count can differ between writing and reading.

The main purpose of scda is to enable the user to implement parallel I/O for numerical appliations, e.g. simulation checkpoint/restart.

We elaborate further on the workflow in Parallel I/O workflow.

The scda functions do not support the deprecated configuration case of MPI being enabled but MPI I/O being disabled. The functions sc_scda_fopen_write and sc_scda_fopen_read abort in this case. All other valid configurations are supported.

User Strings

The functions

• sc_scda_fopen_write,
• sc_scda_fwrite_inline,
• sc_scda_fwrite_block,
• sc_scda_fwrite_array,
• sc_scda_fwrite_varray,
• sc_scda_fopen_read and
• sc_scda_fread_section_header

have user_string and len as an argument.

The user string consisting of the two parameters user_string and len is always a collective parameter.

In the case of writing these arguments have the purpose to pass the user string that is written to the file.

There are two options writing the user string to the file.

1. A nul-terminated string user_string, i.e. a standard C string. In this case len is set to NULL since the length of user_string is implicitly given by the nul-termination.
2. Arbitrary data for the user string including possible '\0' in non-terminating positions. We still require nul-termination for safety. One needs to explicitly pass the length, i.e. the number of bytes excluding the nul-termination.

For both options it must be respected that the number of maximal user_string bytes is SC_SCDA_USER_STRING_BYTES + 1 including the nul-termination.

In case of reading user_string must be at least SC_SCDA_USER_STRING_BYTES + 1 bytes. On output len is set to the number of bytes actually written to user_string excluding the nul-termination.

If it is desired to write arbitrary data without the nul-termination, which is required for the user string, one can write a block data section using the function sc_scda_fwrite_block.

Encoding

scda provides optional transparent, element-wise compression of data of file sections. The compression for writing can be enabled by passing true for encode to one of the functions

• sc_scda_fwrite_inline,
• sc_scda_fwrite_block,
• sc_scda_fwrite_array and
• sc_scda_fwrite_varray.

The compression on writing can be decided by the user for each file section separately. All parameters on the size of the data written to the file refer to the uncompressed data.

On reading it sufficies to pass decode true to the function sc_scda_fread_section_header. Then the file section data is decompressed if it was compressed and otherwise the file section data is read raw. Again all data sizes in reading functions refer to the uncompressed data.

If decode is false, the data is read raw even if it was written according to the compression convention.

Error management

All scda functions that receive a file context have an output parameter called errcode. In case of an unsuccessful function call, the respective function returns NULL instead of the file context and sets the output parameter errcode to the respective error code. If such a case occurs, the file that is associated to the used file context is closed and the file context is deallocated.

The error code can be examined by the user using the two functions

• sc_scda_ferror_class and
• sc_scda_ferror_string.

If MPI is available errcode may encode an MPI error code. In this case the two error examination functions output the error class and error string as it would be output by the corresponding MPI functions, respectively. Without MPI it is still possible that errcode encodes an I/O operation related error code. This case does not differ concerning the error code examiniation. Moreover, errcode can encode an error code related to scda, i.e. the I/O operations were successful but there is some violation of the scda format, workflow or API.

For more technical details on errcode see the documentation of sc_scda_ferror_t. Furthermore, the scda format, workflow or API errors description can be found in the documentation of sc_scda_ret.

Typedef Documentation

sc_scda_fcontext_t

typedef struct sc_scda_fcontext sc_scda_fcontext_t

Opaque context for writing and reading a libsc data file, i.e.

a scda file.

sc_scda_ferror_t

typedef struct sc_scda_ferror sc_scda_ferror_t

Error values for the scda functions.

An error value is a struct since the error can be related to the scda file format or to (MPI) I/O operations. The error code can be converted to a string by sc_scda_ferror_string, mapped to an error class by sc_scda_ferror_class and checked for success by sc_scda_ferror_is_success. The user can rely on these functions to parse sc_scda_ferror and does not need to access the members of this structure.

The parsing logic of sc_scda_ferror_t is that first scdaret is examined and if scdaret != SC_SCDA_FERR_MPI, we know that mpiret = 0. If scdaret = SC_SCDA_FERR_MPI, we know that an MPI error occurred and we can examine mpiret for more informartion.

Moreover, a valid sc_scda_ferror always satisfy that if scdaret = 0 then mpiret = 0 and if scdaret = SC_SCDA_FERR_MPI then mpiret !=0.

sc_scda_params_t

typedef struct sc_scda_params sc_scda_params_t

A parameter struct for the functions sc_scda_fopen_write and sc_scda_fopen_read.

The struct may be extended in the future.

The option struct is a collective structure. If the options structure that is passed to a function is not the same on all processes, the whole following scda workflow has undefined behavior. type for sc_scda_params

sc_scda_ret_t

typedef enum sc_scda_ret sc_scda_ret_t

Error values for scda-related errors.

The error codes are part of the struct sc_scda_ferror_t and can be examined as part of this struct.

Enumeration Type Documentation

sc_scda_ret

enum sc_scda_ret

Error values for scda-related errors.

The error codes are part of the struct sc_scda_ferror_t and can be examined as part of this struct.

Enumerator
SC_SCDA_FERR_SUCCESS	successful function call
SC_SCDA_FERR_FORMAT	File not conforming to the scda format.
SC_SCDA_FERR_USAGE	Incorrect workflow of an scda reading function. For example, the user might have identified a certain file section type using sc_scda_fread_section_header but then calls a function to read the section data for a different type. Another example is to try reading the data of a 'V' section before reading its element sizes. This error also occurs when the user tries to read section data before reading the section header.
SC_SCDA_FERR_DECODE	The decode parameter to sc_scda_fread_section_header is true but the file section header(s) encountered does not conform to the scda encoding convention.
SC_SCDA_FERR_ARG	An argument to a scda file function is invalid. This occurs for example when an essential pointer argument is NULL or a user string for writing is too long.
SC_SCDA_FERR_COUNT	A byte count error that may occur transiently on writing or the file is short on reading.
SC_SCDA_FERR_MPI	An MPI error occurred; see mpiret in the corresponding sc_scda_ferror_t.
SC_SCDA_FERR_LASTCODE	to define own error codes for a higher level application that is using sc_scda functions

Function Documentation

sc_scda_fclose()

int sc_scda_fclose	(	sc_scda_fcontext_t *	fc,
		sc_scda_ferror_t *	errcode
	)

Close a file opened for parallel write/read and the free the file context.

This is a collective function. Every call of sc_scda_fopen_write and sc_scda_fopen_read must be matched by a corresponding call of sc_scda_fclose on the created file context.

Note

All parameters are collective.

This function returns -1 on I/O errors. This function always frees the file context – also in case of an error.

Parameters

[in,out]	fc	File context previously created by sc_scda_fopen_write or sc_scda_fopen_read. This file context is freed after a call of this function.
[out]	errcode	An errcode that can be interpreted by sc_scda_ferror_string or mapped to an error class by sc_scda_ferror_class. errcode encodes success if and only if the function returns return 0.

Returns

SC_SCDA_FERR_SUCCESS for a successful call and -1 in case a of an error. See also errcode argument.

sc_scda_ferror_class()

int sc_scda_ferror_class	(	sc_scda_ferror_t	errcode,
		sc_scda_ferror_t *	errclass
	)

Translate a sc_scda error code to an error class.

The semantic of error class and error code is the same as in the MPI standard, i.e. all error classes are error codes but potentially there are more error codes than error classes.

If errcode is already an error class, errclass if filled with errcode.

This is a non-collective function.

Parameters

[in]	errcode	An errcode that is output by a sc_scda function.
[out]	errclass	On output filled with the error class that corresponds to the given errcode. See the function description above for more information on error classes and error codes in scda.

Returns

SC_SCDA_FERR_SUCCESS on success or something else on invalid arguments.

sc_scda_ferror_is_success()

int sc_scda_ferror_is_success

(

sc_scda_ferror_t

errorcode

)

Check if a scda_errorcode_t encodes success.

Parameters

[in]

errorcode

An errcode that is output by a sc_scda function.

Returns

True if errcode encodes success and false otherwise.

sc_scda_ferror_string()

int sc_scda_ferror_string	(	sc_scda_ferror_t	errcode,
		char *	str,
		int *	len
	)

Translate a sc_scda error code/class to an error string.

This is a non-collective function.

Parameters

[in]	errcode	An errcode that is output by a sc_scda function.
[out]	str	At least sc_MPI_MAX_ERROR_STRING bytes.
[out]	len	On output the length of string on return.

Returns

SC_SCDA_FERR_SUCCESS on success or something else on invalid arguments.

sc_scda_fopen_read()

sc_scda_fcontext_t* sc_scda_fopen_read	(	sc_MPI_Comm	mpicomm,
		const char *	filename,
		char *	user_string,
		size_t *	len,
		sc_scda_params_t *	params,
		sc_scda_ferror_t *	errcode
	)

Open a file for reading and read the file header from the file.

The file must exist and be at least of the size of the file header, i.e. SC_SCDA_HEADER_BYTES bytes. If the file has a file header that does not satisfy the sc_scda file header format, the function reports the error using SC_LERRORF, collectively close the file and deallocate the file context. In this case the function returns NULL on all ranks. A wrong file header format causes SC_SCDA_FERR_FORMAT as errcode.

Note

All parameters are collective.

This function aborts on the deprecated configuration of enabled MPI and disabled MPI I/O. All other valid configurations are supported.

This function differs from the one opening function for writing and reading introduced in this scda preprint.

This function returns NULL on I/O errors.

Parameters

[in]	mpicomm	The MPI communicator that is used to open the parallel file.
[in]	filename	Path to parallel file that is to be created or to be opened.
[out]	user_string	At least SC_SCDA_USER_STRING_BYTES + 1 bytes. user_string is filled with the read user string from file and is nul-terminated.
[out]	len	On output len is set to the number of bytes written to user_string excluding the terminating nul.
[in]	params	A parameter structure that provides the possibility to pass further options. See sc_scda_params for more details. It is valid to pass NULL for params.
[out]	errcode	An errcode that can be interpreted by sc_scda_ferror_string or mapped to an error class by sc_scda_ferror_class. errcode encodes success if and only if the function does not return NULL.

Returns

Newly allocated context to continue reading and eventually closing the file. NULL in case of error, i.e. errcode != SC_SCDA_FERR_SUCCESS.

sc_scda_fopen_write()

sc_scda_fcontext_t* sc_scda_fopen_write	(	sc_MPI_Comm	mpicomm,
		const char *	filename,
		const char *	user_string,
		size_t *	len,
		sc_scda_params_t *	params,
		sc_scda_ferror_t *	errcode
	)

Open a file for writing and write the file header to the file.

This function creates a new file or overwrites an existing one. It is collective and creates the file on a parallel file system.

Note

All parameters are collective. This function leaves the file open if MPI is available. Independent of the availability of MPI the user can write one or more file sections before closing the file (context) using sc_scda_fclose.

It is the user's responsibility to write any further metadata of the file that is required by the application. This can be done by writing file sections. However, the user can use sc_scda_fopen_read to open a not already opened file and then use sc_scda_fread_section_header and skipping the respective data bytes using the respective read functions sc_scda_fread_*_data to parse the structure of a given file and some metadata that is written by sc_scda.

This function aborts on the deprecated configuration of enabled MPI and disabled MPI I/O. All other valid configurations are supported.

This function differs from the one opening function for writing and reading introduced in this scda preprint.

This function returns NULL on I/O errors.

Parameters

[in]	mpicomm	The MPI communicator that is used to open the parallel file.
[in]	filename	Path to parallel file that is to be created or to be opened.
[in]	user_string	At most SC_SCDA_USER_STRING_BYTES + 1 bytes in a nul-terminated string. See the 'User Strings' section in the detailed description of this file for more information.
[in]	len	The number of bytes in user_string excluding the terminating nul. On NULL as input user_string is expected to be a nul-terminated C string.
[in]	params	A parameter structure that provides the possibility to pass further options. See sc_scda_params for more details. It is valid to pass NULL for params.
[out]	errcode	An errcode that can be interpreted by sc_scda_ferror_string or mapped to an error class by sc_scda_ferror_class. errcode encodes success if and only if the function does not return NULL.

Returns

Newly allocated context to continue writing and eventually closing the file. NULL in case of error, i.e. errcode != SC_SCDA_FERR_SUCCESS.

sc_scda_fread_array_data()

sc_scda_fcontext_t* sc_scda_fread_array_data	(	sc_scda_fcontext_t *	fc,
		sc_array_t *	array_data,
		sc_array_t *	elem_counts,
		size_t	elem_size,
		int	indirect,
		sc_scda_ferror_t *	errcode
	)

Read the data of a fixed-size array.

This is a collective function. This function is only valid to call directly after a successful call of sc_scda_fread_section_header. This preceding call gives also the required elem_size and the global number of array elements. The user must pass a parallel partition of the array elements by elem_counts.

Note

All parameters except of array_data are collective.

Warning

The API of this function will change in the next libsc version!

This function returns NULL on I/O errors.

Parameters

[in,out]	fc	File context previously opened by sc_scda_fopen_read.
[out]	array_data	If indirect is false, a sc_array with element count equals to the p-th entry of elem_counts for p being the calling rank. The element size must be equal to elem_size. If indirect is true, a sc_array with the same element count as for indirect false but with sizeof (sc_array_t) as element size. Each array element is then again a sc_array but with element count 1 and element size elem_size. The data can be skipped on each process by locally passing NULL.
[in]	elem_counts	An sc_array that must be equal on all ranks. The element count of elem_counts must be the mpisize of the MPI communicator that was used to create fc. The element size of the sc_array must be equal to sizeof (sc_scda_ulong). The sc_array must contain the local array elements counts. That is why it induces the partition that is used to read the array data in parallel. The sum of all array elements must be equal to elem_count as retrieved from sc_scda_fread_section_header.
[in]	elem_size	The element size of one array element on number of bytes. Must be the same on all ranks and as retrieved from sc_scda_fread_section_header.
[in]	indirect	A Boolean to determine whether array_data must be a sc_array of sc_arrays to read indirectly and in particular to potentially non-contigous memory. See the documentation of the parameter array_data for more information.
[out]	errcode	An errcode that can be interpreted by sc_scda_ferror_string or mapped to an error class by sc_scda_ferror_class. errcode encodes success if and only if the function does not return NULL.

Returns

Return a pointer to the input context fc on success. The context is used to continue reading and eventually closing the file. In case of any error, attempt to close the file and deallocate the context fc.

sc_scda_fread_block_data()

sc_scda_fcontext_t* sc_scda_fread_block_data	(	sc_scda_fcontext_t *	fc,
		sc_array_t *	block_data,
		size_t	block_size,
		int	root,
		sc_scda_ferror_t *	errcode
	)

Read the data of a block of given size.

This is a collective function. This function is only valid to call directly after a successful call of sc_scda_fread_section_header. This preceding call gives also the required block_size.

Note

All parameters except of block_data are collective.

Warning

The API of this function will change in the next libsc version!

This function returns NULL on I/O errors.

Parameters

[in,out]	fc	File context previously opened by sc_scda_fopen_read.
[out]	block_data	A sc_array with element count 1 and element size block_size. On output the sc_array is filled with the block data of the read block data section.
[in]	block_size	The number of bytes of the block as retrieved from the preceding call of sc_scda_fread_section_header.
[in]	root	An integer between 0 and mpisize exclusive of the MPI communicator that was used to create fc. root indicates the MPI rank on that block_data is read from the file.
[out]	errcode	An errcode that can be interpreted by sc_scda_ferror_string or mapped to an error class by sc_scda_ferror_class. errcode encodes success if and only if the function does not return NULL.

Returns

Return a pointer to the input context fc on success. The context is used to continue reading and eventually closing the file. In case of any error, attempt to close the file and deallocate the context fc.

sc_scda_fread_inline_data()

sc_scda_fcontext_t* sc_scda_fread_inline_data	(	sc_scda_fcontext_t *	fc,
		sc_array_t *	data,
		int	root,
		sc_scda_ferror_t *	errcode
	)

Read the data of an inline data section.

This is a collective function. This function is only valid to call directly after a successful call of sc_scda_fread_section_header.

Note

All parameters except of data are collective.

This function returns NULL on I/O errors.

Parameters

[in,out]	fc	File context previously opened by sc_scda_fopen_read.
[out]	data	Exactly 32 bytes on the rank root or NULL on root to not read the bytes. In the first case the sc_array must have an element count of 1 and an element size 32. The parameter is ignored on all ranks unequal to root.
[in]	root	An integer between 0 and mpisize exclusive of the MPI communicator that was used to create fc. root indicates the MPI rank on that data is read from the file.
[out]	errcode	An errcode that can be interpreted by sc_scda_ferror_string or mapped to an error class by sc_scda_ferror_class. errcode encodes success if and only if the function does not return NULL.

Returns

Return a pointer to the input context fc on success. The context is used to continue reading and eventually closing the file. In case of any error, attempt to close the file and deallocate the context fc.

sc_scda_fread_section_header()

sc_scda_fcontext_t* sc_scda_fread_section_header	(	sc_scda_fcontext_t *	fc,
		char *	user_string,
		size_t *	len,
		char *	type,
		size_t *	elem_count,
		size_t *	elem_size,
		int *	decode,
		sc_scda_ferror_t *	errcode
	)

Read the next file section header.

This is a collective function. This functions reads the next file section header and provides the user information on the subsequent file section that can be used to read the actual data in a next calling depending on the file section type one (or for a variable-size array two) functions out of sc_scda_fread_block_data, sc_scda_fread_array_data, sc_scda_fread_varray_sizes and sc_scda_fread_varray_data.

Note

All parameters are collective.

This function returns NULL on I/O errors.

Parameters

[in,out]	fc	File context previously opened by sc_scda_fopen_read.
[out]	user_string	At least SC_SCDA_USER_STRING_BYTES +1 bytes. user_string is filled with the read user string from file and is nul-terminated.
[out]	len	On output len is set to the number of bytes written to user_string excluding the terminating nul.
[out]	type	On output this char is set to 'I' (inline data), 'B' (block of given size), 'A' (fixed-size array) or 'V' (variable-size array) depending on the file section type.
[out]	elem_count	On output set to the global number of array elements if type equals 'A' or 'V'. For 'I' and 'B' as type, elem_count is set 0.
[out]	elem_size	On output set to the byte count of the array elements if type is 'A' and for the type 'B' the number of bytes. Otherwise set to 0.
[in,out]	decode	On input a Boolean to decide whether the file section shall possibly be interpreted as a compressed section, i.e. they were written by a sc_scda_fwrite_* function with encode set to true. For decode true as input the file section is interpreted as a compressed file section if the type and user string of the first raw file section satisfiy the compression convention. If the compression convention is not satisfied the data is read raw. For false as input the data will be read raw by the subsequent sc_scda_fread_* calls. The output value is always false if the input was set to false. Otherwise, the output is a Boolean that indicates if the nex file section contains a file section type and user string matching the compression convention. The subsequent sc_scda_fread_* calls do not require any adjustment dependent on decode. See also the 'Encoding' section in the detailed description in this file.
[out]	errcode	An errcode that can be interpreted by sc_scda_ferror_string or mapped to an error class by sc_scda_ferror_class. errcode encodes success if and only if the function does not return NULL.

Returns

Return a pointer to the input context fc on success. The context is used to continue reading and eventually closing the file. In case of any error, attempt to close the file and deallocate the context fc.

sc_scda_fread_varray_data()

sc_scda_fcontext_t* sc_scda_fread_varray_data	(	sc_scda_fcontext_t *	fc,
		sc_array_t *	array_data,
		sc_array_t *	elem_counts,
		sc_array_t *	elem_sizes,
		sc_array_t *	proc_sizes,
		int	indirect,
		sc_scda_ferror_t *	errcode
	)

Read the data of a variable-size array.

This is a collective function. This function is only valid to call directly after a successful call of sc_scda_fread_varray_sizes. This preceding call gives also the required elem_sizes.

Note

All parameters except of array_data and elem_sizes are collective.

Warning

The API of this function will change in the next libsc version!

This function returns NULL on I/O errors.

Parameters

[in,out]	fc	File context previously opened by sc_scda_fopen_read.
[out]	array_data	Let p be the calling rank. If indirect is false, array_data must have element count 1 and as element size the p-th entry of proc_sizes. On output the data of the array is set to the local array elements conforming with elem_counts, proc_sizes and elem_sizes. If indirect is true, array_data must be a sc_array with element count equal to the p-th array entry of elem_counts and element size equal to sizeof (sc_array_t). Each array element is again a sc_array. Now, with element count 1 and element size equals to the actual array element size as passed in elem_sizes. On output these arrays are filled with the actual elements of the read variable-size array. The data can be skipped on each process by locally passing NULL.
[in]	elem_counts	An sc_array that must be equal on all ranks. The element count of elem_counts must be the mpisize of the MPI communicator that was used to create fc. The element size of the sc_array must be equal to sizeof (sc_scda_ulong). The sc_array must contain the local array elements counts. That is why it induces the partition that is used to read the array data in parallel. The sum of all array elements must be equal to elem_count as retrieved from sc_scda_fread_section_header.
[in]	elem_sizes	The local element sizes conforming to the array element partition elem_counts as retrieved from sc_scda_fread_varray_sizes. This parameter is ignored for ranks to which NULL for array_data was passed.
[in]	proc_sizes	An sc_array that must be equal on all ranks. The element count and element size must be the same as for elem_counts. The array must contain the overall byte count per rank conforming with the passed array element partition elem_counts and the local array element sizes in elem_sizes. This parameter can be computed using sc_scda_proc_sizes.
[in]	indirect	A Boolean to determine whether array_data must be a sc_array of sc_arrays to read indirectly and in particular to potentially non-contigous memory. See the documentation of the parameter array_data for more information.
[out]	errcode	An errcode that can be interpreted by sc_scda_ferror_string or mapped to an error class by sc_scda_ferror_class. errcode encodes success if and only if the function does not return NULL.

Returns

Return a pointer to the input context fc on success. The context is used to continue reading and eventually closing the file. In case of any error, attempt to close the file and deallocate the context fc.

sc_scda_fread_varray_sizes()

sc_scda_fcontext_t* sc_scda_fread_varray_sizes	(	sc_scda_fcontext_t *	fc,
		sc_array_t *	elem_sizes,
		sc_array_t *	elem_counts,
		sc_scda_ferror_t *	errcode
	)

Read the element sizes of a variable-size array.

This is a collective function. This function is only valid to call directly after a successful call of sc_scda_fread_section_header. This preceding call gives also the for elem_counts required global number of array elements. The user must pass a parallel partition of the array elements by elem_counts.

Note

All parameters except of elem_sizes are collective.

Warning

The API of this function will change in the next libsc version!

This function returns NULL on I/O errors.

Parameters

[in,out]	fc	File context previously opened by sc_scda_fopen_read.
[out]	elem_sizes	A sc_array with element count equals to p-th entry of elem_counts for p being the calling rank. The element size must be sizeof (sc_scda_ulong). On output the array is filled with the local array element byte counts, where locality is determined by elem_counts. The element sizes can be skipped on each process by locally passing NULL.
[in]	elem_counts	An sc_array that must be equal on all ranks. The element count of elem_counts must be the mpisize of the MPI communicator that was used to create fc. The element size of the sc_array must be equal to sizeof (sc_scda_ulong). The sc_array must contain the local array elements counts. That is why it induces the partition that is used to read the array data in parallel. The sum of all array elements must be equal to elem_count as retrieved from sc_scda_fread_section_header.
[out]	errcode	An errcode that can be interpreted by sc_scda_ferror_string or mapped to an error class by sc_scda_ferror_class. errcode encodes success if and only if the function does not return NULL.

Returns

Return a pointer to the input context fc on success. The context is used to continue reading and eventually closing the file. In case of any error, attempt to close the file and deallocate the context fc.

sc_scda_fwrite_array()

sc_scda_fcontext_t* sc_scda_fwrite_array	(	sc_scda_fcontext_t *	fc,
		const char *	user_string,
		size_t *	len,
		sc_array_t *	array_data,
		sc_array_t *	elem_counts,
		size_t	elem_size,
		int	indirect,
		int	encode,
		sc_scda_ferror_t *	errcode
	)

Write a fixed-size array file section.

This is a collective function. The fixed-size array is the simplest file section that enables the user to write and read data in parallel. This function writes an array of a given element global count and a fixed element size.

Note

All parameters except of array_data are collective.

Warning

The API of this function will change in the next libsc version!

This function returns NULL on I/O errors.

Parameters

[in,out]	fc	File context previously opened by sc_scda_fopen_write.
[in]	user_string	At most SC_SCDA_USER_STRING_BYTES + 1 bytes in a nul-terminated string. See the 'User Strings' section in the detailed description of this file for more information.
[in]	len	The number of bytes in user_string excluding the terminating nul. On NULL as input user_string is expected to be a nul-terminated C string.
[in]	array_data	On rank p the p-th entry of elem_counts must be the element count of array_data. The element size of the sc_array must be equal to elem_size if indirect is false. Otherwise, array_data must be a sc_array of sc_arrays, i.e. a sc_array with element size sizeof (sc_array_t). Each of the elements of array_data is then a sc_array with element count 1 and element size elem_size. See also the documentation of the parameter indirect.
[in]	elem_counts	An sc_array that must be equal on all ranks. The element count of elem_counts must be the mpisize of the MPI communicator that was used to create fc. The element size of the sc_array must be equal to sizeof (sc_scda_ulong). The sc_array must contain the local array elements counts (sc_scda_ulong). That is why it induces the partition that is used to write the array data in parallel.
[in]	elem_size	The element size of one array element on number of bytes. Must be the same on all ranks.
[in]	indirect	A Boolean to determine whether array_data must be a sc_array of sc_arrays to write indirectly and in particular from potentially non-contigous memory. In the remaining case of indirect being false array_data must be a sc_array with element size equals to elem_size that contains the actual array elements.
[in]	encode	A Boolean to decide whether the file section is written compressed. This results in two written file sections that can be read without the encoding interpretation by using sc_scda_fread_section_header with decode set to false followed by the usual sc_scda_fread functions. The data can be read as passed to this function by using decode true in sc_scda_fread_section_header and calling the usual sc_scda_fread function. See also the 'Encoding' section in the detailed description in this file.
[out]	errcode	An errcode that can be interpreted by sc_scda_ferror_string or mapped to an error class by sc_scda_ferror_class. errcode encodes success if and only if the function does not return NULL.

Returns

Return a pointer to the input context fc on success. The context is used to continue writing and eventually closing the file. In case of any error, attempt to close the file and deallocate the context fc.

sc_scda_fwrite_block()

sc_scda_fcontext_t* sc_scda_fwrite_block	(	sc_scda_fcontext_t *	fc,
		const char *	user_string,
		size_t *	len,
		sc_array_t *	block_data,
		size_t	block_size,
		int	root,
		int	encode,
		sc_scda_ferror_t *	errcode
	)

Write a fixed-size block file section.

This is a collective function. This function writes a data block of fixed size to the file. The block_data is written on the MPI rank root. The number of block bytes must be less or equal 10^{26} - 1.

Note

All parameters except of block_data are collective.

Warning

The API of this function will change in the next libsc version!

This function returns NULL on I/O errors.

Parameters

[in,out]	fc	File context previously opened by sc_scda_fopen_write.
[in]	user_string	At most SC_SCDA_USER_STRING_BYTES + 1 bytes in a nul-terminated string. See the 'User Strings' section in the detailed description of this file for more information.
[in]	len	The number of bytes in user_string excluding the terminating nul. On NULL as input user_string is expected to be a nul-terminated C string.
[in]	block_data	On rank root a sc_array with one element and element size equals to block_size. On all other ranks the parameter is ignored. Since sc_array_init and the other sc_array_t initialization functions do not allow an element size of 0, block sections can not be empty.
[in]	block_size	The size of the data block in bytes. Must be less or equal than 10^{26} - 1.
[in]	root	An integer between 0 and mpisize of the MPI communicator that was used to create fc. root indicates the MPI rank on that block_data is written to the file.
[in]	encode	A Boolean to decide whether the file section is written compressed. This results in two written file sections that can be read without the encoding interpretation by using sc_scda_fread_section_header with decode set to false followed by the usual sc_scda_fread functions. The data can be read as passed to this function by using decode true in sc_scda_fread_section_header and calling the usual sc_scda_fread function. See also the 'Encoding' section in the detailed description in this file.
[out]	errcode	An errcode that can be interpreted by sc_scda_ferror_string or mapped to an error class by sc_scda_ferror_class. errcode encodes success if and only if the function does not return NULL.

Returns

Return a pointer to the input context fc on success. The context is used to continue writing and eventually closing the file. In case of any error, attempt to close the file and deallocate the context fc.

sc_scda_fwrite_inline()

sc_scda_fcontext_t* sc_scda_fwrite_inline	(	sc_scda_fcontext_t *	fc,
		const char *	user_string,
		size_t *	len,
		sc_array_t *	inline_data,
		int	root,
		sc_scda_ferror_t *	errcode
	)

Write an inline data section.

This is a collective function. This function writes 32 bytes of user-defined data preceded by a file section header containing a user string. In contrast to other file sections the inline data section does not end with padded data bytes and therefore require exactly 32 bytes data from the user. This enables the user to implement custom file structuring or padding.

Note

All parameters except of data are collective.

This function returns NULL on I/O errors.

Parameters

[in,out]	fc	File context previously opened by sc_scda_fopen_write.
[in]	user_string	At most SC_SCDA_USER_STRING_BYTES + 1 bytes in a nul-terminated string. See the 'User Strings' section in the detailed description of this file for more information.
[in]	len	The number of bytes in user_string excluding the terminating nul. On NULL as input user_string is expected to be a nul-terminated C string.
[in]	inline_data	On the rank root a sc_array with element count 1 and element size 32. On all other ranks this parameter is ignored.
[in]	root	An integer between 0 and mpisize exclusive of the MPI communicator that was used to create fc. root indicates the MPI rank on that inline_data is written to the file.
[out]	errcode	An errcode that can be interpreted by sc_scda_ferror_string or mapped to an error class by sc_scda_ferror_class. errcode encodes success if and only if the function does not return NULL.

Returns

Return a pointer to the input context fc on success. The context is used to continue writing and eventually closing the file. In case of any error, attempt to close the file and deallocate the context fc.

sc_scda_fwrite_varray()

sc_scda_fcontext_t* sc_scda_fwrite_varray	(	sc_scda_fcontext_t *	fc,
		const char *	user_string,
		size_t *	len,
		sc_array_t *	array_data,
		sc_array_t *	elem_counts,
		sc_array_t *	elem_sizes,
		sc_array_t *	proc_sizes,
		int	indirect,
		int	encode,
		sc_scda_ferror_t *	errcode
	)

Write a variable-size array file section.

This is a collective function. This function can be used instead of sc_scda_fwrite_array if the array elements do not have a constant element size in bytes.

Note

All parameters except of array_data and elem_sizes are collective.

Warning

The API of this function will change in the next libsc version!

This function returns NULL on I/O errors.

Parameters

[in,out]	fc	File context previously opened by sc_scda_fopen_write.
[in]	user_string	At most SC_SCDA_USER_STRING_BYTES + 1 bytes in a nul-terminated string. See the 'User Strings' section in the detailed description of this file for more information.
[in]	len	The number of bytes in user_string excluding the terminating nul. On NULL as input user_string is expected to be a nul-terminated C string.
[in]	array_data	Let p be the calling rank. If indirect is false, array_data must have element count 1 and as element size the p-th entry of proc_sizes. The data of the array must be the local array elements conforming with elem_counts, proc_sizes and elem_sizes. If indirect is true, array_data must be a sc_array with element count equal to the p-th array entry of elem_counts and element size equal to sizeof (sc_array_t). Each array element is again a sc_array. Now, with element count 1 and element size equals to the actual array element size as passed in elem_sizes.
[in]	elem_counts	An sc_array that must be equal on all ranks. The element count of elem_counts must be the mpisize of the MPI communicator that was used to create fc. The element size of the sc_array must be equal to sizeof (unint8_t). The sc_array must contain the local array elements counts. That is why it induces the partition that is used to write the array data in parallel.
[in]	elem_sizes	A sc_array with the element sizes for the local array elements. The sc_array has an element count of p-th entry of elem_counts for p being the calling rank. The element size is sizeof (sc_scda_ulong).
[in]	proc_sizes	An sc_array that must be equal on all ranks. The element count and element size must be the same as for elem_counts. The array must contain the overall byte count per rank conforming with the passed array element partition elem_counts and the local array element sizes in elem_sizes. This parameter can be computed using sc_scda_proc_sizes.
[in]	indirect	A Boolean to determine whether array_data must be a sc_array of sc_arrays to write indirectly and in particular from potentially non-contigous memory. In the remaining case of indirect being false array_data must be a sc_array with the actual array elements as data as further explained in the documentation of array_data.
[in]	encode	A Boolean to decide whether the file section is written compressed. This results in two written file sections that can be read without the encoding interpretation by using sc_scda_fread_section_header with decode set to false followed by the usual sc_scda_fread functions. The data can be read as passed to this function by using decode true in sc_scda_fread_section_header and calling the usual sc_scda_fread function. See also the 'Encoding' section in the detailed description in this file.
[out]	errcode	An errcode that can be interpreted by sc_scda_ferror_string or mapped to an error class by sc_scda_ferror_class. errcode encodes success if and only if the function does not return NULL.

Returns

Return a pointer to the input context fc on success. The context is used to continue writing and eventually closing the file. In case of any error, attempt to close the file and deallocate the context fc.

sc_scda_params_init()

void sc_scda_params_init

(

sc_scda_params_t *

params

)

Initialize a scda parameter structure to the defaults.

Parameters

[out]

params

sc_scda_params structure that is filled with the default parameters.

sc_scda_proc_sizes()

int sc_scda_proc_sizes	(	sc_array_t *	elem_sizes,
		sc_array_t *	elem_counts,
		sc_array_t *	proc_sizes,
		sc_scda_ferror_t *	errcode
	)

This is a collective function to determine the processor sizes.

The purpose of this function is determine the proc_sizes argument as required by sc_scda_fread_varray_data given the data that can be retrieved by calling sc_scda_fread_varray_sizes or as required by sc_scda_fwrite_varray given elem_sizes and elem_counts as passed to sc_scda_fwrite_varray.

Note

All parameters are collective.

Warning

The API of this function will change in the next libsc version!

Parameters

[in]	elem_sizes	The elem_sizes array as retrieved by sc_scda_fread_varray_sizes or passed to sc_scda_fwrite_varray.
[in]	elem_counts	The elem_counts array as retrieved by sc_scda_fread_varray_sizes or passed to sc_scda_fwrite_varray.
[out]	proc_sizes	A sc_array with element size sc_scda_ulong that is resized on output to the length of elem_sizes and elem_counts. The array is filled with the number bytes per process.
[out]	errcode	An errcode that can be interpreted by sc_scda_ferror_string or mapped to an error class by sc_scda_ferror_class. errcode encodes success if and only if the function does not return NULL.

Returns

0 in case of success and -1 otherwise.