/****************************************************************************
**  SIONLIB     http://www.fz-juelich.de/jsc/sionlib                       **
*****************************************************************************
**  Copyright (c) 2008-2019                                                **
**  Forschungszentrum Juelich, Juelich Supercomputing Centre               **
**                                                                         **
**  See the file COPYRIGHT in the package base directory for details       **
****************************************************************************/

#define _XOPEN_SOURCE 700

#include <ctype.h>
#include <inttypes.h>
#include <math.h>
#include <mpi.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>

#include "sion.h"

#define ONRANK if (rank == 0)

#define READ_NOT_COLLECTIVE
#define READ_COLLECTIVE

int main(int argc, char **argv)
{
  int rank, size;

  /* -------------------------- */
  /* PROLOG */
  /* -------------------------- */

  MPI_Init(&argc, &argv);
  MPI_Comm_size(MPI_COMM_WORLD, &size);
  MPI_Comm_rank(MPI_COMM_WORLD, &rank);

  /* ------------------------------------------------------------- */
  /* TEST A1: write a small file with some different pattern (Coll) */
  /* ------------------------------------------------------------- */
  {
#define BUFSIZE 10000
#define CHUNKSIZE 1000
    int sid;
    char buffer[BUFSIZE];
    int64_t bytes_written = -1;
    long sum;
    size_t bwrote;
    int i;
    sion_mpi_options *options = sion_mpi_options_new();
    if (!options)
      return 1;

    for (i = 0; i < BUFSIZE; i++) {
      buffer[i] = 'A' + rank;
    }

    sion_mpi_options_set_chunksize(options, CHUNKSIZE);
    sion_mpi_options_set_fsblksize(options, 2500);

    sid = sion_paropen_mpi("test_coll1.out", SION_OPEN_WRITE, MPI_COMM_WORLD, options);
    sion_mpi_options_delete(options);

    if (sid >= 0) {
      /* write nothing to check 0 as input */
      bwrote = sion_coll_fwrite(buffer, 0, CHUNKSIZE, sid);
      bwrote = sion_coll_fwrite(buffer, 1, 0, sid);

      bwrote = sion_coll_fwrite(buffer, 1, CHUNKSIZE, sid);

      bytes_written = sion_get_bytes_written(sid);
      ONRANK printf("on rank %d: bytes_written=%" PRId64 "\n", rank, bytes_written);
      for (i = 0, sum = 0; i < bwrote; i++) {
        sum = sum + buffer[i];
      }
      ONRANK printf("on rank %d: bwrote=%3d blocksum=%8d \n", rank, (int)bwrote, (int)sum);

      sion_parclose_mpi(sid);
    } else {
      ONRANK fprintf(stderr, "on rank %d: error sid = %d\n", rank, sid);
    }
  }

  fflush(stderr);
  fflush(stdout);
  MPI_Barrier(MPI_COMM_WORLD);
  ONRANK printf("on rank %d: END of TEST A1 write coll\n", rank);
  MPI_Barrier(MPI_COMM_WORLD);

  /* ------------------------------------------------------------------------------- */
  /* TEST A2: write a small file with some different pattern (Coll, without fileptr) */
  /* ------------------------------------------------------------------------------- */
  {
#define BUFSIZE 10000
#define CHUNKSIZE 1000
    int sid;
    char buffer[BUFSIZE];
    int64_t bytes_written = -1;
    long sum;
    size_t bwrote;
    int i;
    sion_mpi_options *options = sion_mpi_options_new();
    if (!options)
      return 1;

    for (i = 0; i < BUFSIZE; i++) {
      buffer[i] = 'A' + rank;
    }

    sion_mpi_options_set_chunksize(options, CHUNKSIZE);
    sion_mpi_options_set_fsblksize(options, 2500);

    sid = sion_paropen_mpi("test_coll2.out", SION_OPEN_WRITE, MPI_COMM_WORLD, options);
    sion_mpi_options_delete(options);

    if (sid >= 0) {
      bwrote = sion_coll_fwrite(buffer, 1, CHUNKSIZE, sid);

      bytes_written = sion_get_bytes_written(sid);
      ONRANK printf("on rank %d: bytes_written=%" PRId64 "\n", rank, bytes_written);
      for (i = 0, sum = 0; i < bwrote; i++) {
        sum = sum + buffer[i];
      }
      ONRANK printf("on rank %d: bwrote=%3d blocksum=%8d \n", rank, (int)bwrote, (int)sum);

      sion_parclose_mpi(sid);
    } else {
      ONRANK fprintf(stderr, "on rank %d: error sid = %d\n", rank, sid);
    }
  }

  fflush(stderr);
  fflush(stdout);
  MPI_Barrier(MPI_COMM_WORLD);
  ONRANK printf("on rank %d: END of TEST A2 write coll\n", rank);
  MPI_Barrier(MPI_COMM_WORLD);

  /* ------------------------------ */
  /* TEST A3: test collsize as flag */
  /* ------------------------------ */
  {
#define BUFSIZE 10000
#define CHUNKSIZE 1000
    int sid;
    char buffer[BUFSIZE];
    int i;
    sion_mpi_options *options1 = sion_mpi_options_new();
    if (!options1)
      return 1;

    for (i = 0; i < BUFSIZE; i++) {
      buffer[i] = 'A' + rank;
    }

    sion_mpi_options_set_chunksize(options1, CHUNKSIZE);
    sion_mpi_options_set_fsblksize(options1, 2500);

    /* without collsize */
    sid = sion_paropen_mpi("test_coll2.out", SION_OPEN_WRITE, MPI_COMM_WORLD, options1);

    if (sid >= 0) {
      sion_coll_fwrite(buffer, 1, CHUNKSIZE, sid);
      sion_parclose_mpi(sid);
    } else {
      ONRANK fprintf(stderr, "on rank %d: error sid = %d\n", rank, sid);
    }

    sid = sion_paropen_mpi("test_coll2.out", SION_OPEN_READ, MPI_COMM_WORLD, options1);

    if (sid >= 0) {
      ONRANK printf("on rank %d: chunksize     =%lld\n", rank, sion_get_chunksize(sid));
      sion_parclose_mpi(sid);
    } else {
      ONRANK fprintf(stderr, "on rank %d: error sid = %d\n", rank, sid);
    }

    /* with collsize */
    sion_mpi_options_set_collective_size(options1, 4);

    sid = sion_paropen_mpi("test_coll2.out", SION_OPEN_WRITE, MPI_COMM_WORLD, options1);
    sion_mpi_options_delete(options1);

    if (sid >= 0) {
      sion_coll_fwrite(buffer, 1, CHUNKSIZE, sid);
      sion_parclose_mpi(sid);
    } else {
      ONRANK fprintf(stderr, "on rank %d: error sid = %d\n", rank, sid);
    }

    sion_mpi_options *options2 = sion_mpi_options_new();
    if (!options2)
      return 1;
    sion_mpi_options_set_chunksize(options2, CHUNKSIZE);
    sion_mpi_options_set_fsblksize(options2, 2500);

    sid = sion_paropen_mpi("test_coll2.out", SION_OPEN_READ, MPI_COMM_WORLD, options2);
    sion_mpi_options_delete(options2);

    if (sid >= 0) {
      ONRANK printf("on rank %d: chunksize     =%" PRId64 "\n", rank, sion_get_chunksize(sid));
      sion_parclose_mpi(sid);
    } else {
      ONRANK fprintf(stderr, "on rank %d: error sid = %d\n", rank, sid);
    }
  }

  fflush(stderr);
  fflush(stdout);
  MPI_Barrier(MPI_COMM_WORLD);
  ONRANK printf("on rank %d: END of TEST A3 write coll\n", rank);
  MPI_Barrier(MPI_COMM_WORLD);

  /* --------------------------------------------------------------- */
  /* TEST B: write a small file with some different pattern (Indiv.) */
  /* --------------------------------------------------------------- */
  {
#define BUFSIZE 10000
#define CHUNKSIZE 1000
    int sid;
    char buffer[BUFSIZE];
    int64_t bytes_written = -1;
    long sum;
    size_t bwrote;
    int i;
    sion_mpi_options *options = sion_mpi_options_new();
    if (!options)
      return 1;

    for (i = 0; i < BUFSIZE; i++) {
      buffer[i] = 'A' + rank;
    }

    sion_mpi_options_set_chunksize(options, CHUNKSIZE);
    sion_mpi_options_set_fsblksize(options, 2500);

    sid = sion_paropen_mpi("test_indiv.out", SION_OPEN_WRITE, MPI_COMM_WORLD, options);
    sion_mpi_options_delete(options);

    if (sid >= 0) {
      bwrote = sion_fwrite(buffer, 1, CHUNKSIZE, sid);

      bytes_written = sion_get_bytes_written(sid);
      ONRANK
      {
        printf("on rank %d: bytes_written=%" PRId64 "\n", rank, bytes_written);
        fflush(stdout);
      }
      for (i = 0, sum = 0; i < bwrote; i++) {
        sum = sum + buffer[i];
      }
      ONRANK
      {
        printf("on rank %d: bwrote=%3d blocksum=%8d \n", rank, (int)bwrote, (int)sum);
        fflush(stdout);
      }

      sion_parclose_mpi(sid);
    } else {
      ONRANK fprintf(stderr, "on rank %d: error sid = %d\n", rank, sid);
    }
  }

  fflush(stderr);
  fflush(stdout);
  MPI_Barrier(MPI_COMM_WORLD);
  ONRANK printf("on rank %d: END of TEST B write coll\n", rank);
  MPI_Barrier(MPI_COMM_WORLD);

#ifdef READ_NOT_COLLECTIVE
  /* --------------------------------------------------------------------------------- */
  /* TEST C.1: read small file with some different pattern (indiv., coll. file)        */
  /* --------------------------------------------------------------------------------- */
  {
    int sid;
    char buffer[BUFSIZE];
    int64_t bytes_avail = -1;
    size_t bytestoread, bread;
    long sum;
    int i, b;
    sion_mpi_options *options = sion_mpi_options_new();
    if (!options)
      return 1;

    for (i = 0; i < BUFSIZE; i++) {
      buffer[i] = 'Q' + rank;
    }

    sion_mpi_options_set_chunksize(options, CHUNKSIZE);
    sion_mpi_options_set_fsblksize(options, 2500);

    sid = sion_paropen_mpi("test_coll1.out", SION_OPEN_READ, MPI_COMM_WORLD, options);
    sion_mpi_options_delete(options);

    if (sid >= 0) {
      ONRANK
      {
        printf("on rank %d: sid           =%d\n", rank, sid);
        printf("on rank %d: numfiles      =%d\n", rank, sion_get_number_of_files(sid));
        printf("on rank %d: chunksize     =%d\n", rank, (int)sion_get_chunksize(sid));
        printf("on rank %d: globalrank    =%d\n", rank, sion_get_globalrank(sid));
        printf("on rank %d: newfname      =%s\n", rank, sion_get_fname(sid));
      }

      b = 0;
      while ((!sion_feof(sid))) {
        bytes_avail = sion_bytes_avail_in_block(sid);
        ONRANK
        {
          printf("on rank %d: block=%d           bytes_avail=%d\n", rank, b, (int)bytes_avail);
          fflush(stdout);
        }
        bytestoread = bytes_avail;
        /* write nothing to check 0 as input */
        bread = sion_fread(buffer, 0, bytestoread, sid);
        bread = sion_fread(buffer, 1, 0, sid);

        bread = sion_fread(buffer, 1, bytestoread, sid);
        for (i = 0, sum = 0; i < bytestoread; i++) {
          sum = sum + buffer[i];
        }
        ONRANK
        {
          printf("on rank %d: block=%d           bread=%3d blocksum=%8d \n", rank, b, (int)bread, (int)sum);
          fflush(stdout);
        }
        b++;
      }

      sion_parclose_mpi(sid);
    } else {
      ONRANK fprintf(stderr, "on rank %d: error sid = %d\n", rank, sid);
    }
  }

  fflush(stderr);
  fflush(stdout);
  MPI_Barrier(MPI_COMM_WORLD);
  ONRANK printf("on rank %d: END of TEST C.1 read individual call from collective file\n", rank);
  MPI_Barrier(MPI_COMM_WORLD);

  /* --------------------------------------------------------------------------------- */
  /* TEST C.2: read small file with some different pattern (indiv., indiv. file)       */
  /* --------------------------------------------------------------------------------- */
  {
    int sid;
    char buffer[BUFSIZE];
    int64_t bytes_avail = -1;
    size_t bytestoread, bread;
    long sum;
    int i, b;
    sion_mpi_options *options = sion_mpi_options_new();
    if (!options)
      return 1;

    for (i = 0; i < BUFSIZE; i++) {
      buffer[i] = 'Q' + rank;
    }

    sion_mpi_options_set_chunksize(options, CHUNKSIZE);
    sion_mpi_options_set_fsblksize(options, 2500);
    sid = sion_paropen_mpi("test_indiv.out", SION_OPEN_READ, MPI_COMM_WORLD, options);
    sion_mpi_options_delete(options);

    if (sid >= 0) {
      ONRANK
      {
        printf("on rank %d: sid           =%d\n", rank, sid);
        printf("on rank %d: numfiles      =%d\n", rank, sion_get_number_of_files(sid));
        printf("on rank %d: chunksize     =%d\n", rank, (int)sion_get_chunksize(sid));
        printf("on rank %d: globalrank    =%d\n", rank, sion_get_globalrank(sid));
        printf("on rank %d: newfname      =%s\n", rank, sion_get_fname(sid));
      }

      b = 0;
      while ((!sion_feof(sid))) {
        bytes_avail = sion_bytes_avail_in_block(sid);
        ONRANK printf("on rank %d: block=%d           bytes_avail=%d\n", rank, b, (int)bytes_avail);
        bytestoread = bytes_avail;
        bread = sion_fread(buffer, 1, bytestoread, sid);
        for (i = 0, sum = 0; i < bytestoread; i++) {
          sum = sum + buffer[i];
        }
        ONRANK printf("on rank %d: block=%d           bread=%3d blocksum=%8d \n", rank, b, (int)bread, (int)sum);
        b++;
      }

      sion_parclose_mpi(sid);
    } else {
      ONRANK fprintf(stderr, "on rank %d: error sid = %d\n", rank, sid);
    }
  }

  fflush(stderr);
  fflush(stdout);
  MPI_Barrier(MPI_COMM_WORLD);
  ONRANK printf("on rank %d: END of TEST C.2 read individual call from individual file\n", rank);
  MPI_Barrier(MPI_COMM_WORLD);

#endif

#ifdef READ_COLLECTIVE
  /* --------------------------------------------------------------------------------- */
  /* TEST D.1: read small file with some different pattern (coll., coll. file)         */
  /* --------------------------------------------------------------------------------- */
  {
    int sid;
    char buffer[BUFSIZE];
    int64_t bytes_avail = -1;
    size_t bytestoread, bread;
    long sum;
    int i, b;
    sion_mpi_options *options = sion_mpi_options_new();
    if (!options)
      return 1;

    for (i = 0; i < BUFSIZE; i++) {
      buffer[i] = 'Q' + rank;
    }

    sion_mpi_options_set_chunksize(options, CHUNKSIZE);
    sion_mpi_options_set_fsblksize(options, 2500);

    sid = sion_paropen_mpi("test_coll1.out", SION_OPEN_READ, MPI_COMM_WORLD, options);
    sion_mpi_options_delete(options);

    if (sid >= 0) {
      ONRANK
      {
        printf("on rank %d: sid           =%d\n", rank, sid);
        printf("on rank %d: numfiles      =%d\n", rank, sion_get_number_of_files(sid));
        printf("on rank %d: chunksize     =%d\n", rank, (int)sion_get_chunksize(sid));
        printf("on rank %d: globalrank    =%d\n", rank, sion_get_globalrank(sid));
        printf("on rank %d: newfname      =%s\n", rank, sion_get_fname(sid));
      }

      /* to check that everyone has data to read, otherwise error */
      bytes_avail = sion_bytes_avail_in_block(sid);
      if (bytes_avail <= 0) {
        ONRANK fprintf(stderr, "on rank %d: ERROR: no bytes available in file bytes_avail=%d\n", rank, (int)bytes_avail);
        MPI_Abort(MPI_COMM_WORLD, 1);
      }

      b = 0;
      while ((!sion_feof(sid))) {
        bytes_avail = sion_bytes_avail_in_block(sid);
        ONRANK
        {
          printf("on rank %d: block=%d           bytes_avail=%d\n", rank, b, (int)bytes_avail);
          fflush(stdout);
        }
        bytestoread = bytes_avail;
        bread = sion_coll_fread(buffer, 1, bytestoread, sid);
        for (i = 0, sum = 0; i < bytestoread; i++) {
          sum = sum + buffer[i];
        }
        ONRANK
        {
          printf("on rank %d: block=%d           bread=%3d blocksum=%8d \n", rank, b, (int)bread, (int)sum);
          fflush(stdout);
        }
        b++;
      }

      sion_parclose_mpi(sid);
    } else {
      ONRANK fprintf(stderr, "on rank %d: error sid = %d\n", rank, sid);
    }
  }
  fflush(stderr);
  fflush(stdout);
  MPI_Barrier(MPI_COMM_WORLD);
  ONRANK printf("on rank %d: END of TEST D.1 read collective from collective file\n", rank);
  MPI_Barrier(MPI_COMM_WORLD);

  /* --------------------------------------------------------------------------------------------- */
  /* TEST D.2: read small file with some different pattern (coll., coll. file, no fileptr)         */
  /* --------------------------------------------------------------------------------------------- */
  {
    int sid;
    char buffer[BUFSIZE];
    int64_t bytes_avail = -1;
    size_t bytestoread, bread;
    long sum;
    int i, b;
    sion_mpi_options *options = sion_mpi_options_new();
    if (!options)
      return 1;

    for (i = 0; i < BUFSIZE; i++) {
      buffer[i] = 'Q' + rank;
    }

    sion_mpi_options_set_chunksize(options, CHUNKSIZE);
    sion_mpi_options_set_fsblksize(options, 2500);

    sid = sion_paropen_mpi("test_coll1.out", SION_OPEN_READ, MPI_COMM_WORLD, options);
    sion_mpi_options_delete(options);

    if (sid >= 0) {
      ONRANK
      {
        printf("on rank %d: sid           =%d\n", rank, sid);
        printf("on rank %d: numfiles      =%d\n", rank, sion_get_number_of_files(sid));
        printf("on rank %d: chunksize     =%d\n", rank, (int)sion_get_chunksize(sid));
        printf("on rank %d: globalrank    =%d\n", rank, sion_get_globalrank(sid));
        printf("on rank %d: newfname      =%s\n", rank, sion_get_fname(sid));
      }

      /* to check that everyone has data to read, otherwise error */
      bytes_avail = sion_bytes_avail_in_block(sid);
      if (bytes_avail <= 0) {
        ONRANK fprintf(stderr, "on rank %d: ERROR: no bytes available in file bytes_avail=%d\n", rank, (int)bytes_avail);
        MPI_Abort(MPI_COMM_WORLD, 1);
      }

      b = 0;
      while ((!sion_feof(sid))) {
        bytes_avail = sion_bytes_avail_in_block(sid);
        ONRANK
        {
          printf("on rank %d: block=%d           bytes_avail=%d\n", rank, b, (int)bytes_avail);
          fflush(stdout);
        }
        bytestoread = bytes_avail;
        bread = sion_coll_fread(buffer, 1, bytestoread, sid);
        for (i = 0, sum = 0; i < bytestoread; i++) {
          sum = sum + buffer[i];
        }
        ONRANK
        {
          printf("on rank %d: block=%d           bread=%3d blocksum=%8d \n", rank, b, (int)bread, (int)sum);
          fflush(stdout);
        }
        b++;
      }

      sion_parclose_mpi(sid);
    } else {
      ONRANK fprintf(stderr, "on rank %d: error sid = %d\n", rank, sid);
    }
  }

  fflush(stderr);
  fflush(stdout);
  MPI_Barrier(MPI_COMM_WORLD);
  ONRANK printf("on rank %d: END of TEST D.2 read collective from collective file\n", rank);
  MPI_Barrier(MPI_COMM_WORLD);

  /* --------------------------------------------------------------------------------- */
  /* TEST D.3: read small file with some different pattern (coll., indiv. file)        */
  /* --------------------------------------------------------------------------------- */
  {
    int sid;
    char buffer[BUFSIZE];
    int64_t bytes_avail = -1;
    size_t bytestoread, bread;
    long sum;
    int i, b;
    sion_mpi_options *options = sion_mpi_options_new();
    if (!options)
      return 1;

    for (i = 0; i < BUFSIZE; i++) {
      buffer[i] = 'Q' + rank;
    }

    sion_mpi_options_set_chunksize(options, CHUNKSIZE);
    sion_mpi_options_set_fsblksize(options, 2500);

    sid = sion_paropen_mpi("test_indiv.out", SION_OPEN_READ, MPI_COMM_WORLD, options);
    sion_mpi_options_delete(options);

    if (sid >= 0) {
      ONRANK
      {
        printf("on rank %d: sid           =%d\n", rank, sid);
        printf("on rank %d: numfiles      =%d\n", rank, sion_get_number_of_files(sid));
        printf("on rank %d: chunksize     =%d\n", rank, (int)sion_get_chunksize(sid));
        printf("on rank %d: globalrank    =%d\n", rank, sion_get_globalrank(sid));
        printf("on rank %d: newfname      =%s\n", rank, sion_get_fname(sid));
      }

      /* to check that everyone has data to read, otherwise error */
      bytes_avail = sion_bytes_avail_in_block(sid);
      if (bytes_avail <= 0) {
        ONRANK fprintf(stderr, "on rank %d: ERROR: no bytes available in file bytes_avail=%d\n", rank, (int)bytes_avail);
        MPI_Abort(MPI_COMM_WORLD, 1);
      }

      b = 0;
      while ((!sion_feof(sid))) {
        bytes_avail = sion_bytes_avail_in_block(sid);
        ONRANK
        {
          printf("on rank %d: block=%d           bytes_avail=%d\n", rank, b, (int)bytes_avail);
          fflush(stdout);
        }
        bytestoread = bytes_avail;
        bread = sion_coll_fread(buffer, 1, bytestoread, sid);
        for (i = 0, sum = 0; i < bytestoread; i++) {
          sum = sum + buffer[i];
        }
        ONRANK
        {
          printf("on rank %d: block=%d           bread=%3d blocksum=%8d \n", rank, b, (int)bread, (int)sum);
          fflush(stdout);
        }
        b++;
      }

      sion_parclose_mpi(sid);
    } else {
      ONRANK fprintf(stderr, "on rank %d: error sid = %d\n", rank, sid);
    }
  }

  fflush(stderr);
  fflush(stdout);
  MPI_Barrier(MPI_COMM_WORLD);
  ONRANK printf("on rank %d: END of TEST D.3 read collective from individual file\n", rank);
  MPI_Barrier(MPI_COMM_WORLD);

#endif

  /* --------------------------------------------------------------- */
  /* TEST E: write collective with a sequence of chunks,calls        */
  /* --------------------------------------------------------------- */
  {
#define BUFSIZE 10000
#define CHUNKSIZE 1000
    int sid;
    char buffer[BUFSIZE];
    int64_t bytes_written = -1;
    long sum;
    size_t bwrote;
    int i, j;
    sion_mpi_options *options = sion_mpi_options_new();
    if (!options)
      return 1;

    for (i = 0; i < BUFSIZE; i++) {
      buffer[i] = 'A' + rank;
    }

    sion_mpi_options_set_chunksize(options, CHUNKSIZE);
    sion_mpi_options_set_fsblksize(options, 2500);

    sid = sion_paropen_mpi("test_coll_sequence.out", SION_OPEN_WRITE, MPI_COMM_WORLD, options);
    sion_mpi_options_delete(options);

    if (sid >= 0) {
      /* 3 times full chunks  */
      for (j = 0; j < 3; j++) {
        bwrote = sion_coll_fwrite(buffer, 1, CHUNKSIZE, sid);

        bytes_written = sion_get_bytes_written(sid);
        ONRANK printf("on rank %d: bytes_written=%" PRId64 "\n", rank, bytes_written);
        for (i = 0, sum = 0; i < bwrote; i++) {
          sum = sum + buffer[i];
        }
        ONRANK printf("on rank %d: bwrote=%3d blocksum=%8d \n", rank, (int)bwrote, (int)sum);
      }

      /* 6 times half chunks  */
      for (j = 0; j < 6; j++) {
        bwrote = sion_coll_fwrite(buffer, 1, CHUNKSIZE / 2, sid);

        bytes_written = sion_get_bytes_written(sid);
        ONRANK printf("on rank %d: bytes_written=%" PRId64 "\n", rank, bytes_written);
        for (i = 0, sum = 0; i < bwrote; i++) {
          sum = sum + buffer[i];
        }
        ONRANK printf("on rank %d: bwrote=%3d blocksum=%8d \n", rank, (int)bwrote, (int)sum);
      }

      /* 6 times 3/4 chunks  */
      for (j = 0; j < 6; j++) {
        bwrote = sion_coll_fwrite(buffer, 1, CHUNKSIZE * 3 / 4, sid);

        bytes_written = sion_get_bytes_written(sid);
        ONRANK printf("on rank %d: bytes_written=%" PRId64 "\n", rank, bytes_written);
        for (i = 0, sum = 0; i < bwrote; i++) {
          sum = sum + buffer[i];
        }
        ONRANK printf("on rank %d: bwrote=%3d blocksum=%8d \n", rank, (int)bwrote, (int)sum);
      }

      sion_parclose_mpi(sid);
    } else {
      ONRANK fprintf(stderr, "on rank %d: error sid = %d\n", rank, sid);
    }
  }

  fflush(stderr);
  fflush(stdout);
  MPI_Barrier(MPI_COMM_WORLD);
  ONRANK printf("on rank %d: END of TEST E write coll sequence\n", rank);
  MPI_Barrier(MPI_COMM_WORLD);

  /* --------------------------------------------------------------- */
  /* TEST F: read collective with a sequence of chunks,calls         */
  /* --------------------------------------------------------------- */
  {
    int sid;
    char buffer[BUFSIZE];
    size_t bytestoread, bread;
    long sum;
    int i, j, b;
    sion_mpi_options *options = sion_mpi_options_new();
    if (!options)
      return 1;

    for (i = 0; i < BUFSIZE; i++) {
      buffer[i] = 'Q' + rank;
    }

    sion_mpi_options_set_chunksize(options, CHUNKSIZE);
    sion_mpi_options_set_fsblksize(options, 2500);

    sid = sion_paropen_mpi("test_coll_sequence.out", SION_OPEN_READ, MPI_COMM_WORLD, options);
    sion_mpi_options_delete(options);

    if (sid >= 0) {
      ONRANK
      {
        printf("on rank %d: sid           =%d\n", rank, sid);
        printf("on rank %d: numfiles      =%d\n", rank, sion_get_number_of_files(sid));
        printf("on rank %d: chunksize     =%d\n", rank, (int)sion_get_chunksize(sid));
        printf("on rank %d: globalrank    =%d\n", rank, sion_get_globalrank(sid));
        printf("on rank %d: newfname      =%s\n", rank, sion_get_fname(sid));
      }

      b = 0;

      /* 3 times full chunks  */
      for (j = 0; j < 3; j++) {
        if (!sion_feof(sid)) {
          bytestoread = CHUNKSIZE;
          bread = sion_coll_fread(buffer, 1, bytestoread, sid);
          for (i = 0, sum = 0; i < bytestoread; i++) {
            sum = sum + buffer[i];
          }
          b++;
          ONRANK
          {
            printf("on rank %d: block=%d           bread=%3d blocksum=%8d \n", rank, b, (int)bread, (int)sum);
            fflush(stdout);
          }
        }
      }

      /* 6 times half chunks  */
      for (j = 0; j < 6; j++) {
        if (!sion_feof(sid)) {
          bytestoread = CHUNKSIZE / 2;
          bread = sion_coll_fread(buffer, 1, bytestoread, sid);
          for (i = 0, sum = 0; i < bytestoread; i++) {
            sum = sum + buffer[i];
          }
          b++;
          ONRANK
          {
            printf("on rank %d: block=%d           bread=%3d blocksum=%8d \n", rank, b, (int)bread, (int)sum);
            fflush(stdout);
          }
        }
      }

      /* 6 times 3/4 chunks  */
      for (j = 0; j < 6; j++) {
        if (!sion_feof(sid)) {
          bytestoread = CHUNKSIZE * 3 / 4;
          bread = sion_coll_fread(buffer, 1, bytestoread, sid);
          for (i = 0, sum = 0; i < bytestoread; i++) {
            sum = sum + buffer[i];
          }
          b++;
          ONRANK
          {
            printf("on rank %d: block=%d           bread=%3d blocksum=%8d \n", rank, b, (int)bread, (int)sum);
            fflush(stdout);
          }
        }
      }

      sion_parclose_mpi(sid);
    } else {
      ONRANK fprintf(stderr, "on rank %d: error sid = %d\n", rank, sid);
    }
  }

  fflush(stderr);
  fflush(stdout);
  MPI_Barrier(MPI_COMM_WORLD);
  ONRANK printf("on rank %d: END of TEST F read collective from collective file\n", rank);
  MPI_Barrier(MPI_COMM_WORLD);

  /* -------------------------- */
  /* EPILOG */
  /* -------------------------- */
  MPI_Finalize();

  return 0;
}
