/****************************************************************************
**  LinkTest                                                               **
*****************************************************************************
**  Copyright (c) 2008-2010                                                **
**  Forschungszentrum Juelich, Juelich Supercomputing Centre               **
**                                                                         **
**  See the file COPYRIGHT in the package base directory for details       **
****************************************************************************/

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include <math.h>

#include "sion.h"
#include "mapping.h"
#include "ppmwrite.h"
#include "postscript_driver.h"
#include "pingponganalysis_tools.h"

#define FEHLERINDATENno

#define FILENAME_LENGTH 1024

#define MAXENTRIES 10000


#define MB *1024*1024

int length_of_message = (128*1024);

double opt_minbw=1;
double opt_maxbw=10000;
double opt_mintime=-1e20;
double opt_maxtime=1e20;
int do_range=0;

void usage(char *name) {
  fprintf(stderr, "Usage: %s options <insionfn> \n\nwith the following optional options (default values in parathesis):\n\n",name);

  fprintf(stderr, "  [-a]                   generate accesspattern file (PPM) \n");
  fprintf(stderr, "  [-A]                   generate accesspattern file (ASCII) \n");
  fprintf(stderr, "  [-B]                   generate badlink list (ASCII) \n");
  fprintf(stderr, "  [-d]                   generate distancepattern file (dat) \n");
  fprintf(stderr, "  [-b]                   generate bandwidthpattern file (PPM) \n");
  fprintf(stderr, "  [-l] <minbw>           min. bandwidth, conection below will be reported  (def. 1 MB/s) \n");
  fprintf(stderr, "  [-L] <maxbw>           max. bandwidth, conection above will be reported  (def. 10000 MB/s) \n");
  fprintf(stderr, "  [-g]                   generate gnuplot 2d input file \n");
  fprintf(stderr, "  [-p]                   generate postscript report\n");
  fprintf(stderr, "  [-v]                   verbose mode \n");
  exit(1);
}

double time2mbs( double time) {
  if(time>0) {
    return((length_of_message * sizeof(char))/1024.0/1024.0/time );
  } else {
    return(0.0);
  }
}

double mbs2time( double mbs) {
  if(mbs>0) {
    return( (length_of_message*sizeof(char))/(mbs*1024*1024));
  } else {
    return(0.0);
  }
}


int main(int argc, char **argv)
{
  FILE *fp, *outfp;
  char infilename[FILENAME_LENGTH];

  int i,rank,f_rank,blknum,p;
  int nullcount;
  sion_int64 chunksize=0;
  char *localbuffer;
  sion_int64 *chunksizes=NULL;
  int  *globalranks=NULL;

  /* options */
  int  verbose=0;
  sion_int64 opt_localsize=-1;
  sion_int32 opt_fsblocksize=-1;
  int opt_steps=500;
  int opt_doaccesspatternppm=0;
  int opt_doaccesspatternascii=0;
  int opt_dobwpatternppm=0;
  int opt_dognuplotfile=0;
  int opt_dopsreport=0;
  int opt_dobadlinkreport=0;

  /* for file infomation */
  int ntasks,numtasks,nfiles;
  sion_int32 fsblksize;
  int sid,outsid,size,blocks;
  sion_int64  globalskip;
  sion_int64  start_of_varheader;
  sion_int64 *sion_chunksizes;
  sion_int64 *sion_globalranks;
  sion_int64 *sion_blockcount;
  sion_int64 *sion_blocksizes;
  sion_int64 *siondump_sum_bytes_per_task;
  sion_int64  siondump_sum_bytes;
  sion_int64  siondump_filesize;
  char filename_ppm[256];
  char distfilename_ppm[256];
  char bwfilename_ppm[256];
  FILE *outfile_ppm,*outdistfile,*accessfile,*stepfile1,*stepfile2, *outbwfile_ppm, *gnuplotfile, *psreportfile;
  int t;
  double opt_factor=1.0;
  char helpstr[256];
  
  int opt_dodistpattern=0;
  int minentriesX[MAXENTRIES];
  int minentriesY[MAXENTRIES];
  double minentriesVAL[MAXENTRIES];
  int minentriesfound=0;
  int maxentriesX[MAXENTRIES];
  int maxentriesY[MAXENTRIES];
  double maxentriesVAL[MAXENTRIES];
  int maxentriesfound=0;
  int maxx,maxy,maxz,maxt,maxdist=0;
  int distx,disty,distz,distt;
  int distx1,disty1,distz1;
  int distx2,disty2,distz2;
  int collectpnum,pos;
  int number_of_messages;
  int do_alltoall,do_serial,do_mix,opt_donodelist;
  int opt_mask=0;
  int opt_torus=0;
  char opt_maskfile[FILENAME_LENGTH];
  double opt_mintimecol=-1.0,opt_maxtimecol=-1.0;

  /* analysis data */
  analysis_fields_t fields;

  char       location[MAX_LOCATION_LENGTH];
  sion_int32 mappingout[4];
  double     statdata_dp[10];
  sion_int32 statdata_int[10];

  /* derived analysis data */
  double  ptimings_min, ptimings_max, ptimings_avg, stepwidth, atimings_min, atimings_max, atimings_avg;
  int     help, doswap;
  opt_donodelist=0;
  

  /* parse command line */
  i=1;
  if (argc < 2)
    usage(argv[0]);

  while( i < argc ) {
    if( argv[i][0] == '-' ) {
      switch( argv[i][1] ) {
      case 'q':
        opt_fsblocksize = atoi(argv[++i]);
        break;
      case 'Q':
        opt_fsblocksize = atoi(argv[++i]) MB;
        break;
      case 'D':
        opt_donodelist=1;
        break;
      case 's':
        opt_steps = atoi(argv[++i]);
        break;
      case 'l':
        opt_minbw = atof(argv[++i]);
	do_range=1;
        break;
      case 'L':
        opt_maxbw = atof(argv[++i]);
	do_range=1;
        break;
      case 'f':
        opt_factor = atof(argv[++i]);
        break;
      case 'v':
        verbose++;
        break;
      case 'a':
        opt_doaccesspatternppm=1;
        break;
      case 'A':
        opt_doaccesspatternascii=1;
        break;
      case 'B':
        opt_dobadlinkreport=1;
        break;
      case 'd':
        opt_dodistpattern=1;
        break;
      case 'b':
        opt_dobwpatternppm=1;
        break;
      case 'g':
        opt_dognuplotfile=1;
        break;
      case 'p':
        opt_dopsreport=1;
        break;
      case 'm':
        opt_mask = 1;
	strcpy(opt_maskfile,argv[++i]);
        break;
      case 't':
        opt_mintimecol = atof(argv[++i]);
        break;
      case 'T':
        opt_maxtimecol = atof(argv[++i]);
        break;

      default:
        usage(argv[0]);
      }
    }
    i++;
  }


  /* STEP: open binary SION file */
  strcpy(infilename,argv[argc-1]);
  printf("pingponganalysis: infilename:                %-30s\n",infilename);
  chunksizes=NULL;globalranks=NULL; /* will be allocated by sion_open */
  sid=sion_open(infilename,"rb",&ntasks,&nfiles,&chunksizes,&fsblksize,&globalranks,&fp);

  /* STEP: print header information */
  printf("pingponganalysis: sid:                       %d\n",sid);
  printf("pingponganalysis: filename:                  %-30s\n",infilename);
  printf("pingponganalysis: fsblksize:                 %lu bytes (%6.2f MB)\n",(unsigned long)fsblksize,fsblksize/1024.0/1024.0);
  printf("pingponganalysis: current endianness:        %s\n",(sion_get_endianness())?"big":"little");
  printf("pingponganalysis: file endianness:           %s\n", (sion_get_file_endianness(sid)) ? "big" : "little");
  
  doswap=sion_get_file_endianness(sid)!=sion_get_endianness();
  printf("pingponganalysis: big/little  swapping:      --> %d\n", doswap);
  
  /* STEP: get more information from SION file and print it */
  sion_get_locations(sid,&size,&blocks,&globalskip,&start_of_varheader,&sion_chunksizes,
		     &sion_globalranks,&sion_blockcount,&sion_blocksizes);
  printf("pingponganalysis: start_of_varheader:        %lld bytes  (%6.2f MB)\n",start_of_varheader,start_of_varheader/1024.0/1024.0);
  printf("pingponganalysis: max number of blocks:      %d\n",blocks);
  printf("pingponganalysis: ntasks:                    %d\n",ntasks);
  if(blocks>1) {
    fprintf(stderr, "wrong sion file max #blocks > 1 (%d)\n", blocks);
    return(1);
  }
  
  /* STEP: read general data of measurement */
  sion_seek(sid,0,0,0);
  freaddata(location,(long) MAX_LOCATION_LENGTH,fp);

  freaddata(&mappingout[0],(long) 4 * sizeof(sion_int32),fp);
  sion_swap(mappingout,mappingout,sizeof(sion_int32),4,doswap);

  freaddata(&statdata_int[0],(long) 10 * sizeof(sion_int32),fp);
  sion_swap(statdata_int,statdata_int,sizeof(sion_int32),10,doswap);

  freaddata(&statdata_dp[0],(long) 10 * sizeof(double),fp);
  sion_swap(&statdata_dp[0],&statdata_dp[0],sizeof(double),10,doswap);

  
  /* STEP: set internal variables */
  number_of_messages=statdata_int[0];  length_of_message=statdata_int[1];
  do_alltoall=statdata_int[2];         collectpnum=statdata_int[3];
  do_serial=statdata_int[4];           do_mix=statdata_int[5];
  if(collectpnum<=0) collectpnum=1;
  ptimings_min=statdata_dp[0];         ptimings_max=statdata_dp[1];  ptimings_avg=statdata_dp[2];

  if(opt_mintimecol>0) ptimings_min=opt_mintimecol;
  if(opt_maxtimecol>0) ptimings_max=opt_maxtimecol;

  printf("pingponganalysis: opt_min max timecol:       %11.2f us %11.2f us\n",opt_mintimecol*1000*1000.0, opt_maxtimecol*1000*1000.0);

  stepwidth=(ptimings_max-ptimings_min)/opt_steps;
  if(do_alltoall) {
    atimings_min=statdata_dp[3];
    atimings_max=statdata_dp[4];
    atimings_avg=statdata_dp[5];
  }

  /* STEP: calculate min max limits for scan */
  opt_maxtime=mbs2time(opt_minbw);
  opt_mintime=mbs2time(opt_maxbw);
  printf("pingponganalysis: reporting link outside range:       %11.2f us   < t < %11.2f us\n",opt_mintime*1000*1000.0, opt_maxtime*1000*1000.0);
  printf("pingponganalysis: reporting link outside range:       %11.2f MB/s < t < %11.2f MB/s\n",opt_minbw, opt_maxbw);

  printf("pingponganalysis: ->stat_data_int:              %10d %10d %10d %10d %10d\n",
         statdata_int[0],statdata_int[1],statdata_int[2],statdata_int[3],statdata_int[4]);
  printf("pingponganalysis: ->statdata_dp:               %e %e %e %e %e\n",
         statdata_dp[0],statdata_dp[1],statdata_dp[2],statdata_dp[3],statdata_dp[4]);
  printf("pingponganalysis: length_of_message:              %d\n",length_of_message);
  printf("pingponganalysis: collectpnum:                    %d\n",collectpnum);
  printf("pingponganalysis: alltoall:                       %d\n",do_alltoall);
  numtasks=ntasks*collectpnum;
  printf("pingponganalysis: matsize:                        %d\n",numtasks);
  printf("pingponganalysis: opt_mask:                       %d\n",opt_mask);
  if(opt_mask) {
    printf("pingponganalysis: opt_maskfile:                   %s\n",opt_maskfile);
  }

  /* STEP: allocate fields for reading data of one task */
  allocate_fields(&fields, numtasks, opt_steps);

  if(opt_mask) {
    read_maskfile(&fields,numtasks,opt_maskfile);
  }


  /* STEP: init output files */
  ppminitsmooth(1); 
  sprintf(filename_ppm,"accesspattern.ppm");
  sprintf(distfilename_ppm,"distpattern.ppm");
  sprintf(bwfilename_ppm,"bwpattern.ppm");
  if(opt_doaccesspatternppm) {
    outfile_ppm=ppmopen(numtasks,numtasks,0.0,1.0*numtasks,filename_ppm);
  }
  if(opt_doaccesspatternascii) {
    accessfile=fopen("accesspattern.dat","w");
  }
  if(opt_dodistpattern) {
    outdistfile=fopen("distancepattern.dat","w");
  }

  if(opt_dobwpatternppm) {
    outbwfile_ppm=ppmopen(numtasks,numtasks,0.0,1.0*numtasks,bwfilename_ppm);
  }
  if(opt_dognuplotfile) {
    gnuplotfile=fopen("gnuplotinput.dat","w");
  }
  if(opt_dopsreport) {
    int offy;
#define FIRSTROW 5
#define SECONDROW 300
#define FONTSIZE 9

    psreportfile=fopen("report_fzjlinktest.ps","w");
    PSheader(psreportfile);
    textr(psreportfile,infilename,PSTITELX,PSTITELY,14);
    rect(psreportfile,PSDESCX,PSDESCY,PSDESCW,PSDESCH,0,0.1);
    offy=PSDESCY+PSDESCH;
    sprintf(helpstr,"length_of_message:   %d bytes (%8.2f KBytes)",length_of_message,length_of_message/1024.0);
    text(psreportfile,helpstr,PSDESCX+FIRSTROW,offy-11,FONTSIZE);
    sprintf(helpstr,"number_of_tasks:     %d ",numtasks);
    text(psreportfile,helpstr,PSDESCX+SECONDROW,offy-11,FONTSIZE);
    offy-=12;
    sprintf(helpstr,"number_of_messages:  %d ",number_of_messages);
    text(psreportfile,helpstr,PSDESCX+FIRSTROW,offy-11,FONTSIZE);
    sprintf(helpstr,"Execution order:     %s ",(do_serial)?"Serial":"Parallel");
    text(psreportfile,helpstr,PSDESCX+SECONDROW,offy-11,FONTSIZE);
    offy-=12;
    sprintf(helpstr,"Alltoall:            %d ",do_alltoall);
    text(psreportfile,helpstr,PSDESCX+FIRSTROW,offy-11,FONTSIZE);
    sprintf(helpstr,"Mixing PE rank:     %s ",(do_mix)?"Yes":"No");
    text(psreportfile,helpstr,PSDESCX+SECONDROW,offy-11,FONTSIZE);
    offy-=12;
    sprintf(helpstr,"Min Value:           %6.1fus (%7.2f MB/s)",ptimings_min*1000*1000,time2mbs(ptimings_min));
    text(psreportfile,helpstr,PSDESCX+FIRSTROW,offy-11,FONTSIZE);
    if(do_alltoall) {
      sprintf(helpstr,"Alltoall Min Value:  %8.1fus (1 Byte)",atimings_min*1000*1000);
      text(psreportfile,helpstr,PSDESCX+SECONDROW,offy-11,FONTSIZE);
    }
    offy-=12;
    sprintf(helpstr,"Max Value:           %6.1fus (%7.2f MB/s)",ptimings_max*1000*1000,time2mbs(ptimings_max));
    text(psreportfile,helpstr,PSDESCX+FIRSTROW,offy-11,FONTSIZE);
    if(do_alltoall) {
      sprintf(helpstr,"Alltoall Max Value:  %8.1fus (1 Byte)",atimings_max*1000*1000);
      text(psreportfile,helpstr,PSDESCX+SECONDROW,offy-11,FONTSIZE);
    }
    offy-=12;
    sprintf(helpstr,"Avg Value:           %6.1fus (%7.2f MB/s)",ptimings_avg*1000*1000,time2mbs(ptimings_avg));
    text(psreportfile,helpstr,PSDESCX+FIRSTROW,offy-11,FONTSIZE);
    if(do_alltoall) {
      sprintf(helpstr,"Alltoall Avg Value:  %8.1fus (1 Byte)",atimings_avg*1000*1000);
      text(psreportfile,helpstr,PSDESCX+SECONDROW,offy-11,FONTSIZE);
    }
    
  }

  /* STEP: scan all data for mappingdata -> calculate size of torus */
  maxx=maxy=maxz=maxt=0;
  for(f_rank=0;f_rank<size;f_rank++) {
    for(p=0;p<collectpnum;p++) {
      rank=f_rank*collectpnum+p;
      pos=p*(MAX_LOCATION_LENGTH+4 * sizeof(sion_int32) + numtasks*sizeof(double) + numtasks*sizeof(sion_int32)
             + ( (do_alltoall>0)?numtasks*sizeof(double):0) );
      if ((f_rank==0) && (p>0)) {
        pos+= 10*sizeof(sion_int32) + 10*sizeof(double);
        pos+= 2*sizeof(sion_int32)*numtasks + 2*sizeof(double)*numtasks;
      }
      sion_seek(sid,f_rank,0,pos);
      freaddata(location,(long) MAX_LOCATION_LENGTH,fp);
      strcpy(fields.alllocation+rank*MAX_LOCATION_LENGTH,location);
      freaddata(&mappingout[0],(long) 4 * sizeof(sion_int32),fp);
      sion_swap(mappingout,mappingout,sizeof(sion_int32),4,doswap);

      if((rank)%collectpnum==0) {
        printf("pingponganalysis: ->rank: %3d, f_rank=%3d, p=%3d -> location=>%s< %dx%dx%dx%d pos=%d\n",
	       rank,f_rank,p,fields.alllocation+rank*MAX_LOCATION_LENGTH,
               mappingout[0],mappingout[1],mappingout[2],mappingout[3],pos);
      }
      fields.mapping_x[rank]=mappingout[0];      fields.mapping_y[rank]=mappingout[1];
      fields.mapping_z[rank]=mappingout[2];      fields.mapping_t[rank]=mappingout[3];
      if (mappingout[0]>maxx) maxx=mappingout[0];
      if (mappingout[1]>maxy) maxy=mappingout[1];
      if (mappingout[2]>maxz) maxz=mappingout[2];
      if (mappingout[3]>maxt) maxt=mappingout[3];
    }
  }
  maxx++;maxy++;maxz++;maxt++;
  maxdist=(int) (maxx/2.0+maxy/2.0+maxz/2.0);
  printf("pingponganalysis: -> size: %dx%dx%dx%d maxdist=%d\n",maxx,maxy,maxz,maxt,maxdist);

  
  if(opt_dopsreport) {
    PSplotscala (psreportfile, PSSCALENELEM, ptimings_min,ptimings_max);
  }

  /* STEP: read data for each rank and process data */
  for(f_rank=0;f_rank<size;f_rank++) {
   
    sion_seek(sid,f_rank,0,0);
    
    for(p=0;p<collectpnum;p++) {
      rank=f_rank*collectpnum+p;

      /* read data from block */
      freaddata(location,(long) MAX_LOCATION_LENGTH,fp);
      strcpy(fields.alllocation+rank*MAX_LOCATION_LENGTH,location);
      freaddata(&mappingout[0],(long) 4 * sizeof(sion_int32),fp);
      sion_swap(mappingout,mappingout,sizeof(sion_int32),4,doswap);
      if((rank)%collectpnum==0) {
        printf("pingponganalysis: ->rank:                    %d \n",rank);
      }

      /* read parameter and statistic data */
      if(rank==0) {
        freaddata(&statdata_int[0],(long) 10 * sizeof(sion_int32),fp);
	sion_swap(statdata_int,statdata_int,sizeof(sion_int32),10,doswap);
        freaddata(&statdata_dp[0],(long) 10 * sizeof(double),fp);
	sion_swap(&statdata_dp[0],&statdata_dp[0],sizeof(double),10,doswap);
      }

      /* timings, first run  */
      freaddata(&fields.ptimings[0],(long) numtasks * sizeof(double),fp);
      sion_swap(&fields.ptimings[0],&fields.ptimings[0],sizeof(double),numtasks,doswap);

      /* analysis of ptimings */
      for(t=0;t<numtasks;t++) {

	if(opt_mask) {
	  if(!fields.mask[rank*numtasks+t]) fields.ptimings[t]=-1;
	}
	
	if(opt_dognuplotfile) {
	  if(fields.ptimings[t]>=0) {
	    fprintf(gnuplotfile,"%10d %10d %18.8f %18.8f\n",rank,t,fields.ptimings[t],time2mbs(fields.ptimings[t]));
	  } else {
	    fprintf(gnuplotfile,"%10d %10d %18.8f %18.8f\n",rank,t,0.0,0.0);
	  }
	}

        if(rank != t ) {
          if(fields.ptimings[t]<fields.timings_min) fields.timings_min=fields.ptimings[t];
          if(fields.ptimings[t]>fields.timings_max) fields.timings_max=fields.ptimings[t];
        
	  fields.nodeslist_fromavg[rank]+=fields.ptimings[t];
	  fields.nodeslist_toavg[t]+=fields.ptimings[t];
	  if(fields.ptimings[t]>fields.nodeslist_frommax[rank]) fields.nodeslist_frommax[rank]=fields.ptimings[t];
          if(fields.ptimings[t]>fields.nodeslist_tomax[t])      fields.nodeslist_tomax[t]=     fields.ptimings[t];
        
	  if(time2mbs(fields.ptimings[t])<1000)          fields.nodeslist_fromcntmax[rank]++;
          if(time2mbs(fields.ptimings[t])<1000)          fields.nodeslist_tocntmax[t]++;

          if((do_range) && (fields.ptimings[t]<opt_mintime) && (minentriesfound<MAXENTRIES)) {
            minentriesX[minentriesfound]=rank;
            minentriesY[minentriesfound]=t;
            minentriesVAL[minentriesfound]=fields.ptimings[t];
            /*           printf("[%d: %dx%d %e]\n",minentriesfound,(rank),t,ptimings[t]); */
            minentriesfound++;
          }

          if((do_range) && (fields.ptimings[t]>opt_maxtime) && (maxentriesfound<MAXENTRIES)) {
            maxentriesX[maxentriesfound]=rank;
            maxentriesY[maxentriesfound]=t;
            maxentriesVAL[maxentriesfound]=fields.ptimings[t];
            maxentriesfound++;
          }
                
	  if(fields.ptimings[t]>=0) {
	    help= (int) ((fields.ptimings[t]-ptimings_min)/stepwidth);
	    if(help>=opt_steps) {
	      /* 	    printf("r=%d t=%d %d>%d val=%e stepwidth=%e\n",(rank),t,help,opt_steps,ptimings[t],stepwidth);  */
	      help=opt_steps-1;
	    }
	    if(help<0) {
	      printf("r=%d t=%d %d<0 val=%e stepwidth=%e\n",rank,t,help,fields.ptimings[t],stepwidth);
	      help=0;
	    }
	    /* 	  printf("r=%d t=%4d help=%4d val=%10.4f(%7.1fMB/s) stepwidth=%10.4f(%7.1fMB/s)\n", */
	    /* 		 rank,t,help,ptimings[t]*1000*1000,time2mbs(ptimings[t]),stepwidth*1000*1000,time2mbs(stepwidth)); */
	    fields.ptimings_stepcnt[help]++;
	  }
        }
      }

      if(opt_dopsreport) {
	PSplotrow(psreportfile,fields.ptimings,rank,numtasks,ptimings_min,ptimings_max,fields.alllocation+rank*MAX_LOCATION_LENGTH,
		  do_range,opt_mintime,opt_maxtime);
      }


      /* scan accesspattern */
      freaddata(&fields.accesspattern[0],(long) numtasks * sizeof(sion_int32),fp);
      sion_swap(&fields.accesspattern[0],&fields.accesspattern[0],sizeof(sion_int32),numtasks,doswap);
      if(opt_doaccesspatternascii) {
	for(t=0;t<numtasks;t++) {
	  fprintf(accessfile,"%8d %8d %8d\n",rank,t,fields.accesspattern[t]);  
	}
      }
      if(opt_doaccesspatternppm) {
	for(t=0;t<numtasks;t++) {
	  if(fields.accesspattern[t]<0) {
	    printf("pingponganalysis: accesspattern[%d]=%d < 0, rank=%d\n",t,fields.accesspattern[t],rank);
	    fields.accesspattern[t]=0;
	  }
	  fields.accesspatterndp[t]=fields.accesspattern[t];
	}
	ppmwriteblock(outfile_ppm,fields.accesspatterndp,numtasks,1,0.0,1.0*numtasks);
      }

      /* analysis of ptimings in respect to distance */
      if(opt_dodistpattern) {
	for(t=0;t<numtasks;t++) {

	  distx1=abs(fields.mapping_x[rank]-fields.mapping_x[t]);
	  distx2=maxx-abs(fields.mapping_x[rank]-fields.mapping_x[t]);
	  if(maxx>1) distx=(distx1<distx2)?distx1:distx2;
	  else       distx=0;

	  disty1=abs(fields.mapping_y[rank]-fields.mapping_y[t]);
	  disty2=maxy-abs(fields.mapping_y[rank]-fields.mapping_y[t]);
	  if(maxy>1) disty=(disty1<disty2)?disty1:disty2;
	  else       disty=0;

	  distz1=abs(fields.mapping_z[rank]-fields.mapping_z[t]);
	  distz2=maxz-abs(fields.mapping_z[rank]-fields.mapping_z[t]);
	  if(maxz>1) distz=(distz1<distz2)?distz1:distz2;
	  else       distz=0;

	  if(maxt>1) distt=abs(fields.mapping_t[rank]-fields.mapping_t[t]);
	  else       distt=0;
	  distt=0;

	  fields.distance[t]=distx+disty+distz+distt;  
	  /*         distance[t]=mapping_z[rank];  */
	  
	  fprintf(outdistfile,
		  "%4d %4d  %2d %2d %2d %2d  %2d %2d %2d %2d  %4d %18.8e\n",rank,t,
		 fields.mapping_x[rank],fields.mapping_y[rank],fields.mapping_z[rank],fields.mapping_t[rank],
		 fields.mapping_x[t],   fields.mapping_y[t],   fields.mapping_z[t],   fields.mapping_t[t],
		 fields.distance[t],    fields.ptimings[t]);
	  
	  /* fprintf(outdistfile, */
	  /* 	  "T: %4d %4d %4d %18.8e\n",rank,t, fields.distance[t],    fields.ptimings[t]); */
	}
	
      }
      
      /* Blue Gene/P torus report  */
      if(opt_torus) {
	for(t=0;t<numtasks;t++) {
	    }
      }

      if(opt_dobwpatternppm) {
	for(t=0;t<numtasks;t++) {
	  if(fields.ptimings[t]<0) fields.ptimings[t]=0.0;
	  /*           if(distance[t]>0) { */
	  /*             ptimings[t]*=(double) opt_factor * 1.0/distance[t]; */
	  /*           } */
	}
	ppmwriteblock(outbwfile_ppm,fields.ptimings,numtasks,1,0.0,opt_factor*fields.timings_max);
      }

      /* all to all timings  */
      if(do_alltoall==1) {
	freaddata(&fields.atimings[0],(long) numtasks * sizeof(double),fp);
	sion_swap(&fields.atimings[0],&fields.atimings[0],sizeof(double),numtasks,doswap);
      } 

      /* single link tests, only store on task 0  */
      if(rank==0) {
	freaddata(&fields.stimings1[0],(long) numtasks * sizeof(double),fp);
	sion_swap(&fields.stimings1[0],&fields.stimings1[0],sizeof(double),numtasks,doswap);

	freaddata(&fields.stimings2[0],(long) numtasks * sizeof(double),fp);
	sion_swap(&fields.stimings2[0],&fields.stimings2[0],sizeof(double),numtasks,doswap);

	freaddata(&fields.fromlist[0],(long) numtasks * sizeof(sion_int32),fp);
	sion_swap(&fields.fromlist[0],&fields.fromlist[0],sizeof(sion_int32),numtasks,doswap);

	freaddata(&fields.tolist[0],(long) numtasks * sizeof(sion_int32),fp);
	sion_swap(&fields.tolist[0],&fields.tolist[0],sizeof(sion_int32),numtasks,doswap);

#ifdef FEHLERINDATEN
	freaddata(&fromlist[0],(long) numtasks * sizeof(sion_int32),fp);
	freaddata(&tolist[0],(long) numtasks * sizeof(sion_int32),fp);
#endif
      } 

    } /* p */

  } /* rank */


  /* STEP: report timings outside valid range (min,max) */
  printf("pingponganalysis: timings_min:          %e s %10.6f MB/s\n",fields.timings_min,time2mbs(fields.timings_min));
  printf("pingponganalysis: timings_max:          %e s %10.6f MB/s\n",fields.timings_max,time2mbs(fields.timings_max));
  for(t=0;t<minentriesfound;t++) {
    printf("pingponganalysis:  suspected link #%05d: %6d <-> %6d: %e s %16.10f MB/s (%s <-> %s)\n",t,
           minentriesX[t],minentriesY[t],minentriesVAL[t],time2mbs(minentriesVAL[t]),
           fields.alllocation+minentriesX[t]*MAX_LOCATION_LENGTH,fields.alllocation+minentriesY[t]*MAX_LOCATION_LENGTH);
    
  }

  for(t=0;t<maxentriesfound;t++) {
    printf("pingponganalysis:  FAST      link #%05d: %6d <-> %6d: %e s %16.10f MB/s (%s <-> %s)\n",t,
           maxentriesX[t],maxentriesY[t],maxentriesVAL[t],time2mbs(maxentriesVAL[t]),
           fields.alllocation+maxentriesX[t]*MAX_LOCATION_LENGTH,fields.alllocation+maxentriesY[t]*MAX_LOCATION_LENGTH);
    
  }

  /* STEP: write some histograms */
  stepfile1=fopen("distribution.dat","w");
  for(t=0;t<opt_steps;t++) fprintf(stepfile1,"%5d %10e %10ld\n",t,t*stepwidth+ptimings_min,
                                   fields.ptimings_stepcnt[t] ) ;
  fclose(stepfile1);
  printf(" wrote %s\n","distribution.dat");

  stepfile1=fopen("mapping.dat","w");
  for(t=0;t<numtasks;t++) fprintf(stepfile1,"%6d %4d %4d %4d %4d %s\n",t,fields.mapping_x[t],fields.mapping_y[t],
				  fields.mapping_z[t],fields.mapping_t[t],fields.alllocation+t*MAX_LOCATION_LENGTH);
  fclose(stepfile1);
  printf(" wrote %s\n","mapping.dat");

  stepfile2=fopen("distribution_bw.dat","w");
  for(t=0;t<opt_steps;t++) fprintf(stepfile2,"%5d %10e %10ld\n",t,time2mbs(t*stepwidth+ptimings_min),
                                   fields.ptimings_stepcnt[t] ) ;
  fclose(stepfile2);
  printf(" wrote %s\n","distribution_bw.dat");

  if(opt_dopsreport) {
    PSplothist(psreportfile,fields.ptimings_stepcnt,opt_steps,ptimings_min,ptimings_max,stepwidth);
  }

  if(opt_dobadlinkreport) {
    stepfile1=fopen("bad_links.dat","w");
    fprintf(stepfile1,"# %5s %5s %5s %16s %16s %16s %16s\n","Nr.","From","To","1st (s)","1st (MB/s)","2nd (s)","2nd (MB/s)");
    for(t=0;t<numtasks;t++) {
      fprintf(stepfile1," %5d %5d %5d %16.8e %16.10f %16.8e %16.10f %s -> %s\n",t,
	      fields.fromlist[t],fields.tolist[t],fields.stimings2[t],time2mbs(fields.stimings2[t]),fields.stimings1[t],time2mbs(fields.stimings1[t]),
	      fields.alllocation+fields.fromlist[t]*MAX_LOCATION_LENGTH,fields.alllocation+fields.tolist[t]*MAX_LOCATION_LENGTH);
    }
    fclose(stepfile1);
    printf(" wrote %s\n","bad_links.dat");
  }

  if(opt_donodelist) {
    stepfile1=fopen("nodelist.dat","w");
    fprintf(stepfile1,"# %5s %15s %25s %25s %25s %25s %10s %10s\n","Nr.","Host",
	    "from_avg_bw (MB/s)","to_avg_bw (MB/s)",
	    "from_min_bw (MB/s)","to_min_bw (MB/s)",
	    "fromcntmax<2000","tocntmax<2000");
    for(t=0;t<numtasks;t++) {
      fprintf(stepfile1," %5d %15s %25.10f %25.10f %25.10f %25.10f %10d %10d\n",t,
	      fields.alllocation+t*MAX_LOCATION_LENGTH,
	      time2mbs(fields.nodeslist_fromavg[t]/(numtasks-1.0)),
	      time2mbs(fields.nodeslist_toavg[t]/(numtasks-1.0)),
	      time2mbs(fields.nodeslist_frommax[t]),
	      time2mbs(fields.nodeslist_tomax[t]),
	      fields.nodeslist_fromcntmax[t],
	      fields.nodeslist_tocntmax[t]
	      );
    }
    fclose(stepfile1);
    printf(" wrote %s\n","nodelist.dat");
  }

  /* STEP: close all files */
  sion_close(sid);
  printf(" after sion_close\n");
  if(opt_doaccesspatternppm) {
    ppmclose(outfile_ppm);
    printf(" after close outfile_ppm\n");
  }
  if(opt_dodistpattern) {
    fclose(outdistfile);
    printf(" after close outdistfile\n");
  }
  if(opt_dobwpatternppm) {
    ppmclose(outbwfile_ppm);
    printf(" after close outdistfile_ppm\n");
  }
  if(opt_doaccesspatternascii) {
    fclose(accessfile);
    printf(" after close accessfile\n");
  }
  if(opt_dognuplotfile) {
    fclose(gnuplotfile);
    printf(" after close gnuplotfile\n");
  }
  if(opt_dopsreport) {
    PSfooter(psreportfile);
    fclose(psreportfile);
    printf(" after close psreportfile\n");
  }
  
  return(0);
}


int PSheader(FILE *OUT) {
  fprintf( OUT, "%%!PS-Adobe-2.0 EPSF-2.0\n");
  fprintf( OUT, "%%%%Title: Communication analysis FZJ Linktest \n");
  fprintf( OUT, "%%%%Creator: fzjlinktest Version 1.0, Forschungszentrum Juelich, JSC\n");
  fprintf( OUT, "%%%%DocumentFonts: Times-Roman, Times-Bold Courier-Bold\n");
  fprintf( OUT, "%%%%BoundingBox: 0 0 610 790\n");
  fprintf( OUT, "%%%%Orientation: Portrait\n");
  fprintf( OUT, "%%%%EndComments\n");
/*   fprintf( OUT, "40 60 translate\n"); */
/*   fprintf( OUT, "0.45 0.56 scale\n"); */
  fprintf( OUT, "/fr {\n");
  fprintf( OUT, "  /col exch def /lw exch def /y2 exch def /x2 exch def /y1 exch def /x1 exch def \n");
  fprintf( OUT, "  gsave lw setlinewidth col 0.8 0.8 sethsbcolor x1 y1 moveto x2 y1 lineto\n");
  fprintf( OUT, "        x2 y2 lineto x1 y2 lineto closepath fill\n");
  fprintf( OUT, "  grestore\n");
  fprintf( OUT, "} bind def\n");

  fprintf( OUT, "/frectgrey {\n");
  fprintf( OUT, "  /col exch def /lw exch def /y2 exch def /x2 exch def /y1 exch def /x1 exch def \n");
  fprintf( OUT, "  gsave lw setlinewidth col col col setrgbcolor x1 y1 moveto x2 y1 lineto\n");
  fprintf( OUT, "        x2 y2 lineto x1 y2 lineto closepath fill\n");
  fprintf( OUT, "  grestore\n");
  fprintf( OUT, "} bind def\n");

  fprintf( OUT, "/rect {\n");
  fprintf( OUT, "  /col exch def /lw exch def /y2 exch def /x2 exch def /y1 exch def /x1 exch def \n");
  fprintf( OUT, "  gsave lw setlinewidth col 0.8 0.8 sethsbcolor x1 y1 moveto x2 y1 lineto\n");
  fprintf( OUT, "        x2 y2 lineto x1 y2 lineto closepath stroke\n");
  fprintf( OUT, "  grestore\n");
  fprintf( OUT, "} bind def\n");

  fprintf( OUT, "/rightText {				%% (str) rightText -\n");
  fprintf( OUT, "  	dup stringwidth	        	%% (str) wx wy\n");
  fprintf( OUT, "  	pop     			%% (str) wx\n");
  fprintf( OUT, "  	neg 0 rmoveto			%% (str) \n");
  fprintf( OUT, "  	show\n");
  fprintf( OUT, "} def\n");

  return(1);
}

int PSfooter(FILE *OUT) {
  fprintf( OUT, "%% Footer \n");
  fprintf(OUT, "gsave\n");
  fprintf(OUT, "/Times-Bold findfont 6 scalefont setfont 0 0 0 setrgbcolor\n");
  fprintf(OUT, "340 20 moveto\n");
  fprintf(OUT, "(Report generated by FZJ Linktest Result Analyzer, Forschungszentrum Juelich GmbH) show\n");
  fprintf(OUT, "grestore\n");
  fprintf( OUT, "showpage \n");
   
  return(1);
}

double nval2col (double val) {
  double col=1.0-(0.2+0.8*val); 
/*   double col=0.2+0.8*val;  */
/*     double col=1.0-0.9*val; */
    return(col);
}



int PSplotscala (FILE *OUT, int nelem, double minval, double maxval) {
  int t;
  double col,rval,val;
  double posx,posy,bheight,bwidth;
  char helpstr[256];
  bheight=(double) PSSCALEH/nelem;
  bwidth =(double) PSSCALEW;
  posx=PSSCALEX;
  for(t=0;t<nelem;t++) {
    posy=PSSCALEY+PSSCALEH-(t+1)*bheight;
    rval=minval+(maxval-minval)*(double) t/ (double) nelem;
    val=(rval-minval)/(maxval-minval);
    col=nval2col(val);
    frect(OUT,posx,posy,bwidth,bheight,0,col);
    rect(OUT,posx,posy,bwidth,bheight,0.1,0.1);
    sprintf(helpstr,"% 7.1fus (%7.2f MB/s)",rval*1000*1000,time2mbs(rval));
    text(OUT,helpstr,posx+bwidth+1,posy+2,bheight);
  }
  return(1);
}

int PSplothist (FILE *OUT, long *count, int numsteps, double minval, double maxval, double stepwidth) {
  int s;
  long c,maxcount;
  double maxcountr,cr;
  double col,val,rval;
  double posx,posy,bheight,bwidth;
  char helpstr[256];

  bwidth =(double) PSHISTW/ (double)numsteps;
  maxcount=0;
  for(s=0;s<numsteps;s++) {
    if(count[s]>maxcount) maxcount=count[s];
  }
  if(maxcount==0) return(0);

  rect(OUT,PSHISTX,PSHISTY,PSHISTW,PSHISTH,0.1,0.1);

  maxcountr=log(maxcount);
  posy=PSHISTY;

  for(c=1;c<maxcount;c*=2) {
      bheight=log((double) c)/maxcountr * PSHISTH;
      sprintf(helpstr,"% 7d",(int) c);
      text(OUT,helpstr,PSHISTX-40,posy+bheight,8);
      line(OUT,PSHISTX,posy+bheight,PSHISTX+PSHISTW,posy+bheight,0.1,0.1);
  }
  for(s=0;s<=10;s++) {
    posx=PSHISTX+PSHISTW-(s+0.0)*PSHISTW/10.0;
    rval=minval+(maxval-minval)*(double) s/ (double) 10.0;
    sprintf(helpstr,"%7.1fus",rval*1000*1000);
    text(OUT,helpstr,posx-PSHISTW/10*0.4,posy-10,6);
    sprintf(helpstr,"%7.1fMB/s",time2mbs(rval));
    text(OUT,helpstr,posx-PSHISTW/10*0.4,posy-15,6);
    line(OUT,posx,PSHISTY,posx,PSHISTY-5,0.1,0.1);
  }

  for(s=0;s<numsteps;s++) {
    posx=PSHISTX+PSHISTW-((double)s+1.0)*bwidth;
    rval=minval+(maxval-minval)*(double) s/ (double) numsteps; 
/*     rval=minval+s*stepwidth; */
    val=(rval-minval)/(maxval-minval);
    col=nval2col(val);
    if(count[s]>0) {
      bheight=log((double) count[s])/maxcountr * PSHISTH;
      frect(OUT,posx,posy,bwidth,bheight,0.1,col);
    }
  }

  return(1);
}

int frect (FILE *OUT, double x1, double y1, double w, double h, double lw, double color ) {
  double x2=x1+w;
  double y2=y1+h;
  fprintf(OUT, "%6.2f %6.2f %6.2f %6.2f %6.2f %5.3f fr\n",x1,y1,x2,y2,lw,color);
  return(1);
}

int frectgrey (FILE *OUT, double x1, double y1, double w, double h, double lw, double color ) {
  double x2=x1+w;
  double y2=y1+h;
  fprintf(OUT, "%f %f %f %f %f %f frectgrey\n",x1,y1,x2,y2,lw,color);
  return(1);
}

int rect (FILE *OUT, double x1, double y1, double w, double h, double lw, double color ) {
  double x2=x1+w;
  double y2=y1+h;
  fprintf(OUT, "%f %f %f %f %f %f rect\n",x1,y1,x2,y2,lw,color);
  return(1);
}

int line(FILE *OUT, double x1, double y1, double x2, double y2, double lw, double color ) {
  fprintf(OUT, "gsave %f setlinewidth %f 0.8 0.8 sethsbcolor %f %f moveto %f %f lineto stroke grestore\n",lw,color,x1,y1,x2,y2);
  return(1);
}

int text (FILE *OUT, char *strtext, double x, double y, double size) {
  fprintf(OUT, "gsave\n");
  fprintf(OUT, "/Courier-Bold findfont %f scalefont setfont 0 0 0 setrgbcolor\n",size);
  fprintf(OUT, "%f %f moveto\n",x,y);
  fprintf(OUT, "(%s) show\n",strtext);
  fprintf(OUT, "grestore\n");
  return(1);
}

int righttext (FILE *OUT, char *strtext, double x, double y, double size) {
  fprintf(OUT, "gsave\n");
  fprintf(OUT, "/Courier-Bold findfont %f scalefont setfont 0 0 0 setrgbcolor\n",size);
  fprintf(OUT, "%f %f moveto\n",x,y);
  fprintf(OUT, "(%s) rightText\n",strtext);
  fprintf(OUT, "grestore\n");
  return(1);
}

int textr (FILE *OUT, char *strtext, double x, double y, double size) {
  fprintf(OUT, "gsave\n");
  fprintf(OUT, "/Times-Bold findfont %f scalefont setfont 0 0 0 setrgbcolor\n",size);
  fprintf(OUT, "%f %f moveto\n",x,y);
  fprintf(OUT, "(%s) show\n",strtext);
  fprintf(OUT, "grestore\n");
  return(1);
}

int textrot (FILE *OUT, char *strtext, double x, double y, double size) {
  fprintf(OUT, "gsave\n");
  fprintf(OUT, "/Courier-Bold findfont %f scalefont setfont 0 0 0 setrgbcolor\n",size);
  fprintf(OUT, "%f %f moveto\n",x,y);
  fprintf(OUT, "gsave\n");
  fprintf(OUT, "90 rotate\n");
  fprintf(OUT, "(%s) show\n",strtext);
  fprintf(OUT, "grestore\n");
  fprintf(OUT, "grestore\n");
  return(1);
}

int righttextrot (FILE *OUT, char *strtext, double x, double y, double size) {
  fprintf(OUT, "gsave\n");
  fprintf(OUT, "/Courier-Bold findfont %f scalefont setfont 0 0 0 setrgbcolor\n",size);
  fprintf(OUT, "%f %f moveto\n",x,y);
  fprintf(OUT, "gsave\n");
  fprintf(OUT, "90 rotate\n");
  fprintf(OUT, "(%s) rightText\n",strtext);
  fprintf(OUT, "grestore\n");
  fprintf(OUT, "grestore\n");
  return(1);
}
