/****************************************************************************
**  LinkTest                                                               **
*****************************************************************************
**  Copyright (c) 2008-2010                                                **
**  Forschungszentrum Juelich, Juelich Supercomputing Centre               **
**                                                                         **
**  See the file COPYRIGHT in the package base directory for details       **
****************************************************************************/
#include <mpi.h>
#include <stdio.h>
#include <stdlib.h>

#include <sys/times.h>
#include <sys/resource.h>

#include "mapping.h"
#include "limprioqueue.h"

#define USESION
#ifdef USESION
#include "sion.h"
#endif

#define VERSION  3

#define FNAMELEN 255

#define LEVEL_us  1000 
#define LEVEL_mbs 100 
#define proc_A 0
#define proc_B 1
#define PING 100
#define PONG 101
#define WAKEUP 102
#define PTIMINGS 103
#define LOCATION 104
#define MAPPING  105
#define ACCESS   106
#define ATIMINGS 107
#define number_of_messages_default 3
#define length_of_message_max   (1*1024*1024) 
#define length_of_message_a2a   (72*4*1024) 
#define length_of_message_default   (128*1024) 

int number_of_messages = 3;
int length_of_message = (128*1024);

double avg_time_all=0.0, min_time_all=0.0, max_time_all=0.0;
double a2a_avg_time_all=0.0, a2a_min_time_all=0.0, a2a_max_time_all=0.0;
double *ptimings=NULL;
double *stimings=NULL;
double *atimings=NULL;
int        *pepartner=NULL;
int        *taskvec=NULL;
#ifdef USESION
sion_int32 *accesspattern=NULL;
#endif

int work ( int my_rank, int step, int do_alltoall, int do_serial, int sleeptime, 
	   int size, int *pepartner, lim_queue *lqueue, int outerloop);
int getpartnerplan_pe (int penr, int n, int *pepartner, int mix, int my_rank);
int getrandomize_vector_search (int n, int *vec);
int getrandomize_vector_swap (int n, int *vec);
int getpartneropt (int penr, int n, int *vec, int *pepartner);

struct rusage rusagestr;
int print_memusage(int i);

double time2mbs( double time) {
  if(time>0) {
    return(length_of_message*sizeof(char)/time/1024/1024);
  } else {
    return(0.0);
  }
}


double alltoall (	      int my_rank,
			      int tasks,
			      int iterations
			      ) {
  double timeused=0.0;
  char in[length_of_message_a2a];
  char out[length_of_message_a2a];
  int i,maxlen;   
  double start, finish, time;
/*   maxlen=length_of_message_a2a/tasks; */
  maxlen=1;
  for(i=0;i<tasks*maxlen;i++)  out[i] = my_rank%256;
/*   if(my_rank<10) { */
/*     printf("starting MPI_Alltoall: maxlen=%d tasks=%d\n",maxlen,tasks); */
/*   } */
  start = MPI_Wtime();
  MPI_Alltoall(out,maxlen,MPI_BYTE,in,maxlen,MPI_BYTE,MPI_COMM_WORLD);
  finish = MPI_Wtime();
  
  time = finish - start;
  timeused=(float)(time);
  /* '!!! ckeck results !!! */
  return(timeused);
}

double pingpong (
	      int from,
	      int to,
	      int my_rank,
	      int iterations
	      ) {
  double timeused=0.0;
  char buffer[length_of_message_max];
  int i;   
  double start, finish, time;
  MPI_Status status;

  /* printf("pingpong on %d:  from: %4d To: %4d iterations=%d\n",my_rank,from,to,iterations); */

  for (i = 1; i <= length_of_message; i++) buffer[i]=(char) i%256;
  
  if (from==to) return(0.0);

/*   MPI_Barrier(MPI_COMM_WORLD);  */
  if (my_rank == from) 
  {

    start = MPI_Wtime();
     
    for (i = 1; i <= iterations; i++)
    {
      MPI_Send(buffer, length_of_message, MPI_BYTE, to, PING,
	       MPI_COMM_WORLD);

      MPI_Recv(buffer, length_of_message, MPI_BYTE, to, PONG,
	       MPI_COMM_WORLD, &status);
    }

    finish = MPI_Wtime();

    time = finish - start;
    timeused=(float)(time / (2 * iterations));
  }
  else if (my_rank == to) 
  {
    for (i = 1; i <= iterations; i++)
    { 
      MPI_Recv(buffer, length_of_message, MPI_BYTE, from, PING,
	       MPI_COMM_WORLD, &status);

      MPI_Send(buffer, length_of_message, MPI_BYTE, from, PONG,
	       MPI_COMM_WORLD);
    }
  }
/*   MPI_Barrier(MPI_COMM_WORLD);  */

/*   printf("pingpong on %d:  from: %4d To: %4d ready \n",my_rank,from,to); */

  return(timeused);
} 


void usage(char *name) {
  fprintf(stderr, "Usage: %s options\n\nwith the following optional options (default values in parathesis):\n\n",name);

  fprintf(stderr, "  [-a 0|1]                  do alltoall mode (0 or 1)             (0) \n");
  fprintf(stderr, "  [-i <iterations>]         number of pingpong iterations         (3) \n");
  fprintf(stderr, "  [-s <size>]               message size in Bytes            (131072) \n");
  fprintf(stderr, "  [-k <size>]               message size in KBytes             (128k) \n");
  fprintf(stderr, "  [-T <min>]                minmal runtime in min                (-1) \n");
  fprintf(stderr, "  [-M 0|1]                  randomized processor numbers          (0) \n");
  fprintf(stderr, "  [-S 0|1]                  do searialized test                  (no) \n");
  fprintf(stderr, "  [-W 0|1]                  write full protocol                   (1) \n");
  fprintf(stderr, "  [-C <num>]                number of collected tasks for output (32) \n");
  exit(1);
}

int main(int argc, char *argv[])
{
  int my_rank,size,grank;
  int t;
  double starttime,timedelay,globalstarttime;
  double starttimewrk,timedelaywrk,wrktime;
  double starttimestep,timedelaystep,steptime;
  int i,p, sid;
  int do_alltoall,sleeptime, do_mix,collectpnum=32,do_serial=0;
  int do_sion;
  double     time, control_wtime;
  int        dummy, step;
  FILE      *fp;
  char       filename_bin[256]="pingpong_results_bin.sion";
  MPI_Status status;
  MPI_Comm   comm_local;
#ifdef USESION
  sion_int64 sion_chunksize;
  sion_int32 sion_fsblksize;
  sion_int32 mappingout[4];
  sion_int32 statdata_int[10];
  int        sion_numfiles;
#endif
  char      *newfname;
  int        left,bwrote;
  char      *ptr,*location;
  int       *mapping, inioset;
  MPI_Comm   iocomm;
  double     statdata_dp[10];
  int        restart_loop, outerloop; 

  /* for searching slowest pairs */
  lim_queue    lqueue;
  lim_queue    lqueue_help;
  lim_queue   *sendq, *recvq;
  MPI_Datatype node_dt;
  node         testnode[2];
  MPI_Datatype type[4] = {MPI_DOUBLE,MPI_INT,MPI_INT,MPI_UB};
  MPI_Aint     disp[4];
  int          blocklen[4]={1,1,1,1};
  MPI_Aint     base;
  

  MPI_Init(&argc, &argv);
  MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
  MPI_Comm_size(MPI_COMM_WORLD, &size);

  do_alltoall=   0;
  sleeptime=     0;
  do_mix=        0;
  do_sion=       1;
  control_wtime=-1.0;

  if(my_rank<=3) {  
    print_memusage(my_rank);
  }

  if(my_rank==0) {
    /* parse command line */
    i=1;
    while( i < argc ) {
      if( argv[i][0] == '-' ) {
        switch( argv[i][1] ) {
        case 'a':
          do_alltoall = atoi(argv[++i]);
          break;
        case 'M':
          do_mix = atoi(argv[++i]);
          break;
	  /*         case 'm': */
	  /*           min_bandwidth = (float) atof(argv[++i]); */
	  /*           break; */
        case 'i':
          number_of_messages = atoi(argv[++i]);
          break;
        case 'D':
          sleeptime = atoi(argv[++i]);
          break;
        case 'S':
          do_serial = atoi(argv[++i]);
          break;
        case 'T':
          control_wtime= atof(argv[++i]) * 60.0;
          break; 
        case 'C':
          collectpnum = atoi(argv[++i]);
          break; 
        case 'W':
          do_sion = atoi(argv[++i]);
          break;
	case 's':
          length_of_message = atoi(argv[++i]);
          if(length_of_message>length_of_message_max) {
            fprintf(stderr,"Message %d size reduced to %d\n",length_of_message,length_of_message);
            length_of_message=length_of_message_max;
          }
          break;
        case 'k':
          length_of_message = atoi(argv[++i]);
          length_of_message*=1024;
          if(length_of_message>length_of_message_max) {
            fprintf(stderr,"Message %d size reduced to %d\n",length_of_message,length_of_message);
            length_of_message=length_of_message_max;
          }
          break;
        default:
          usage(argv[0]);
        }
      } else {
        usage(argv[0]);
      }
      i++;
    }
  }

  MPI_Bcast(&do_alltoall,1, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(&do_mix, 1, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(&do_serial, 1, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(&do_sion, 1, MPI_INT, 0, MPI_COMM_WORLD);
  /*   MPI_Bcast(&min_bandwidth, 1, MPI_FLOAT, 0, MPI_COMM_WORLD); */
  MPI_Bcast(&sleeptime, 1, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(&collectpnum, 1, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(&number_of_messages, 1, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(&length_of_message, 1, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(&control_wtime, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD);

  /*                                    */  globalstarttime = starttime = MPI_Wtime();
  MPI_Barrier(MPI_COMM_WORLD);
  /*                                    */  timedelay = MPI_Wtime()-starttime;
  if(my_rank==0) printf("timings[%03d] %-30s t=%10.6fs\n",my_rank,"[first sync]",timedelay);

  if(my_rank==0) { 
    printf("\n\n");
    printf("----------------------------------------------------------\n");
    printf("mpilinktest: Number of MPI-Task:        %10d\n",size);
    printf("mpilinktest: Message length:            %10d Bytes\n",(int) (length_of_message*sizeof(char)));
    printf("mpilinktest: Message length:            %10.4f KBytes\n",(float) length_of_message*sizeof(char)/1024);
    printf("mpilinktest: Number of Iteration:       %10d\n",number_of_messages);
    /*     printf("mpilinktest: Level for suspected Links: <%9f MB/s\n",min_bandwidth); */
    printf("mpilinktest: alltoall:                  %d\n",do_alltoall);
    printf("mpilinktest: sleeptime:                 %d us\n",sleeptime);
    printf("mpilinktest: collectpnum:               %d\n",collectpnum);
    printf("mpilinktest: mixing pe order:           %d\n",do_mix);
    printf("mpilinktest: serial test:               %d\n",do_serial);
    printf("mpilinktest: control wtime:             %f\n",control_wtime);
    printf("----------------------------------------------------------\n");
  } 

  
  /***********************************************************************************/
  /* Allocate and initialize buffers                                                 */
  /***********************************************************************************/
  ptimings = (double *) malloc(sizeof(double)*size);
  if (ptimings == NULL) {
    fprintf(stderr, "cannot allocate ptimings[%lu] ...\n", (unsigned long) sizeof(double)*size);
    MPI_Abort(MPI_COMM_WORLD,1);
  }  
  for(i=0;i<size;i++) ptimings[i]=0.0;
  stimings = (double *) malloc(sizeof(double)*size);
  if (stimings == NULL) {
    fprintf(stderr, "cannot allocate stimings[%lu] ...\n", (unsigned long) sizeof(double)*size);
    MPI_Abort(MPI_COMM_WORLD,1);
  }
  for(i=0;i<size;i++) stimings[i]=0.0;
  atimings = (double *) malloc(sizeof(double)*size);
  if (stimings == NULL) {
    fprintf(stderr, "cannot allocate atimings[%lu] ...\n", (unsigned long) sizeof(double)*size);
    MPI_Abort(MPI_COMM_WORLD,1);
  }
  for(i=0;i<size;i++) atimings[i]=0.0;
#ifdef USESION
  accesspattern = (sion_int32 *) malloc(sizeof(sion_int32)*size);
  if (stimings == NULL) {
    fprintf(stderr, "cannot allocate accesspattern[%lu] ...\n", (unsigned long) sizeof(sion_int32)*size);
    MPI_Abort(MPI_COMM_WORLD,1);
  }
#endif
  taskvec = (int *) malloc(sizeof(int)*size);
  if (taskvec == NULL) {
    fprintf(stderr, "cannot allocate taskvec[%lu] ...\n", (unsigned long) sizeof(int)*size);
    MPI_Abort(MPI_COMM_WORLD,1);
  }
  pepartner = (int *) malloc(sizeof(int)*size);
  if (pepartner == NULL) {
    fprintf(stderr, "cannot allocate pepartner[%lu] ...\n", (unsigned long) sizeof(int)*size);
    MPI_Abort(MPI_COMM_WORLD,1);
  }


  /***********************************************************************************/
  /* Write Mapping                                                                   */
  /***********************************************************************************/
  /*                                    */  starttime = MPI_Wtime();
  get_mapping(&location,&mapping);
  printf("task[%06d] on %s\n", my_rank,location);

  /*                                    */  timedelay = MPI_Wtime()-starttime;
  MPI_Barrier(MPI_COMM_WORLD);
  if(my_rank==0) printf("timings[%03d] %-30s t=%10.6fs\n",my_rank,"[mapping]",timedelay);


  /***********************************************************************************/
  /* PRETEST                                                                         */
  /***********************************************************************************/
#ifdef PRETEST
  /*                                    */  starttime = MPI_Wtime();
  if(my_rank==0) {
    printf("\n\nPRE-test if all links are up:\n");
    printf("-----------------------------\n");
  }  

  for(t=1;t<size;t++) {
    if(my_rank==0) {
      printf(" Task %3d to Task %3d:",0,t);
      fflush(stdout);
    }
    time=pingpong(0,t,my_rank,3);
    if(my_rank==0) {
      printf(" %10.3fus (%7.2f MB/s)\n",time*1000*1000,time2mbs(time));
    }
  }
  MPI_Barrier(MPI_COMM_WORLD);
  /*                                    */  timedelay = MPI_Wtime()-starttime;
  if(my_rank==0) printf("timings[%03d] %-30s t=%10.6fs\n",my_rank,"[pretest]",timedelay);
#endif

  if(my_rank==0) {
    printf("\n\nStarting Test of all connections:\n");
    printf("--------------------------------\n");
    min_time_all=1e20;
    max_time_all=-1e20;
    avg_time_all=0;
    a2a_min_time_all=1e20;
    a2a_max_time_all=-1e20;
    a2a_avg_time_all=0;
  }  

  for(t=0;t<size;t++) ptimings[t]=0.0;  


  /***********************************************************************************/
  /* Init partnerplan                                                                */
  /***********************************************************************************/

  /*                                    */  starttime = MPI_Wtime();
  if(do_mix) {
    getrandomize_vector_swap(size,taskvec);
  } else {
    for(i=0;i<size;i++) {taskvec[i]=i;}
  }
  /*                                    */  timedelay = MPI_Wtime()-starttime;
  if(my_rank==0) printf("timings[%03d] %-30s t=%10.6fs\n",my_rank,"[randvec]",timedelay);

  /*                                    */  starttime = MPI_Wtime();
  getpartneropt(my_rank,size,taskvec,pepartner);
  /*                                    */  timedelay = MPI_Wtime()-starttime;
  if(my_rank==0) printf("timings[%03d] %-30s t=%10.6fs\n",my_rank,"[getpart]",timedelay);

  /* getpartnerplan_pe(my_rank,size,pepartner,do_mix,my_rank);  */
  /* pepartner contains for each step number of communication (from -> to) partner:
     pepartner[step]<0   rank is 'to'   part, 'from'-rank is  -pepartner[step]-1 
     pepartner[step]>0   rank is 'from' part, 'to'-rank   is   pepartner[step]-1 
  */

  /* check partner list */
  MPI_Barrier(MPI_COMM_WORLD);
  {
    int psum=0,pasum=0;
    for(t=0;t<size;t++) {
      psum+=pepartner[t];
      if(pepartner[t]<0)  pasum+= -pepartner[t];
      else                pasum+=  pepartner[t];
    }
    if(my_rank%1000==0) {
      printf("PE%05d: psum=%d pasum=%d do_mix=%d\n",my_rank,psum,pasum,do_mix);
    }
  }
  if(my_rank==0)  printf(" ... after getpartnerplan_pe\n"); 


  /* print partner list */
  if(0) {
    for(t=0;t<size;t++) { 
      MPI_Barrier(MPI_COMM_WORLD);
      if(t==my_rank) {
	printf("partners of %2d: ",my_rank);
	for(i=0;i<size;i++) {
	  if(pepartner[i]<0)  printf("(%3d->   ) ",-pepartner[i]-1);
	  else                printf("[   ->%3d] ",pepartner[i]-1);
	}
	printf("\n");
      }
    }

  }

  if(my_rank<=0) {  
    print_memusage(my_rank);
  }

  /***********************************************************************************/
  /* Work                                                                            */
  /***********************************************************************************/
  /*                                    */  starttime = MPI_Wtime();

  limqueue_initalize(&lqueue, size);
  limqueue_initalize(&lqueue_help, size);

  restart_loop=1;
  outerloop=0;
  wrktime=steptime=0.0;

  while(restart_loop) {
  
    /*                                    */  starttimestep = MPI_Wtime();
    for(step=1;step<size;step++) {
      /*                                    */  starttimewrk = MPI_Wtime();
      work(my_rank, step, do_alltoall, do_serial, sleeptime, size, pepartner, &lqueue_help, outerloop);
      /*                                    */  timedelaywrk = MPI_Wtime()-starttimewrk;
      wrktime+=timedelaywrk;

      if((my_rank==0) && (step%collectpnum==0)) {
	
	printf("analyse summary: min. %11.6fus (%7.2f MB/s) max. %11.6fus (%7.2f MB/s) avg. %11.6fus (%7.2f MB/s)\n",
	       min_time_all*1000*1000,time2mbs(min_time_all),
	       max_time_all*1000*1000,time2mbs(max_time_all),
	       avg_time_all/step*1000*1000,time2mbs(avg_time_all/step));
	printf("timing summary:  %d steps used %10.6fs (%10.6fs/step),        to do: %d steps,                   estimate time: %10.6fs\n",
	       step,wrktime,wrktime/step,
	       size-step,wrktime/step*(size-step)
	       );
	printf("\n");
	
	
      }
    }
    /*                                    */  timedelaystep = MPI_Wtime()-starttimestep;
    steptime+=timedelaystep;
    MPI_Barrier(MPI_COMM_WORLD);
    /*                                    */  timedelay = MPI_Wtime()-starttime;
    if(my_rank==0) printf("timings[%03d] %-30s t=%10.6fs\n",my_rank,"[work]",timedelay);
    
    /***********************************************************************************/
    /* Average results                                                                 */
    /***********************************************************************************/
    if(my_rank==0) {
      avg_time_all/=(size-1);
      a2a_avg_time_all/=(size-1);
      printf("\n\n");
      printf("RESULT: Min Time:            %20.8fus (%7.2f MB/s)\n",min_time_all*1000*1000,time2mbs(min_time_all));
      printf("RESULT: Max Time:            %20.8fus (%7.2f MB/s)\n",max_time_all*1000*1000,time2mbs(max_time_all));
      printf("RESULT: Avg Time:            %20.8fus (%7.2f MB/s)\n",avg_time_all*1000*1000,time2mbs(avg_time_all));
    }
    MPI_Barrier(MPI_COMM_WORLD);
    
    
    /***********************************************************************************/
    /* searching slowest pairs                                                         */
    /***********************************************************************************/
    /*                                    */  starttime = MPI_Wtime();
    limqueue_sort(&lqueue_help);
    limqueue_merge_in(&lqueue,&lqueue_help);
    limqueue_reinitalize(&lqueue_help);

    outerloop++;
    /* check if test should run again */
    if(my_rank==0) {
      if(control_wtime>0) {
	if( (steptime + timedelaystep) < control_wtime) restart_loop=1;
	else                                            restart_loop=0;
      } else {
	restart_loop=0;
      }
      if(restart_loop==1) {
	fprintf(stderr,"LINKTEST[%2d]: end test not reached: control_wtime=%10.2fs, elapsed time=%10.4fs, estimate step time=%10.4fs\n",
		outerloop,control_wtime,steptime,timedelaystep);fflush(stderr);
	fprintf(stdout,"LINKTEST[%2d]: end test not reached: control_wtime=%10.2fs, elapsed time=%10.4fs, estimate step time=%10.4fs\n",
		outerloop,control_wtime,steptime,timedelaystep);fflush(stdout);
      }
    }
    MPI_Bcast(&restart_loop,1,MPI_INT,0,MPI_COMM_WORLD);
    
  }
  
  if(outerloop>0) {
    for(i=0;i<size;i++) atimings[i]/=outerloop;
    for(i=0;i<size;i++) ptimings[i]/=outerloop;
  }


  /*   limqueue_print(&lqueue);  */
  /*   limqueue_print(&lqueue_help);  */

  /* preparing MPI datatype for lim_queue */
  {
    MPI_Address(testnode,disp);
    MPI_Address(&testnode[0].from,disp+1);
    MPI_Address(&testnode[0].to,  disp+2);
    MPI_Address(testnode+1,       disp+3);
    base=disp[0];
    for(i=0;i<4;i++) disp[i]-=base;
    if(my_rank==0) {
      printf("PE[%d] disp %d %d %d %d base=%10ld sizeof(node)=%d sizeof(MPI_Aint)=%d\n",my_rank,
	     (int) disp[0],(int) disp[1],(int) disp[2],(int) disp[3],(long) base,(int) sizeof(node),(int) sizeof(MPI_Aint));
    }
    fflush(stdout);
    MPI_Type_struct(4,blocklen,disp,type,&node_dt);
    MPI_Type_commit(&node_dt);
    MPI_Barrier(MPI_COMM_WORLD);
  }  

  /* merging  */
  {
    int stride1=1;
    int stride2=2;
    int sender,receiver;
    MPI_Status status;


    sendq=&lqueue;
    recvq=&lqueue_help;

    while(stride1<size) {
      if(my_rank==0) printf(" %03d: starting collect for stride  %6d\n",my_rank,stride1);
      if( (my_rank%stride2) == 0) { /* receive data */
	sender=my_rank+stride1;
	if(sender<size) {
	  /* 	  printf(" %03d: RECV from %03d %d el\n",my_rank,sender,recvq->max_elems);fflush(stdout); */
	  MPI_Recv(recvq->elements,recvq->max_elems,node_dt,sender,1001,MPI_COMM_WORLD,&status);
	  MPI_Get_count(&status,node_dt,&recvq->qsize);
	  /* 	  printf(" %03d: RECV from %03d ready %d el\n",my_rank,sender,recvq->qsize);fflush(stdout); */
	  limqueue_merge_in(sendq,recvq);
	}
      }
      if( (my_rank%stride2) == stride1) { /* send data */
	receiver=my_rank-stride1;
	/* 	printf(" %03d: SEND to   %03d %d el\n",my_rank,receiver,sendq->qsize);fflush(stdout);  */
	MPI_Send(sendq->elements,sendq->qsize,node_dt,receiver,1001,MPI_COMM_WORLD);
      }
      stride1*=2;
      stride2*=2;
    }
  }


  MPI_Bcast(&sendq->qsize,1,MPI_INT,0,MPI_COMM_WORLD);
  MPI_Bcast(sendq->elements,sendq->qsize,node_dt,0,MPI_COMM_WORLD);

  /*                                    */  timedelay = MPI_Wtime()-starttime;
  if(my_rank==0) printf("timings[%03d] %-30s t=%10.6fs\n",my_rank,"[search_slow]",timedelay);

  /* test again the slowest pairs */
  /*                                    */  starttime = MPI_Wtime();
  {
    int i;
    MPI_Status status;

    for(i=0;i<size;i++) {
      if(my_rank==0) {
	printf(" % 6d: PINGPONG % 6d  <->  % 6d: ",i,sendq->elements[i].from,sendq->elements[i].to);fflush(stdout);
      }
      MPI_Barrier(MPI_COMM_WORLD);
      if(sendq->elements[i].from == my_rank) {
	/* 	printf(" %03d: PINGPONG %03d   ->  %03d\n",my_rank,my_rank,sendq->elements[i].to);fflush(stdout); */
	time =pingpong(my_rank,                 sendq->elements[i].to,my_rank,number_of_messages); 
      } 
      if(sendq->elements[i].to == my_rank) {
	/* 	printf(" %03d: PINGPONG %03d  <-   %03d\n",my_rank,sendq->elements[i].from,my_rank);fflush(stdout); */
	dummy =pingpong(sendq->elements[i].from, my_rank,              my_rank,number_of_messages); 
      }
      MPI_Barrier(MPI_COMM_WORLD);
      
      if(sendq->elements[i].from != 0) {
	/* send result to task zero */
	if(sendq->elements[i].from == my_rank) {
	  MPI_Send(&time,1,MPI_DOUBLE,0,1002,MPI_COMM_WORLD);
	}
	if(my_rank==0) {
	  MPI_Recv(&time,1,MPI_DOUBLE,sendq->elements[i].from,1002,MPI_COMM_WORLD,&status);
	}
      }
      if(my_rank==0) {
	stimings[i]=time;
	printf(" 1st: %10.4fus (%7.2f MB/s)    2nd: %10.4fus (%7.2f MB/s)\n",
	       sendq->elements[i].time*1000*1000,time2mbs(sendq->elements[i].time),
	       stimings[i]*1000*1000,time2mbs(stimings[i]));
      }
    }
    if(my_rank==0) {
      double min_time_stest=1e20;
      double max_time_stest=-1e20;
      double avg_time_stest=0.0;
      for(i=0;i<size;i++) {
	if(stimings[i]<min_time_stest) {
	  min_time_stest=stimings[i];
	}
	if(stimings[i]>max_time_stest) {
	  max_time_stest=stimings[i];
	}
	avg_time_stest+=stimings[i];
      }
      avg_time_stest/=size;

      printf("RESULT: Min Time Stest:      %20.8fus (%7.2f MB/s)\n",min_time_stest*1000*1000,time2mbs(min_time_stest));
      printf("RESULT: Max Time Stest:      %20.8fus (%7.2f MB/s)\n",max_time_stest*1000*1000,time2mbs(max_time_stest));
      printf("RESULT: Avg Time Stest:      %20.8fus (%7.2f MB/s)\n",avg_time_stest*1000*1000,time2mbs(avg_time_stest));
    }
  }
  /*                                    */  timedelay = MPI_Wtime()-starttime;
  if(my_rank==0) printf("timings[%03d] %-30s t=%10.6fs\n",my_rank,"[test_slow]",timedelay);


  /***********************************************************************************/
  /* output to sion file                                                             */
  /***********************************************************************************/
  if(do_sion) {
#ifdef USESION
    sion_numfiles = 1;
    
    if((size%collectpnum)!=0) {
      if(my_rank==0) {
	printf("WARNING: reduce collectpnum to 1 size=%d, collectpnum=%d \n",size,collectpnum);
      }
      collectpnum=1;
    }
    
    sion_chunksize=   MAX_LOCATION_LENGTH;        /* location */
    sion_chunksize+=  4 * sizeof(sion_int32);     /* mapping */
    sion_chunksize+=  size*sizeof(double);        /* timings */
    sion_chunksize+=  size*sizeof(sion_int32);    /* accesspattern */
    if(do_alltoall)   sion_chunksize+=  size*sizeof(double);                   /* alltoall timings, if option selected */
    
    sion_chunksize*=collectpnum;
    
    if(my_rank==0)    sion_chunksize+=10*sizeof(double)+10*sizeof(sion_int32);        /* statistics */
    if(my_rank==0)    sion_chunksize+=size * (2*sizeof(double)+2*sizeof(sion_int32)); /* single tests results */
    
    sion_fsblksize=2*1024*1024;
    
    inioset=((my_rank%collectpnum)==0);
    /*   printf("PE%04d: .. inioset %d / %d\n",my_rank,inioset,collectpnum); */
    MPI_Comm_split(MPI_COMM_WORLD,inioset,my_rank,&iocomm);
    
    
    MPI_Barrier(MPI_COMM_WORLD);
    
    /* open sion file */
    if(inioset) {
      /*                                    */  starttime = MPI_Wtime();
      sid=sion_paropen_mpi(filename_bin,"bw",
			   &sion_numfiles,
			   iocomm,
			   &comm_local,
			   &sion_chunksize,
			   &sion_fsblksize,
			   &grank,
			   &fp,
			   &newfname);
    }
    MPI_Barrier(MPI_COMM_WORLD);
    /*                                    */  timedelay = MPI_Wtime()-starttime;
    if(my_rank==0) printf("timings[%03d] %-30s t=%10.6fs\n",my_rank,"[sionopen]",timedelay);
    
    /*                                    */  starttime = MPI_Wtime();
    /*   printf("PE%04d: inioset %d / %d\n",my_rank,inioset,collectpnum); */
    
    if(inioset) {
      for(p=my_rank;p<(my_rank+collectpnum);p++) {
	/*       printf("PE%04d: start loop for %d\n",my_rank,p); */
	
	if(p>my_rank) {
	  /*         printf("PE%04d: send wakeup to %d\n",my_rank,p);fflush(stdout); */
	  MPI_Send(&dummy, 1, MPI_INT, p, WAKEUP, MPI_COMM_WORLD);
	  /*         printf("PE%04d: receive data from %d\n",my_rank,p); */
	  MPI_Recv(location, MAX_LOCATION_LENGTH, MPI_CHAR, p, LOCATION, MPI_COMM_WORLD, &status);
	  MPI_Recv(mapping, 4, MPI_INT, p, MAPPING, MPI_COMM_WORLD, &status);
	  MPI_Recv(ptimings, size, MPI_DOUBLE, p, PTIMINGS, MPI_COMM_WORLD, &status);
	  MPI_Recv(accesspattern, size, MPI_INTEGER4, p, ACCESS, MPI_COMM_WORLD, &status);
	  if(do_alltoall) {
	    MPI_Recv(atimings, size, MPI_DOUBLE, p, ATIMINGS, MPI_COMM_WORLD, &status);
	  }
	  /*         printf("PE%04d: received data from %d\n",my_rank,p); */
	}
	
	/* write location */
	left=MAX_LOCATION_LENGTH;
	ptr=location;
	while(left>0)  {
	  bwrote=fwrite(ptr, 1, left, fp);
	  left-=bwrote;
	  ptr+=bwrote;
	}
	
	/* write mapping */
	mappingout[0]=mapping[0]; mappingout[1]=mapping[1]; mappingout[2]=mapping[2]; mappingout[3]=mapping[3];
	/*       printf("MAPPING[%04d] = %d,%d,%d,%d (%s)\n",p,mapping[0],mapping[1],mapping[2],mapping[3],location); */
	left=4*sizeof(sion_int32);
	ptr=(char *) &mappingout[0];
	while(left>0)  {
	  bwrote=fwrite(ptr, 1, left, fp);
	  left-=bwrote;
	  ptr+=bwrote;
	}
	
	if((my_rank==0) && (p==0))  {
	  /* write statistics on rank 0 */
	  for(i=0;i<10;i++)  statdata_int[i]=0;
	  for(i=0;i<10;i++)  statdata_dp[i]=0.0;
	  statdata_int[0]=number_of_messages;
	  statdata_int[1]=length_of_message;
	  statdata_int[2]=do_alltoall;
	  statdata_int[3]=collectpnum;
	  statdata_int[4]=do_serial;
	  statdata_int[5]=do_mix;
	  statdata_int[9]=VERSION;
	  statdata_dp[0]=min_time_all;
	  statdata_dp[1]=max_time_all;
	  statdata_dp[2]=avg_time_all;    
	  if(do_alltoall) {
	    statdata_dp[3]=a2a_min_time_all;
	    statdata_dp[4]=a2a_max_time_all;
	    statdata_dp[5]=a2a_avg_time_all;
	  }
	  left=10*sizeof(sion_int32);
	  ptr=(char *) &statdata_int[0];
	  while(left>0)  {
	    bwrote=fwrite(ptr, 1, left, fp);
	    left-=bwrote;
	    ptr+=bwrote;
	  }
	  left=10*sizeof(double);
	  ptr=(char *) &statdata_dp[0];
	  while(left>0)  {
	    bwrote=fwrite(ptr, 1, left, fp);
	    left-=bwrote;
	    ptr+=bwrote;
	  }
	}


	/*       for(t=0;t<size;t++) printf("ptimings[%5d]=%e\n",t,ptimings[t]);   */
    
	/* write timings */
	left=size*sizeof(double);
	ptr=(char *) ptimings;
	while(left>0)  {
	  bwrote=fwrite(ptr, 1, left, fp);
	  left-=bwrote;
	  ptr+=bwrote;
	}
	/* write accesspattern */
	left=size*sizeof(sion_int32);
	ptr=(char *) accesspattern;
	while(left>0)  {
	  bwrote=fwrite(ptr, 1, left, fp);
	  left-=bwrote;
	  ptr+=bwrote;
	}


	/* write a2a timings */
	if(do_alltoall) {
	  left=size*sizeof(double);
	  ptr=(char *) atimings;
	  while(left>0)  {
	    bwrote=fwrite(ptr, 1, left, fp);
	    left-=bwrote;
	    ptr+=bwrote;
	  }
	}

	/* write single test results */
	if((my_rank==0) && (p==0))  {

	  /* new timings */
	  left=size*sizeof(double);
	  ptr=(char *) stimings;
	  while(left>0)  {
	    bwrote=fwrite(ptr, 1, left, fp);
	    left-=bwrote;
	    ptr+=bwrote;
	  }
	  
	  /* old timings */
	  for(i=0;i<size;i++)   stimings[i]=sendq->elements[i].time;
	  left=size*sizeof(double);
	  ptr=(char *) stimings;
	  while(left>0)  {
	    bwrote=fwrite(ptr, 1, left, fp);
	    left-=bwrote;
	    ptr+=bwrote;
	  }

	  /* from */
	  for(i=0;i<size;i++)   accesspattern[i]=sendq->elements[i].from;
	  left=size*sizeof(sion_int32);
	  ptr=(char *) accesspattern;
	  while(left>0)  {
	    bwrote=fwrite(ptr, 1, left, fp);
	    left-=bwrote;
	    ptr+=bwrote;
	  }

	  /* to */
	  for(i=0;i<size;i++)   accesspattern[i]=sendq->elements[i].to;
	  left=size*sizeof(sion_int32);
	  ptr=(char *) accesspattern;
	  while(left>0)  {
	    bwrote=fwrite(ptr, 1, left, fp);
	    left-=bwrote;
	    ptr+=bwrote;
	  }
	}
	/*       printf("PE%04d: wrote data for %d\n",my_rank,p); */

      } /* for p */

    } else { /* ! inioset */
      int sendp= ((int) (my_rank/collectpnum)) * collectpnum;
      /*     printf("PE%04d: waiting for wakeup from %d\n",my_rank,sendp); */
      MPI_Recv(&dummy, 1, MPI_INT, sendp, WAKEUP, MPI_COMM_WORLD,&status);
      /*     printf("PE%04d: received wakeup from %d\n",my_rank,sendp); */
      MPI_Send(location, MAX_LOCATION_LENGTH, MPI_CHAR, sendp, LOCATION, MPI_COMM_WORLD);
      MPI_Send(mapping, 4, MPI_INT, sendp, MAPPING, MPI_COMM_WORLD);
      MPI_Send(ptimings, size, MPI_DOUBLE, sendp, PTIMINGS, MPI_COMM_WORLD);
      MPI_Send(accesspattern, size, MPI_INTEGER4, sendp, ACCESS, MPI_COMM_WORLD);
      if(do_alltoall) {
	MPI_Send(atimings, size, MPI_DOUBLE, sendp, ATIMINGS, MPI_COMM_WORLD);
      }
      /*     printf("PE%04d: ready with send to %d\n",my_rank,sendp); */
    } /* inioset */

    MPI_Barrier(MPI_COMM_WORLD);
    /*                                    */  timedelay = MPI_Wtime()-starttime;
    if(my_rank==0) printf("timings[%03d] %-30s t=%10.6fs\n",my_rank,"[sionwrite]",timedelay);

    /* flush sion file */
    /*                                    */  starttime = MPI_Wtime();
    if(inioset) {
      sion_flush(sid);
    }
    MPI_Barrier(MPI_COMM_WORLD);
    /*                                    */  timedelay = MPI_Wtime()-starttime;
    if(my_rank==0) printf("timings[%03d] %-30s t=%10.6fs\n",my_rank,"[sionflush]",timedelay);

    /* close sion file */
    /*                                    */  starttime = MPI_Wtime();
  
    if(inioset) {
      sion_parclose_mpi(sid);
    }
    MPI_Barrier(MPI_COMM_WORLD);
    /*                                    */  timedelay = MPI_Wtime()-starttime;
    if(my_rank==0) printf("timings[%03d] %-30s t=%10.6fs\n",my_rank,"[sionclose]",timedelay);
#endif

  } /* (do_sion) */

  /*                                    */  timedelay = MPI_Wtime()-globalstarttime;
  if(my_rank==0) printf("timings[%03d] %-30s t=%10.6fs\n",my_rank,"[all]",timedelay);
  MPI_Finalize();
  return(0);
}


int work ( int my_rank, int step, int do_alltoall, int do_serial, 
	   int sleeptime, int size, int *pepartner, lim_queue *lqueue, int outerloop) {
  double timep_max,timep_min,timep,timea_max,timea_min, timea;
  double time, dummy, timep_avg, timea_avg;
  int partner, partner_rank,l;
  
  
  timea_min=0.0;
  timea_max=0.0;

  if(my_rank==0) {
      if(do_serial)  printf(" Serial   PingPong for step %5d: \n",step);
      else           printf(" Parallel PingPong for step %5d: ",step);
  }
  if(do_alltoall) {
    MPI_Barrier(MPI_COMM_WORLD);
    timea=alltoall(my_rank,size,1);
    MPI_Barrier(MPI_COMM_WORLD);
    MPI_Reduce(&timea,&timea_min,1,MPI_DOUBLE,MPI_MIN,0,MPI_COMM_WORLD);
    MPI_Reduce(&timea,&timea_max,1,MPI_DOUBLE,MPI_MAX,0,MPI_COMM_WORLD);
    MPI_Reduce(&timea,&timea_avg,1,MPI_DOUBLE,MPI_SUM,0,MPI_COMM_WORLD);
    timea_avg/=size;
    atimings[step]+=timea;
  } else {
    MPI_Barrier(MPI_COMM_WORLD);
  }

  partner=pepartner[step];
  if(partner<0) partner_rank=-partner-1;
  else          partner_rank= partner-1;
  
  if(!do_serial) {
    if(partner<0) {
      MPI_Barrier(MPI_COMM_WORLD);
      dummy=pingpong(partner_rank,my_rank     ,my_rank,number_of_messages); 
      MPI_Barrier(MPI_COMM_WORLD);
      time =pingpong(my_rank     ,partner_rank,my_rank,number_of_messages); 
      MPI_Barrier(MPI_COMM_WORLD);
    } else {
      MPI_Barrier(MPI_COMM_WORLD);
      time= pingpong(my_rank     ,partner_rank,my_rank,number_of_messages); 
      MPI_Barrier(MPI_COMM_WORLD);
      dummy=pingpong(partner_rank,my_rank     ,my_rank,number_of_messages); 
      MPI_Barrier(MPI_COMM_WORLD);
    }
  } else {
    
    for(l=0;l<size;l++) {
     
      MPI_Barrier(MPI_COMM_WORLD);
      if (l==my_rank) {
	  time= pingpong(my_rank     ,partner_rank,my_rank,number_of_messages); 
	  printf(" %6d->%6d:%12.5fus (%7.2f MB/s) (l=%d)\n",my_rank, partner_rank, 	   time*1000*1000,time2mbs(time),l);
      }
      if (l==partner_rank) {
	  dummy=pingpong(partner_rank,my_rank     ,my_rank,number_of_messages); 
      }
      MPI_Barrier(MPI_COMM_WORLD);
   }
  }
  ptimings[partner_rank]+=time;

  /* printf("WF[%4d] set ptimings[%d]=%e (%d)\n",my_rank,partner_rank,ptimings[partner_rank],partner);  */
#ifdef USESION
  accesspattern[partner_rank]=step;
#endif
  limqueue_unsort_insert(lqueue, time, my_rank, partner_rank);


  /* get results of this step (min,max) */
  timep=time; 
  MPI_Reduce(&timep,&timep_min, 1,  MPI_DOUBLE,MPI_MIN,0,MPI_COMM_WORLD);
  MPI_Reduce(&timep,&timep_max, 1,  MPI_DOUBLE,MPI_MAX,0,MPI_COMM_WORLD);
  MPI_Reduce(&timep,&timep_avg, 1,  MPI_DOUBLE,MPI_SUM,0,MPI_COMM_WORLD);
  timep_avg/=size;

  
  if(my_rank==0) {

    if(timep_min<min_time_all) {
      min_time_all=timep_min;
    }
    if(timep_max>max_time_all) {
      max_time_all=timep_max;
    }
    avg_time_all+=timep_avg;

    if(do_alltoall) {
	if(timea_min<a2a_min_time_all) {
	    a2a_min_time_all=timea_min;
	}
	if(timea_max>a2a_max_time_all) {
	    a2a_max_time_all=timea_max;
	}
	a2a_avg_time_all+=timea_avg;
    }
    printf("  avg=%10.3fus (%7.2f MB/s) min=%10.3fus (%7.2f MB/s) max=%10.3fus (%7.2f MB/s)",
	   timep_avg*1000*1000,time2mbs(timep_avg),
	   timep_min*1000*1000,time2mbs(timep_min),
	   timep_max*1000*1000,time2mbs(timep_max)	   );
    if(do_alltoall) {
      printf(" [ alltoall min=%10.3fus max=%10.3fus ]\n",
	     timea_min*1000*1000,
	     timea_max*1000*1000                   );
    } else {
      printf("\n");
    }
  }
  
  return(1);
}

int perm (int step, int n, int *line) {
    int i,b,help;
    int numblocks=n/step;
    for(b=0;b<numblocks;b+=2) {
	for(i=0;i<step;i++) {
	    help=line[b*step+i];
	    line[b*step+i]=line[(b+1)*step+i];
	    line[(b+1)*step+i]=help;
	}
    }
  return(1);
}



/*
     a0      a1
       2  3  4               1  2  3
   0-1 |  |  |     -->	 0-7 |  |  |
       7  6  5 	             6  5  4
       b1    b0  
*/

int getpartnerplan_pe (int penr, int n, int *pepartner, int do_mix, int my_rank) {
  int *line=NULL;
  int *helpline=NULL;
  int l,i,b,help,sum;
  int a0,a1,b0,b1,maxp,from,to,p;
  double starttime, timedelay;

  starttime = MPI_Wtime();
  
  line = (int *) malloc(sizeof(int)*n);
  if (line == NULL) {
    fprintf(stderr, "cannot allocate line[%lu] ...\n", (unsigned long) sizeof(int)*n);
    MPI_Abort(MPI_COMM_WORLD,1);
  }
  helpline = (int *) malloc(sizeof(int)*n);
  if (line == NULL) {
    fprintf(stderr, "cannot allocate helpline[%lu] ...\n", (unsigned long) sizeof(int)*n);
    MPI_Abort(MPI_COMM_WORLD,1);
  }

  if(do_mix) {
    if(my_rank==0) {
      /* generate randomized vector */
#ifdef OLDRANDOMGEN      
      int pos;
      for(i=0;i<n;i++) {helpline[i]=i;}
      srand(1);
      for(i=0;i<n;i++) {
        do {
          pos= (int) ((double) rand() * (double) (n) / (double) RAND_MAX );
/*           printf("%03d: pos=%3d -->%3d\n",i,pos,helpline[pos]);  */
        } while((helpline[pos]<0 || (pos>=n)));
        line[i]=pos;
        helpline[pos]=-1;
/*         printf("line[%03d]=%03d (%d)\n",i,pos,helpline[pos]);  */
      }
#else
      int help,pos;
      for(i=0;i<n;i++) {line[i]=i;}
      for(i=0;i<n;i++) {
	pos= (int) ((double) rand() * (double) (n) / (double) RAND_MAX );
/* 	printf(" exch i=%3d <-> pos=%3d\n",i,pos);  */
	help=line[i];
        line[i]=line[pos];
	line[pos]=help;
      }
#endif      
    }

    timedelay = MPI_Wtime()-starttime;
    if(my_rank==0) printf("timings[%03d] %-30s t=%10.6fs\n",my_rank,"[mixpe]",timedelay);
    starttime = MPI_Wtime();
    
    MPI_Bcast(&line[0],n,MPI_INT,0,MPI_COMM_WORLD);

    timedelay = MPI_Wtime()-starttime;
    if(my_rank==0) printf("timings[%03d] %-30s t=%10.6fs\n",my_rank,"[bcast]",timedelay);

    sum=0;
    for(i=0;i<n;i++) {sum+=line[i];}
/*     printf("PE[%06d] do_mix=1 sum=%08d\n",penr,sum);   */
    
  } else {
    for(i=0;i<n;i++) {line[i]=i;}

    sum=0;
    for(i=0;i<n;i++) {sum+=line[i];}
/*     printf("PE[%06d] domix=0 sum=%08d\n",penr,sum);   */
  }
  
#ifdef OUTPUTRANDOMDIST
  if(my_rank==0) {
    printf("Random distribution: "); 
    for(i=0;i<n;i++) { printf("%d,",line[i]); };
    printf("\n");
  }
#endif

  a0=1;maxp=a1=(int) (n/2);
  b0=a1+1;b1=n-1;
    
  for(l=0;l<n;l++) {
    /*   get partners */
    for(p=0;p<(maxp-1);p++) {
      from=line[a1-p];  to=line[b0+p];
      if(from==penr) { pepartner[l]=to+1;	    } 
      if(to==penr)   { pepartner[l]=-from-1;    } 
    }
    from=line[0];	to=line[a0];
    if(from==penr) { pepartner[l]=to+1;  } 
    if(to==penr)   { pepartner[l]=-from-1; } 

    /*  rotate */
    help=line[b1];
    for(b=b1;b>0;b--) {
      line[b]=line[b-1];
    }
    line[a0]=help;
  }

  timedelay = MPI_Wtime()-starttime;
  if(my_rank==0) printf("timings[%03d] %-30s t=%10.6fs\n",my_rank,"[pelist]",timedelay);

  return(1);
}


int getrandomize_vector_search (int n, int *vec) {
  int *helpline=NULL;
  int i;
  int pos;

  helpline = (int *) malloc(sizeof(int)*n);
  if (helpline == NULL) {
    fprintf(stderr, "cannot allocate helpline[%lu] ...\n", (unsigned long) sizeof(int)*n);
    exit(0);
  }
  for(i=0;i<n;i++) {helpline[i]=i;}

  srand(1);
  for(i=0;i<n;i++) {
    do {
      pos= (int) ((double) rand() * (double) (n) / (double) RAND_MAX );
      /*           printf("%03d: pos=%3d -->%3d\n",i,pos,helpline[pos]);  */
    } while((helpline[pos]<0 || (pos>=n)));
    vec[i]=pos;
    helpline[pos]=-1;
    /*         printf("line[%03d]=%03d (%d)\n",i,pos,helpline[pos]);  */
  }
  free(helpline);
  return(1);
}

int getrandomize_vector_swap (int n, int *vec) {

  int help,pos;
  int i;
  for(i=0;i<n;i++) {vec[i]=i;}
  for(i=0;i<n;i++) {
    pos= (int) ((double) rand() * (double) (n) / (double) RAND_MAX );
    /* 	printf(" exch i=%3d <-> pos=%3d\n",i,pos);  */
    help=vec[i];
    vec[i]=vec[pos];
    vec[pos]=help;
  }
  return(1);
}

/*
             a1
    a0 2  3  4               1  2  3
 0 - 1 |  |  |     -->	 0-7 |  |  |
       7  6  5 	             6  5  4
       b1    b0  
*/

#define v(i) vecn1[(i-l+(n-1))%(n-1)]

int getpartneropt (int penr, int n, int *vec, int *pepartner) {

  int pos;
  int a1,a2,an,b1,bn;
  int p,l,i;
  int *vecn1;


  vecn1=vec+1;

  a1=0;
  a2=1;
  an=(int) (n/2) -1;
  b1=an+1;
  bn=n-2;

  /* printf("n=%d: a1=%d, a2=%d,an=%d b1=%d bn=%d\n",n,a1,a2,an,b1,bn);  */

  if(vec[0]==penr) {
    for(l=0;l<n;l++) {
      pepartner[l]=v(0)+1;          /* p -> x */
      /* printf(" penr=%2d p=%2d partner[%2d]=%2d\n",penr,p,l,pepartner[l]);  */
    }
  } else {
    p=-1;
    for(pos=a1;(pos<=bn && (p==-1));pos++) {
      if(vecn1[pos]==penr) p=pos;
    }
    
    for(l=0;l<n;l++) {
      if((p>=a2) && (p<=an)) {
	pepartner[l]=v(bn-(p-a2))+1; /* p -> x */
	p=p+1;
      }else if((p>=b1) && (p<=bn)) {
	pepartner[l]=-v(a2+(bn-p))-1; /* x <- p */
	p=p+1;
	if(p>bn) p=a1;
      } else if(p==a1) {
	pepartner[l]=-vec[0]-1;       /* x <- p */
	p=p+1;
      }

      if(0) {
	printf("opt  l=%2d: ",l);
	printf(" %2d -",vec[0]);
	for(i=0;i<n-1;i++) {
	  printf(" %2d",v(i)); 
	  if(i==0) printf("  ");
	  if(i==an) printf("  - ");
	}
	printf(" penr=%2d p=%2d partner[%2d]=%2d\n",penr,p,l,pepartner[l]); 
      }

    }
    
  }
    

  
  return(1);
}


struct rusage rusagestr;

int print_memusage(int i)
{
  if (getrusage(RUSAGE_SELF, &rusagestr) != 0)
    return -1;

  printf("PE: %04d: MEMUSAGE: %12.8f\n",i, rusagestr.ru_maxrss / 1024.0);

  return 1;
} 


