// $Id$
//
// Earth System Modeling Framework
// Copyright (c) 2002-2025, University Corporation for Atmospheric Research,
// Massachusetts Institute of Technology, Geophysical Fluid Dynamics
// Laboratory, University of Michigan, National Centers for Environmental
// Prediction, Los Alamos National Laboratory, Argonne National Laboratory,
// NASA Goddard Space Flight Center.
// Licensed under the University of Illinois-NCSA License.
//
//-----------------------------------------------------------------------------

#include "ESMCI_VMKernel.h"
#include "ESMCI_VM.h"

#define WITHBLOCKER_on

#define VM_PTHREADLOG_on
#define VM_PETMANAGEMENTLOG_off
#define VM_MEMLOG_off
#define VM_COMMQUEUELOG_off
#define VM_EPOCHLOG_off
#define VM_EPOCHMEMLOG_off
#define VM_SSISHMLOG_off
#define VM_SIZELOG_off

// On SunOS systems there are a couple of macros that need to be set
// in order to get POSIX compliant functions for IPC, pthreads
#ifdef __sun
#define _POSIX_SOURCE
#define _POSIX_C_SOURCE 199309L
#define __EXTENSIONS__
#define _POSIX_PTHREAD_SEMANTICS
#endif

#include <sys/types.h>

// On OSF1 (i.e. Tru64) systems there is a problem with picking up some
// prototypes from unistd.h from within C++....
#ifdef __osf__
#define _XOPEN_SOURCE_EXTENDED
#endif

#if !defined (ESMF_OS_MinGW)
#include <unistd.h>
#include <sys/time.h>
#else
#include <windows.h>
#endif

// On OSF1 (i.e. Tru64) systems there is a problem with picking up some
// prototypes from unistd.h from within C++....
#ifdef __osf__
#undef _XOPEN_SOURCE_EXTENDED
#endif

// Where available use the sched header
#if (defined ESMF_OS_Linux || defined ESMF_OS_Unicos)
#include <sched.h>
#endif

// Standard headers
#include <iostream>
#include <iomanip>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <ctime>
#include <cfloat>
#include <cmath>
#include <vector>
#include <set>
#include <algorithm>
#ifdef __sun
#include <signal.h>
#else
#include <csignal>
#endif
#include <limits.h>

using namespace std;

// Memory mapped files may not be available on all systems
#ifndef ESMF_NO_POSIXIPC
#include <sys/mman.h>
#endif

#include <fcntl.h>

#include "ESMCI_AccInfo.h"
#include "ESMCI_LogErr.h"

#ifndef ESMF_NO_OPENACC
#include <openacc.h>
#endif

// NUMA support
#ifdef ESMF_NUMA
#include <numa.h>
#endif

// NVML support
#ifdef ESMF_NVML
#include <nvml.h>
#endif

// macros used within this source file
#define VERBOSITY             (1)       // 0: off, 10: max
#define VM_TID_MPI_TAG        (10)      // mpi tag used to send/recv TID
#ifdef SIGRTMIN
#define VM_SIG1               (SIGRTMIN)  // avoid sigusr1 and sigusr2 if avail.
#else
#ifdef ESMF_NO_SIGUSR2
#define VM_SIG1               (SIGUSR1)
// Note that SIGUSR1 interferes with MPICH's CH_P4 device!
#else
#define VM_SIG1               (SIGUSR2)
// Note that SIGUSR2 interferes with LAM!
#endif
#endif

#if defined (ESMF_OS_MinGW)
// Windows equivalent to POSIX getpid(2)
#if !defined (__GNUC__)
typedef DWORD pid_t;
#endif
#define getpid GetCurrentProcessId
#endif

// Default requested MPI thread support level
#ifdef ESMF_NO_PTHREADS
#define VM_MPI_THREAD_LEVEL MPI_THREAD_SINGLE
#else
#define VM_MPI_THREAD_LEVEL MPI_THREAD_MULTIPLE
#endif

// Utility macros to get a macro value into a string
#define XSTR(X) STR(X)
#define STR(X) #X

namespace ESMCI {

// Definition of class static data members
std::vector<MPI_Datatype> VMK::customType(10);  // up to 2^10 = 1024 byte
MPI_Comm VMK::default_mpi_c;
int VMK::mpi_thread_level_requested;
int VMK::mpi_thread_level;
int VMK::mpi_init_outside_esmf;
int VMK::pre_mpi_init = 0;
int VMK::nssiid;
int VMK::ncores;
int VMK::ndevs;
int VMK::ndevsSSI;
int *VMK::cpuid;
int *VMK::ssiid;
int *VMK::ssipe;
int *VMK::ssidevs;
double VMK::wtime0;
// Static data members to support command line arguments
int VMK::argc;
char *VMK::argv_store[100];
char **VMK::argv = &(argv_store[0]);
// Second set of command line argument variables to support MPICH1.2
int VMK::argc_mpich;
char *VMK::argv_mpich_store[100];
char **VMK::argv_mpich = &(argv_mpich_store[0]);
// NUMA support
bool VMK::nvmlEnabled;
// NUMA support
bool VMK::numaEnabled;

} // namespace ESMCI

// -----------------------------------------------------------------------------
// vmkt encapsulation: begin
typedef struct{
  volatile int flag;
  esmf_pthread_t tid;
  esmf_pthread_mutex_t mut0;
  esmf_pthread_cond_t cond0;
  esmf_pthread_mutex_t mut1;
  esmf_pthread_cond_t cond1;
  esmf_pthread_mutex_t mut_extra1;
  esmf_pthread_cond_t cond_extra1;
  esmf_pthread_mutex_t mut_extra2;
  esmf_pthread_cond_t cond_extra2;
  void *arg;
  int released;
}vmkt_t;

int vmkt_create(vmkt_t *vmkt, void *(*vmkt_spawn)(void *), void *arg,
  bool service, size_t minStackSize){
  vmkt->flag = 0;     // initialize
  vmkt->released = 0; // initialize
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_init(&(vmkt->mut0), NULL);
  pthread_mutex_lock(&(vmkt->mut0));
  pthread_cond_init(&(vmkt->cond0), NULL);
  pthread_mutex_init(&(vmkt->mut1), NULL);
  //pthread_mutex_lock(&(vmkt->mut1));
  pthread_cond_init(&(vmkt->cond1), NULL);
  pthread_mutex_init(&(vmkt->mut_extra1), NULL);
  //pthread_mutex_lock(&(vmkt->mut_extra1));
  pthread_cond_init(&(vmkt->cond_extra1), NULL);
  pthread_mutex_init(&(vmkt->mut_extra2), NULL);
  pthread_mutex_lock(&(vmkt->mut_extra2));
  pthread_cond_init(&(vmkt->cond_extra2), NULL);
  pthread_attr_t *pthreadAttrsPtr = NULL;
  if (_POSIX_THREAD_ATTR_STACKSIZE){
    // this Pthread implementation supports stack size attribute
    pthread_attr_t pthreadAttrs;
    pthread_attr_init(&pthreadAttrs);
    if (service){
      // setting stack size for a service thread
      pthread_attr_setstacksize(&pthreadAttrs,
        (size_t)VM_PTHREAD_STACKSIZE_SERVICE);
    }else{
      // setting stack size for a user thread
      size_t default_stack_size;
      pthread_attr_getstacksize(&pthreadAttrs, &default_stack_size);
      if (default_stack_size < minStackSize){
        pthread_attr_setstacksize(&pthreadAttrs, minStackSize);
      }
    }
#ifdef VM_PTHREADLOG_on
    size_t stack_size;
    pthread_attr_getstacksize(&pthreadAttrs, &stack_size);
    std::stringstream msg;
    msg << "vmkt_create()#" << __LINE__
      << " Pthread: service=" << service
      << " (0: actual PET, 1: service thread) -"
      << " PTHREAD_STACK_MIN: " << PTHREAD_STACK_MIN << " bytes"
      << " stack_size: " << stack_size << " bytes";
    ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
#endif
    pthreadAttrsPtr = &pthreadAttrs;
  }
  int error = pthread_create(&(vmkt->tid), pthreadAttrsPtr, vmkt_spawn, arg);
  if (!error){ // only wait if the thread was successfully created
    pthread_cond_wait(&(vmkt->cond0), &(vmkt->mut0));   // back-sync #1
    pthread_cond_wait(&(vmkt->cond_extra2), &(vmkt->mut_extra2)); // back-s. #2
  }
  return error;
#else
  return 0;
#endif
}

int vmkt_release(vmkt_t *vmkt, void *arg){
  vmkt->arg = arg;
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_lock(&(vmkt->mut1));  
  pthread_cond_signal(&(vmkt->cond1));
  pthread_mutex_unlock(&(vmkt->mut1));
#endif
  vmkt->released = 1; // set flag
  return 0;
}

int vmkt_catch(vmkt_t *vmkt){
#ifndef ESMF_NO_PTHREADS
  pthread_cond_wait(&(vmkt->cond0), &(vmkt->mut0)); //wait for the child
#endif
  vmkt->released = 0; // reset flag  
  return 0;
}

int vmkt_join(vmkt_t *vmkt){
  if (vmkt->released){
    // need to first catch the released threads
    vmkt_catch(vmkt);
  }
  vmkt->flag = 1; // set flag to indicate that this is a wrap up call
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_lock(&(vmkt->mut1));  
  pthread_cond_signal(&(vmkt->cond1));
  pthread_mutex_unlock(&(vmkt->mut1));
  pthread_join(vmkt->tid, NULL); //wait for the child
#endif
  return 0;
}
// vmkt encapsulation: end
// -----------------------------------------------------------------------------


namespace ESMCI {

void VMK::obtain_args(){
  // obtain command line args for this process
#ifndef ESMF_NO_SYSTEMCALL
  int mypid = getpid();
  char command[160], fname[80], args[8000];
  FILE *fp;
#ifdef ESMF_OS_Linux
  // this is a SUS3=sysV derived OS
  // fgets() changes dir on Linux/MPICH -> use fscanf() instead
  sprintf(command, "env COLUMNS=8000 ps -p %d -o args= > .args.%d", mypid,
    mypid);
  system(command);
  sprintf(fname, ".args.%d", mypid);
  fp=fopen(fname, "r");
  if (fp){
    fscanf(fp, "%[^\n]", args);
    fclose(fp);
  }else{
    args[0]='\0'; // empty args string
  }
  sprintf(command, "rm -f .args.%d", mypid);
  system(command);
#elif defined ESMF_OS_Darwin
  // this is a BSD derived OS
  sprintf(command, "env COLUMNS=8000 ps -w -w -p %d -o command > .args.%d",
          mypid, mypid);
  system(command);
  sprintf(fname, ".args.%d", mypid);
  fp=fopen(fname, "r");
  if (fp){
    fgets(args, 8000, fp);  // scan off header line of ps output
    fgets(args, 8000, fp);
    fclose(fp);
  }else{
    args[0]='\0'; // empty args string
  }
  sprintf(command, "rm -f .args.%d", mypid);
  system(command);
#else
  args[0]='\0'; // empty args string
#endif
  // now the string 'args' holds the complete command line with arguments
  argc=0;
  int i=0;
  int j=0;
  // chop up args and thus set up argc and argv
  while (args[i] != '\0'){
    if (args[i] != ' '){
      argv[argc][j] = args[i];
      ++j;
    }else{
      argv[argc][j] = '\0';
      ++argc;
      j=0;
    }
    ++i;
  }
  if (i){
    // only if this isn't for a complete NULL case
    argv[argc][j] = '\0';
    ++argc;
  }
  // now argc and argv are valid
  //printf("argc=%d\n", argc);
  //for (i=0; i<argc; i++)
  //  printf("%s\n", argv[i]);
#endif
}


void VMK::InitPreMPI(){
#if !defined (ESMF_NO_SIGNALS)
  // initialize signal handling -> this MUST happen before MPI_Init is called!!
  struct sigaction action;
  action.sa_handler = SIG_DFL;
  action.sa_flags   = 0;
  sigemptyset (&(action.sa_mask));
  sigaction(VM_SIG1, &action, NULL);  // restore default handle for VM_SIG1
  sigset_t sigs_to_block;
  sigemptyset(&sigs_to_block);
  sigaddset(&sigs_to_block, VM_SIG1);
  sigprocmask(SIG_BLOCK, &sigs_to_block, NULL); // block VM_SIG1
#endif
  pre_mpi_init = 1; // set flag
}


void VMK::set(bool globalResourceControl){
#ifndef ESMF_NO_PTHREADS
  if (globalResourceControl){
#if !defined(ESMF_OS_Darwin) && !defined(ESMF_OS_Cygwin)
    // setting affinity on this level might interfer with user level pinning
    // therefore only do it by user request
    // set thread affinity
    cpu_set_t cpuset;
    CPU_ZERO(&cpuset);
    CPU_SET(ssipe[mypet], &cpuset);
    pthread_setaffinity_np(mypthid, sizeof(cpu_set_t), &cpuset);
#endif
#ifndef ESMF_NO_OPENMP
    omp_set_num_threads(1);
#endif
#ifndef ESMF_NO_OPENACC
#ifdef __NVCOMPILER
    // under NVHPC, multi-cpu-core threading (for OpenACC and standard language
    // constructs) is implemented based on OpenACC: NVHPC extended OpenACC API
    acc_set_num_cores(1);
#endif
#endif
  }
#endif
}


void VMK::init(MPI_Comm mpiCommunicator, bool globalResourceControl){
#undef  ESMC_METHOD
#define ESMC_METHOD "ESMCI::VMK::init()"
  // initialize the physical machine and a default (all MPI) virtual machine
  // obtain command line arguments and store in the VM class
  argc = 0; // reset
  for (int k=0; k<100; k++)
    argv[k] = new char[1600];
#ifdef ESMF_MPICH1
  // currently only obtain arguments for MPICH because it needs it!!!
  obtain_args();
#endif
  // determine mpi_thread_level_requested
  mpi_thread_level_requested = VM_MPI_THREAD_LEVEL; // default
  char const *esmfRuntimeVarValue =
    std::getenv("ESMF_RUNTIME_MPI_THREAD_SUPPORT");
  if (esmfRuntimeVarValue && strlen(esmfRuntimeVarValue) > 0) {
    std::string mpi_thread_level_string(esmfRuntimeVarValue);
    transform(mpi_thread_level_string.begin(), mpi_thread_level_string.end(),
      mpi_thread_level_string.begin(), ::toupper);
    if (mpi_thread_level_string == "MPI_THREAD_SINGLE"){
      mpi_thread_level_requested = MPI_THREAD_SINGLE;
    }else if (mpi_thread_level_string == "MPI_THREAD_FUNNELED"){
      mpi_thread_level_requested = MPI_THREAD_FUNNELED;
    }else if (mpi_thread_level_string == "MPI_THREAD_SERIALIZED"){
      mpi_thread_level_requested = MPI_THREAD_SERIALIZED;
    }else if (mpi_thread_level_string == "MPI_THREAD_MULTIPLE"){
      mpi_thread_level_requested = MPI_THREAD_MULTIPLE;
    }else{
      // error: unknown MPI thread support level
      int localrc;
      ESMC_LogDefault.MsgFoundError(ESMF_RC_VAL_WRONG,
        "Unkown MPI thread support level requested (typo?): "
        + mpi_thread_level_string, ESMC_CONTEXT, &localrc);
      throw localrc;  // bail out with exception
    }
  }
  // next check is whether MPI has been initialized yet
  // there is a check in vmk_sigcatcher() to make sure signals between processes
  // are only used if ESMF initialized MPI to make sure all the threads
  // were reached with the SIG_BLOCK above.
  MPI_Initialized(&mpi_init_outside_esmf);
#ifndef ESMF_MPIUNI
  if (!mpi_init_outside_esmf){
    InitPreMPI(); // must call before MPI is initialized
#ifdef ESMF_MPICH1
    // MPICH1.2 is not standard compliant and needs valid args
    // make copy of argc and argv for MPICH because it modifies them and
    // the original values are needed to delete the memory during finalize()
    argc_mpich = argc;
    for (int k=0; k<100; k++)
      argv_mpich[k] = argv[k];
    MPI_Init_thread(&argc_mpich, (char ***)&argv_mpich,
      mpi_thread_level_requested, &mpi_thread_level);
#else
    MPI_Init_thread(NULL, NULL, mpi_thread_level_requested, &mpi_thread_level);
#endif
  }else{
    // query the MPI thread support level as set by external MPI initialization
    MPI_Query_thread(&mpi_thread_level);
  }
#else
  // MPIUNI simply set the thread level
  mpi_thread_level = MPI_THREAD_SERIALIZED;
#endif
  // so now MPI is for sure initialized...
  wtime0 = MPI_Wtime();
  // TODO: now it should be safe to call obtain_args() for all MPI impl.
  // Obtain MPI variables
  int rank, size;
  MPI_Comm_rank(mpiCommunicator, &rank);
  MPI_Comm_size(mpiCommunicator, &size);
  // since this method is only supposed to be called my the main_vmachine 
  // and the main_vmachine is all MPI pets we can do the following:
  npets=size;           // user is required to start with #processes=#cores!!!!
  mypet=rank;
#ifndef ESMF_NO_PTHREADS
  mypthid=pthread_self();
#else
  mypthid=0;
#endif
#ifdef ESMF_MPIUNI
  mpionly=false;        // this way the commtype will be checked in comm calls
#else
  if (npets==1)
    mpionly=false;      // this way the commtype will be checked in comm calls
  else
    mpionly=true;       // normally the default VM can only be MPI-only
#endif
  // no threading in default global VM
  threadsflag = false;
  // set up private Group and Comm objects across "mpiCommunicator"
  MPI_Group mpi_g;
  MPI_Comm_group(mpiCommunicator, &mpi_g);
  MPI_Comm_create(mpiCommunicator, mpi_g, &mpi_c);
  MPI_Group_free(&mpi_g);
  // ... and copy the Comm object into the class static default variable...
  default_mpi_c = mpi_c;
#if (MPI_VERSION >= 3)
  // set up communicator across single-system-images SSIs
  MPI_Comm_split_type(mpi_c, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL,
    &mpi_c_ssi);
  // set up communicator across root pets of each SSI
  int color;
  MPI_Comm_rank(mpi_c_ssi, &color);
  if (color>0) color = MPI_UNDEFINED; // only root PETs on each SSI
  MPI_Comm_split(mpi_c, color, 0, &mpi_c_ssi_roots);
#else
  mpi_c_ssi = MPI_COMM_NULL;
  mpi_c_ssi_roots = MPI_COMM_NULL;
#endif
  // check whether NVML support is enabled
#ifdef ESMF_NVML
  nvmlEnabled = true;
#else
  nvmlEnabled = false;
#endif
  // check whether NUMA support is enabled
#ifdef ESMF_NUMA
  numaEnabled = true;
  if (numa_available()<0) numaEnabled = false;  // NUMA not available at runtime
#else
  numaEnabled = false;
#endif
  // initialize the shared memory variables
  pth_finish_count = NULL;
  pth_mutex = new esmf_pthread_mutex_t;
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_init(pth_mutex, NULL);
#endif
  pth_mutex2 = new esmf_pthread_mutex_t;
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_init(pth_mutex2, NULL);
#endif
  // the mutex flag must be reset
  if (mpi_thread_level<MPI_THREAD_MULTIPLE)
    mpi_mutex_flag = 1; // must use muteces around mpi comms
  else
    mpi_mutex_flag = 0; // don't need to use muteces around mpi comms
  // setup the communication channels
  sendChannel = new comminfo[npets];
  recvChannel = new comminfo[npets];
#ifdef ESMF_MPIUNI
  // for mpiuni the default communication is via MPIUNI branch
  sendChannel[0].comm_type = VM_COMM_TYPE_MPIUNI;
  sendChannel[0].shmp = new shared_mp;
  sync_reset(&(sendChannel[0].shmp->shms));
  sendChannel[0].shmp->tcounter = 0;
  sendChannel[0].shmp->recvCount = 0;
  sendChannel[0].shmp->sendCount = 0;
  for (int i=0; i<SHARED_NONBLOCK_CHANNELS; i++){
    sendChannel[0].shmp->ptr_src_nb[i] = NULL;
    sendChannel[0].shmp->ptr_dst_nb[i] = NULL;
  }
  recvChannel[0] = sendChannel[0];
#else
  if (npets==1){
    // for single PET VMs use the MPIUNI branch
    sendChannel[0].comm_type = VM_COMM_TYPE_MPIUNI;
    sendChannel[0].shmp = new shared_mp;
    sync_reset(&(sendChannel[0].shmp->shms));
    sendChannel[0].shmp->tcounter = 0;
    sendChannel[0].shmp->recvCount = 0;
    sendChannel[0].shmp->sendCount = 0;
    for (int i=0; i<SHARED_NONBLOCK_CHANNELS; i++){
      sendChannel[0].shmp->ptr_src_nb[i] = NULL;
      sendChannel[0].shmp->ptr_dst_nb[i] = NULL;
    }
    recvChannel[0] = sendChannel[0];
  }else{
    for (int i=0; i<npets; i++){
      // normally by default all communication is via MPI-1
      sendChannel[i].comm_type = VM_COMM_TYPE_MPI1;
      recvChannel[i].comm_type = VM_COMM_TYPE_MPI1;
    }
  }
#endif
  // setup the IntraProcessSharedMemoryAllocation List
  ipshmTop = new ipshmAlloc*;
  *ipshmTop = NULL;      // reset
  ipshmLocalTop = NULL;   // reset
  ipshmMutex = new esmf_pthread_mutex_t;
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_init(ipshmMutex, NULL);
#endif
  ipSetupMutex = new esmf_pthread_mutex_t;
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_init(ipSetupMutex, NULL);
#endif
  // set up the request queue
  nhandles=0;
  firsthandle=NULL;
  // set up physical machine info
  ncores=size;          // user is required to start with #processes=#cores!!!!
  // determine CPU ids
  cpuid = new int[ncores];
  for (int i=0; i<ncores; i++){
    cpuid[i]=i;                 // hardcoded assumption of single-core CPUs
  }
  // determine SSI ids and ssipe
  ssiid = new int[ncores];
  ssipe = new int[ncores];
#if (MPI_VERSION >= 3)
  // use mpi_c_ssi and mpi_c_ssi_roots communicators to discover SSIs
  if (mpi_c_ssi_roots != MPI_COMM_NULL)
    MPI_Comm_size(mpi_c_ssi_roots, &ssiCount);
  MPI_Bcast(&ssiCount, 1, MPI_INTEGER, 0, mpi_c_ssi);
  int temp_int;
  if (mpi_c_ssi_roots != MPI_COMM_NULL)
    MPI_Comm_rank(mpi_c_ssi_roots, &temp_int);
  MPI_Bcast(&temp_int, 1, MPI_INTEGER, 0, mpi_c_ssi);
  MPI_Allgather(&temp_int, 1, MPI_INT, ssiid, 1, MPI_INT, mpi_c);
  MPI_Comm_rank(mpi_c_ssi, &temp_int);
  MPI_Allgather(&temp_int, 1, MPI_INT, ssipe, 1, MPI_INT, mpi_c);
  MPI_Comm_size(mpi_c_ssi, &ssiLocalPetCount);
  MPI_Allreduce(&ssiLocalPetCount, &ssiMinPetCount, 1, MPI_INT, MPI_MIN, mpi_c);
  MPI_Allreduce(&ssiLocalPetCount, &ssiMaxPetCount, 1, MPI_INT, MPI_MAX, mpi_c);
  int localSsi = ssiid[mypet];
#else
  // use MPI_Get_processor_name() to discover SSIs
  std::vector<int> temp_ssiPetCount(ncores);
  char hostname[MPI_MAX_PROCESSOR_NAME];
  memset(hostname, 0, MPI_MAX_PROCESSOR_NAME);  // ensure null termination
  int resultLen;
  MPI_Get_processor_name(hostname, &resultLen);
#if 0
{
  std::stringstream msg;
  msg << "VMK::init()#" << __LINE__
    << " hostname=" << string(hostname);
  ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
}
#endif
  struct hname{
    char name[MPI_MAX_PROCESSOR_NAME];
  };
  std::vector<hname> hostnames(ncores);
  MPI_Allgather(hostname, MPI_MAX_PROCESSOR_NAME, MPI_CHAR,
    &(hostnames[0]), MPI_MAX_PROCESSOR_NAME, MPI_CHAR, mpi_c);
  // now number the ssiid[] to go like 0, 1, 2, ...
  ssiCount=0;
  for (int i=0; i<ncores; i++){
    int j;
    for (j=0; j<i; j++)
      if (string(hostnames[j].name) == string(hostnames[i].name)) break;
    if (j==i){
      // found new ssiid
      ssiid[i]=ssiCount;
      ++ssiCount;
      ssipe[i]=0;
      temp_ssiPetCount[ssiid[i]] = 1;
    }else{
      // found previous ssiid
      ssiid[i]=ssiid[j];
      ssipe[i]=temp_ssiPetCount[ssiid[i]];
      temp_ssiPetCount[ssiid[i]]++;
    }
  }
  ssiMinPetCount=ncores;
  ssiMaxPetCount=0;
  for (int i=0; i<ssiCount; i++){
    if (temp_ssiPetCount[i] < ssiMinPetCount)
      ssiMinPetCount = temp_ssiPetCount[i];
    if (temp_ssiPetCount[i] > ssiMaxPetCount)
      ssiMaxPetCount = temp_ssiPetCount[i];
  }
  int localSsi = ssiid[mypet];
  ssiLocalPetCount=temp_ssiPetCount[localSsi];
#endif
#if 0
{
  std::stringstream msg;
  msg << "VMK::init()#" << __LINE__
    << " ssiLocalPetCount=" << ssiLocalPetCount;
  ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
}
#endif
  nssiid = ssiCount;
  ssiLocalPetList = new int[ssiLocalPetCount];
  int j=0;
  for (int i=0; i<ncores; i++){
    if (ssiid[i]==localSsi){
      ssiLocalPetList[j] = i;
      if (i == mypet) ssiLocalPet=j;
      ++j;
    }
  }
  set(globalResourceControl);
  // ESMCI::VMK pet -> core mapping
  lpid = new int[npets];
  pid = new int[npets];
  tid = new int[npets];
  ncpet = new int[npets];
  nadevs = new int[npets];
  cid = new int*[npets];
  for (int i=0; i<npets; i++){
    lpid[i]=i;
    pid[i]=i;
    tid[i]=0;
    ncpet[i]=1;
    nadevs[i]=0;
    cid[i] = new int[ncpet[i]];
    cid[i][0]=i;
  }
#ifdef ESMF_ACC_SOFTWARE_STACK
  int num_adevices = 0;
  num_adevices = VMAccFwGetNumDevices();
  MPI_Allgather(&num_adevices, 1, MPI_INTEGER,
                nadevs, 1, MPI_INTEGER, mpi_c);
#endif
  // device discovery
  ndevsSSI = 0; // no devices by default.
  bool ndevInit = false;
#ifdef ESMF_NVML
  // Have access to NVIDIA management library (NVML)
  nvmlReturn_t nvlmRC;
  nvlmRC = nvmlInit_v2();  // initialize NVML
  if (nvlmRC == NVML_SUCCESS){
    unsigned int nvmlDeviceCount;
    nvmlDeviceGetCount_v2(&nvmlDeviceCount);
    ndevsSSI = nvmlDeviceCount;
    ndevInit = true;
  }
#endif
#ifndef ESMF_NO_OPENACC
  // Have access to OpenACC
  int openaccDeviceCount;
  openaccDeviceCount = acc_get_num_devices(acc_device_not_host);
  if (ndevInit){
    if (openaccDeviceCount != ndevsSSI){
      int localrc;
      ESMC_LogDefault.MsgFoundError(ESMC_RC_INTNRL_BAD,
        "OpenACC device count must match already determined device count!",
        ESMC_CONTEXT, &localrc);
      throw localrc;  // bail out with exception
    }
  }else{
    ndevsSSI = openaccDeviceCount;
    ndevInit = true;
  }
#endif
#ifndef ESMF_NO_OPENMP
#ifdef ESMF_OPENMP4
  // Have access to OpenMP
  int openmpDeviceCount;
  openmpDeviceCount = omp_get_num_devices();
  if (ndevInit){
    if (openmpDeviceCount != ndevsSSI){
      int localrc;
      ESMC_LogDefault.MsgFoundError(ESMC_RC_INTNRL_BAD,
        "OpenMP device count must match already determined device count!",
        ESMC_CONTEXT, &localrc);
      throw localrc;  // bail out with exception
    }
  }else{
    ndevsSSI = openmpDeviceCount;
    ndevInit = true;
  }
#endif
#endif
  // Now that ndevsSSI is known on every PET, determine total number ndevs
  if (mpi_c_ssi_roots != MPI_COMM_NULL)
    MPI_Allreduce(&ndevsSSI, &ndevs, 1, MPI_INTEGER, MPI_SUM, mpi_c_ssi_roots);
  if (mpi_c_ssi != MPI_COMM_NULL)
    MPI_Bcast(&ndevs, 1, MPI_INTEGER, 0, mpi_c_ssi);
  // Set up global device indices for the list of SSI local devices
  ssidevs = new int[ndevsSSI];
  if (mpi_c_ssi_roots != MPI_COMM_NULL){
    // ssi roots figure out global labeling of ssi local devices
    std::vector<int> ndevsOnSSI(ssiCount);
    MPI_Allgather(&ndevsSSI, 1, MPI_INTEGER, &(ndevsOnSSI[0]), 1, MPI_INTEGER,
      mpi_c_ssi_roots);
    int devStart=0;
    for (auto i=0; i<localSsi; i++)
      devStart += ndevsOnSSI[i];
    for (auto i=0; i<ndevsSSI; i++)
      ssidevs[i] = devStart + i;
  }
  if (mpi_c_ssi != MPI_COMM_NULL){
    // ssi roots broadcast locally constructed ssidevs list to PET on same SSI
    MPI_Bcast(ssidevs, ndevsSSI, MPI_INTEGER, 0, mpi_c_ssi);
  }
  // Set the instance variables for devices for the global VM
  devCount = ndevs;
  ssiLocalDevCount = ndevsSSI;
  ssiLocalDevList = new int[ssiLocalDevCount];
  for (auto i=0; i<ssiLocalDevCount; i++)
    ssiLocalDevList[i] = i;  // default each PET associated with all local devices
  // Creating large contiguous MPI data types in support of large MPI messages
  // within the 32-bit count limit of MPI version < 4.
  int byteCount=1;
  for (auto i=0; i<(signed)customType.size(); i++){
    byteCount *= 2;
    MPI_Type_contiguous(byteCount, MPI_BYTE, &(customType[i]));
    MPI_Type_commit(&(customType[i]));
  }
  // Start epoch support in the global VM
  epochInit();
}


void VMK::finalize(int finalizeMpi){
#undef  ESMC_METHOD
#define ESMC_METHOD "ESMCI::VMK::finalize()"
  // finalize default (all MPI) virtual machine, deleting all its allocations
  epochFinal(); // close down epoch handling
  for (int k=0; k<100; k++)
    delete [] argv[k];
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_destroy(pth_mutex);
#endif
  delete pth_mutex;
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_destroy(pth_mutex2);
#endif
  delete pth_mutex2;
  if (npets==1)
    delete sendChannel[0].shmp; // covers mpiuni and mpi 1PET VM
  delete [] sendChannel;
  delete [] recvChannel;  
  while (*ipshmTop != NULL){
    if ((*ipshmTop)->auxCounter > 0)
      free((*ipshmTop)->allocation);
    ipshmAlloc *ipshmPrev = *ipshmTop;
    *ipshmTop = (*ipshmTop)->next;
    delete ipshmPrev;
  }
  delete ipshmTop;
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_destroy(ipshmMutex);
#endif
  delete ipshmMutex;
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_destroy(ipSetupMutex);
#endif
  delete ipSetupMutex;
  delete [] cpuid;
  delete [] ssiid;
  delete [] ssipe;
  delete [] ssidevs;
  delete [] lpid;
  delete [] pid;
  delete [] tid;
  delete [] ncpet;
  delete [] nadevs;
  for (int i=0; i<npets; i++)
    delete [] cid[i];
  delete [] cid;
  delete [] ssiLocalPetList;
  delete [] ssiLocalDevList;
#ifdef ESMF_NVML
  // Have access to NVIDIA management library (NVML)
  nvmlShutdown();  // shut down NVML
#endif
  // conditionally finalize MPI
  int finalized;
  MPI_Finalized(&finalized);
  if (!finalized){
    MPI_Comm_free(&mpi_c);
#if (MPI_VERSION >= 3)
    MPI_Comm_free(&mpi_c_ssi);
#endif
    if (finalizeMpi)
      MPI_Finalize();
  }
}


struct contrib_id{
  esmf_pthread_t blocker_tid; // POSIX thread id of blocker thread
  vmkt_t *blocker_vmkt;       // pointer to blocker's vmkt structure
  int mpi_pid;                // MPI rank in the context of the default VMK
  pid_t pid;                  // POSIX process id
  esmf_pthread_t tid;         // POSIX thread id
};


struct SpawnArg{
  // members which are different for each new pet
  VMK *myvm;                  // pointer to vm instance on heap
  esmf_pthread_t pthid;       // pthread id of the spawned thread
  int mypet;                  // new mypet 
  int ncontributors;          // number of pets that contribute cores to mypet
  contrib_id *contributors;   // info about the contributors to mypet
  vmkt_t vmkt;                // this pet's vmkt
  vmkt_t vmkt_extra;          // extra vmkt for this pet (sigcatcher)
  // members which are identical for all new pets
  void *(*fctp)(void *, void *);  // pointer to the user function
  // 1st (void *) points to the provided object (child of VMK class)
  // 2nd (void *) points to data that shall be passed to the user function
  int npets;                  // new number of pets
  int *lpid;
  int *pid;
  int *tid;
  int *ncpet;
  int *nadevs;
  int **cid;
  MPI_Comm mpi_c;
  MPI_Comm mpi_c_ssi;
  MPI_Comm mpi_c_ssi_roots;
  int mpi_c_freeflag;
  bool threadsflag;
  int openmphandling;
  int openmpnumthreads;
  string stdoutName;
  string stderrName;
  // device variables
  int devCount;
  int ssiLocalDevCount;
  int *ssiLocalDevList;
  // shared memory variables
  esmf_pthread_mutex_t *pth_mutex2;
  esmf_pthread_mutex_t *pth_mutex;
  int *pth_finish_count;
  VMK::comminfo *sendChannel;
  VMK::comminfo *recvChannel;
  VMK::ipshmAlloc **ipshmTop;
  esmf_pthread_mutex_t *ipshmMutex;
  esmf_pthread_mutex_t *ipSetupMutex;
  int pref_intra_ssi;
  // cargo
  void *cargo;
};


void VMK::abort(){
  // abort default (all MPI) virtual machine
  int finalized;

  char const *envRTabort = VM::getenv("ESMF_RUNTIME_ABORT_ACTION");
  if (envRTabort != NULL && strlen(envRTabort) > 0) {
    std::string RTabort(envRTabort);
    transform(RTabort.begin(), RTabort.end(), RTabort.begin(), ::toupper);
    if (RTabort == "MPI_ABORT") {
      ; // no-op, do not raise signal
    } else if (RTabort == "SIGABRT") {
      raise (SIGABRT);
    } else if (RTabort == "SIGQUIT") {
#ifdef SIGQUIT
      raise (SIGQUIT);
#else
      fprintf(stderr, "SIGQUIT not defined for ESMF_RUNTIME_ABORT_ACTION\n"
        "-> default to MPI_Abort.\n");
#endif
    } else {
      fprintf(stderr, "Invalid ESMF_RUNTIME_ABORT_ACTION setting - %s\n"
        "-> default to MPI_Abort.\n", RTabort.c_str());
    }
  }
  // signal may be caught therefore call MPI_Abort if code gets here
  MPI_Finalized(&finalized);
  if (!finalized)
    MPI_Abort(default_mpi_c, EXIT_FAILURE);
}


VMK::Affinities VMK::setAffinities(void *ssarg){
  SpawnArg *sarg = (SpawnArg *)ssarg;
  Affinities affs;
  affs.mypthid = mypthid;
#ifndef ESMF_NO_PTHREADS
#if !defined(ESMF_OS_Darwin) && !defined(ESMF_OS_Cygwin)
  // get the current thread affinity
  pthread_getaffinity_np(mypthid, sizeof(cpu_set_t), &(affs.cpuset));
  // set thread affinity
  cpu_set_t cpuset;
  CPU_ZERO(&cpuset);
  for (int i=0; i<ncpet[mypet]; i++)
    CPU_SET(ssipe[cid[mypet][i]], &cpuset);
  pthread_setaffinity_np(mypthid, sizeof(cpu_set_t), &cpuset);
#endif
#ifndef ESMF_NO_OPENMP
  // get the current omp max thread count
  affs.omp_num_threads = omp_get_max_threads();
  // OpenMP handling according to sarg->openmphandling
  if (sarg->openmphandling>0){
    // Set the number of OpenMP threads
    int numthreads = ncpet[mypet]; // default
    if (sarg->openmpnumthreads>=0)
      numthreads = sarg->openmpnumthreads;
    omp_set_num_threads(numthreads);
#ifndef ESMF_NO_OPENACC
#ifdef __NVCOMPILER
    // under NVHPC, multi-cpu-core threading (for OpenACC and standard language
    // constructs) is implemented based on OpenACC: NVHPC extended OpenACC API
    acc_set_num_cores(numthreads);
#endif
#endif
#if !defined(ESMF_OS_Darwin) && !defined(ESMF_OS_Cygwin)
    if (sarg->openmphandling>1){
#pragma omp parallel
      {
        cpu_set_t cpuset;
        CPU_ZERO(&cpuset);
        int cIndex = omp_get_thread_num()%ncpet[mypet];
        CPU_SET(ssipe[cid[mypet][cIndex]], &cpuset);
        if (sarg->openmphandling>2){
          // set affinity on all OpenMP threads
          pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
        }
      }
    }
#endif
  }
#endif
#endif
  // return stucture with original values
  return affs;
}


VMK::Redirects VMK::setRedirects(void *ssarg){
  SpawnArg *sarg = (SpawnArg *)ssarg;
  Redirects reds;
  // stdout and stderr redirect
  if (sarg->stdoutName.length()){
    reds.oldStdout = fcntl(STDOUT_FILENO, F_DUPFD, 0);  // keep access to stdout
    if (reds.oldStdout == -1){
      int localrc;
      ESMC_LogDefault.MsgFoundError(ESMC_RC_INTNRL_BAD,
        "Did not obtain a valid file descriptor", ESMC_CONTEXT, &localrc);
      throw localrc;  // bail out with exception
    }
    int newStdout = open(sarg->stdoutName.c_str(),
      O_WRONLY | O_CREAT | O_APPEND, 0644);
    if (newStdout == -1){
      int localrc;
      ESMC_LogDefault.MsgFoundError(ESMC_RC_INTNRL_BAD,
        "Did not obtain a valid file descriptor", ESMC_CONTEXT, &localrc);
      throw localrc;  // bail out with exception
    }
    dup2(newStdout, STDOUT_FILENO); // redirect
    close(newStdout);               // free up file descriptor, file stays open
  }else{
    reds.oldStdout = -1;  // indicate no redirect
  }
  if (sarg->stderrName.length()){
    reds.oldStderr = fcntl(STDERR_FILENO, F_DUPFD, 0);  // keep access to stderr
    if (reds.oldStderr == -1){
      int localrc;
      ESMC_LogDefault.MsgFoundError(ESMC_RC_INTNRL_BAD,
        "Did not obtain a valid file descriptor", ESMC_CONTEXT, &localrc);
      throw localrc;  // bail out with exception
    }
    int newStderr = open(sarg->stderrName.c_str(),
      O_WRONLY | O_CREAT | O_APPEND, 0644);
    if (newStderr == -1){
      int localrc;
      ESMC_LogDefault.MsgFoundError(ESMC_RC_INTNRL_BAD,
        "Did not obtain a valid file descriptor", ESMC_CONTEXT, &localrc);
      throw localrc;  // bail out with exception
    }
    dup2(newStderr, STDERR_FILENO); // redirect
    close(newStderr);               // free up file descriptor, file stays open
  }else{
    reds.oldStderr = -1;  // indicate no redirect
  }
  // return stucture with original values
  return reds;
}


void VMK::construct(void *ssarg){
  SpawnArg *sarg = (SpawnArg *)ssarg;
  // fill an already existing VMK object with info
  mypet=sarg->mypet;
  mypthid=sarg->pthid;
  npets = sarg->npets;
  lpid = new int[npets];
  pid = new int[npets];
  tid = new int[npets];
  ncpet = new int[npets];
  nadevs = new int[npets];
  cid = new int*[npets];
  ssiCount=0;
  std::vector<int> temp_ssiPetCount(npets);
  std::vector<int> temp_ssiid(npets);
  for (int i=0; i<npets; i++){
    lpid[i]=sarg->lpid[i];
    pid[i]=sarg->pid[i];
    tid[i]=sarg->tid[i];
    ncpet[i]=sarg->ncpet[i];
    nadevs[i]=sarg->nadevs[i];
    cid[i] = new int[ncpet[i]];
    for (int k=0; k<ncpet[i]; k++)
      cid[i][k] = sarg->cid[i][k];
    // determine ssiCount
    int j;
    for (j=0; j<i; j++)
      if (ssiid[cid[j][0]] == ssiid[cid[i][0]]) break;
    if (j==i){
      // found new ssiid
      temp_ssiid[i]=ssiCount;
      ++ssiCount;
      temp_ssiPetCount[temp_ssiid[i]] = 1;
    }else{
      // found previous ssiid
      temp_ssiid[i]=temp_ssiid[j];
      temp_ssiPetCount[temp_ssiid[i]]++;
    }
  }
  int localSsi = temp_ssiid[mypet];
  ssiMinPetCount=npets;
  ssiMaxPetCount=0;
  for (int i=0; i<ssiCount; i++){
    if (temp_ssiPetCount[i] < ssiMinPetCount)
      ssiMinPetCount = temp_ssiPetCount[i];
    if (temp_ssiPetCount[i] > ssiMaxPetCount)
      ssiMaxPetCount = temp_ssiPetCount[i];
  }
  ssiLocalPetCount=temp_ssiPetCount[localSsi];
#if 0
{
  std::stringstream msg;
  msg << "VMK::init()#" << __LINE__
    << " ssiLocalPetCount=" << ssiLocalPetCount;
  ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
}
#endif
  ssiLocalPetList = new int[ssiLocalPetCount];
  int j=0;
  for (int i=0; i<npets; i++){
    if (ssiid[cid[i][0]]==ssiid[cid[mypet][0]]){
      ssiLocalPetList[j] = i;
      if (i == mypet) ssiLocalPet=j;
      ++j;
    }
  }
  mpi_c = sarg->mpi_c;
#if (MPI_VERSION >= 3)
  mpi_c_ssi       = sarg->mpi_c_ssi;
  mpi_c_ssi_roots = sarg->mpi_c_ssi_roots;
#else
  mpi_c_ssi       = MPI_COMM_NULL;
  mpi_c_ssi_roots = MPI_COMM_NULL;
#endif

  // device management
  devCount = sarg->devCount;
  ssiLocalDevCount = sarg->ssiLocalDevCount;
  ssiLocalDevList = sarg->ssiLocalDevList;

  // pthread mutex control
  pth_mutex2 = sarg->pth_mutex2;
  pth_mutex = sarg->pth_mutex;
  pth_finish_count = sarg->pth_finish_count;
  if (mpi_thread_level<MPI_THREAD_MULTIPLE)
    mpi_mutex_flag = 1; // must use muteces around mpi comms
  else
    mpi_mutex_flag = 0; // don't need to use muteces around mpi comms
  sendChannel = sarg->sendChannel;
  recvChannel = sarg->recvChannel;
  // setup the IntraProcessSharedMemoryAllocation List
  ipshmTop = sarg->ipshmTop;
  ipshmLocalTop = *ipshmTop;
  ipshmMutex = sarg->ipshmMutex;
  ipSetupMutex = sarg->ipSetupMutex;
  // initialize the request queue
  nhandles=0;
  firsthandle=NULL;
  // preference dependent settings
  if (sarg->pref_intra_ssi == PREF_INTRA_SSI_POSIXIPC){
#ifdef ESMF_NO_POSIXIPC
    fprintf(stderr, "PREF_INTRA_SSI_POSIXIPC not supported on this platform!\n"
      "-> default into PREF_INTRA_SSI_MPI1.\n");
    sarg->pref_intra_ssi = PREF_INTRA_SSI_MPI1;
#else
    // now set up the POSIX IPC shared memory resources between pets
    // that run within the same SSI but different PID
    for (int i=0; i<npets; i++){
      // looping over all pets
      if (getSsi(i) == getSsi(mypet)){
        // found a pet under same SSI ...
        if (pid[i] != pid[mypet]){
          // ... and with different PID (which also excludes mypet!)
          // ready to set up shared memory segment using POSIX IPC
          char shm_file[80];
          int shm_fd;
          int size = sizeof(pipc_mp);
          void *shm_segment;
          // first: sendChannel
#ifdef ESMF_OS_Linux
          sprintf(shm_file, "/shm_channel_%d_%d", mypet, i);
#else
          sprintf(shm_file, "/tmp/shm_channel_%d_%d", mypet, i);
#endif
          // get a descriptor for this shared memory resource
          // which ever PET comes first will create this resource, the other
          // will just open it...
          shm_fd = shm_open(shm_file, O_RDWR | O_CREAT | O_EXCL , 0600);
          if (shm_fd == -1){
            // resource existed
            shm_fd = shm_open(shm_file, O_RDWR, 0600);
            shm_segment = mmap(NULL, size, PROT_WRITE, MAP_SHARED, shm_fd, 
            (off_t)0);
          }else{
            // resource needs to be created and sized
            ftruncate(shm_fd, size);
            shm_segment = mmap(NULL, size, PROT_WRITE, MAP_SHARED, shm_fd, 
            (off_t)0);
            strcpy(((pipc_mp *)shm_segment)->shm_name, shm_file);
            sync_reset(&((pipc_mp *)shm_segment)->shms);
          }
          // enter the address into the sendChannel
          sendChannel[i].pipcmp = (pipc_mp *)shm_segment;
          sendChannel[i].comm_type = VM_COMM_TYPE_POSIXIPC;
//fprintf(stderr, "Setting sendChannel[%d].pipcmp = %p, %p\n", i, shm_segment, MAP_FAILED);
          // then: recvChannel
#ifdef ESMF_OS_Linux
          sprintf(shm_file, "/shm_channel_%d_%d", i, mypet);
#else
          sprintf(shm_file, "/tmp/shm_channel_%d_%d", i, mypet);
#endif
          // get a descriptor for this shared memory resource
          // which ever PET comes first will create this resource, the other
          // will just open it...
          shm_fd = shm_open(shm_file, O_RDWR | O_CREAT | O_EXCL , 0600);
          if (shm_fd == -1){
            // resource existed
            shm_fd = shm_open(shm_file, O_RDWR, 0600);
            shm_segment = mmap(NULL, size, PROT_WRITE, MAP_SHARED, shm_fd, 
            (off_t)0);
          }else{
            // resource needs to be created and sized
            ftruncate(shm_fd, size);
            shm_segment = mmap(NULL, size, PROT_WRITE, MAP_SHARED, shm_fd, 
            (off_t)0);
            strcpy(((pipc_mp *)shm_segment)->shm_name, shm_file);
            sync_reset(&((pipc_mp *)shm_segment)->shms);
          }
          // enter the address into the recvChannel
          recvChannel[i].pipcmp = (pipc_mp *)shm_segment;
          recvChannel[i].comm_type = VM_COMM_TYPE_POSIXIPC;
//fprintf(stderr, "Setting recvChannel[%d].pipcmp = %p, %p\n", i, shm_segment, MAP_FAILED);
        }
      }
    }
#endif
  }
#ifdef ESMF_MPIUNI
  // don't set mpionly flag so that comm call check for commtype
  mpionly=false;
#else
  if (npets==1)
    mpionly=false;
  else{
    // determine whether we are dealing with an MPI-only VMK
    mpionly=true;  // assume this is MPI-only VMK until found otherwise
    for (int i=0; i<npets; i++)
      if (tid[i]>0) mpionly=false;    // found multi-threading PET
  }
#endif
  threadsflag = sarg->threadsflag;
  epochInit();  // start epoch support

  // need a barrier here before any of the PETs get into user code...
  //barrier();
}


void VMK::destruct(){
  // determine how many pets are of the same pid as mypet is
  int num_same_pid=0;
  for (int i=0; i<npets; i++)
    if (pid[i]==pid[mypet])
      ++num_same_pid;
  // check with the other pets under this pid where we are in wrap-up
  int last_flag=0;
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_lock(pth_mutex);
#endif
  ++(*pth_finish_count);          // increment counter
#if (VERBOSITY > 9)
  printf("wrap-up counts: %d %d\n", *pth_finish_count, num_same_pid);
#endif
  if (*pth_finish_count == num_same_pid)
    last_flag=1; // indicate that I am the last pet for this pid to wrap up...
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_unlock(pth_mutex);
#endif
  // now we know if we are the last pet for this pid
  if (last_flag){
    // mypet is the last pet of this pid to wrap up:
#if (VERBOSITY > 9)
    printf("mypet is the last one to wrap up for this pid..MPI & shared mem\n");
#endif
    //  - free the shared memory variables
#ifndef ESMF_NO_PTHREADS
    pthread_mutex_destroy(pth_mutex2);
#endif
    delete pth_mutex2;
#ifndef ESMF_NO_PTHREADS
    pthread_mutex_destroy(pth_mutex);
#endif
    delete pth_mutex;
    delete pth_finish_count;
    // free - the IntraProcessSharedMemoryAllocation List
    while (*ipshmTop != NULL){
      if ((*ipshmTop)->auxCounter > 0)
        free((*ipshmTop)->allocation);
      ipshmAlloc *ipshmPrev = *ipshmTop;
      *ipshmTop = (*ipshmTop)->next;
      delete ipshmPrev;
    }
    delete ipshmTop;
#ifndef ESMF_NO_PTHREADS
    pthread_mutex_destroy(ipshmMutex);
#endif
    delete ipshmMutex;
#ifndef ESMF_NO_PTHREADS
    pthread_mutex_destroy(ipSetupMutex);
#endif
    delete ipSetupMutex;
  }
  // only the sendChannels of all PETs free -> this also deletes recvChannels
  for (int i=0; i<npets; i++){
    if(sendChannel[i].comm_type==VM_COMM_TYPE_SHMHACK
      ||sendChannel[i].comm_type==VM_COMM_TYPE_PTHREAD
      ||sendChannel[i].comm_type==VM_COMM_TYPE_MPIUNI){
      // intra-process shared memory structure to be deleted
      shared_mp *shmp=sendChannel[i].shmp;
#ifndef ESMF_NO_PTHREADS
      if(sendChannel[i].comm_type==VM_COMM_TYPE_PTHREAD){
        pthread_mutex_destroy(&(shmp->mutex1));
        pthread_cond_destroy(&(shmp->cond1));
        pthread_mutex_destroy(&(shmp->mutex2));
        pthread_cond_destroy(&(shmp->cond2));
      }
#endif
#if (VERBOSITY > 9)
      printf("deleting shmp=%p for sendChannel[%d], mypet=%d\n", 
        shmp, i, mypet);
#endif
      delete shmp;
    }else if (sendChannel[i].comm_type==VM_COMM_TYPE_POSIXIPC){
#ifdef ESMF_NO_POSIXIPC
#else
      pipc_mp *pipcmp=sendChannel[i].pipcmp;
      char shm_name[80];
      strcpy(shm_name, pipcmp->shm_name);
      munmap((void *)pipcmp, sizeof(pipc_mp));
      shm_unlink(shm_name);
#if (VERBOSITY > 9)
      printf("deleting pipcmp=%p (%s) for sendChannel[%d], mypet=%d\n", 
        pipcmp, shm_name, i, mypet);
#endif
#endif
    }
  }
  delete [] lpid;
  delete [] pid;
  delete [] tid;
  delete [] ncpet;
  delete [] nadevs;
  for (int i=0; i<npets; i++)
    delete [] cid[i];
  delete [] cid;
  delete [] ssiLocalPetList;
}


static void enter_callback(SpawnArg *sarg, void *mutex){
  vmkt_t *vmkt = &(sarg->vmkt);
  VMK *vm = sarg->myvm;
#ifndef ESMF_NO_PTHREADS
  esmf_pthread_mutex_t *pmutex = (esmf_pthread_mutex_t *)mutex;
#endif

#ifdef VM_PETMANAGEMENTLOG_on
  {
    std::stringstream msg;
    msg << "enter_callback()#" << __LINE__
      << " tid=" << vmkt->tid << ": pthread=" << pthread_self()
      << " just before user code callback, pid=" << getpid();
    ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
  }
#endif

  // set affinities and OpenMP details according to plan, keep current settings
  VMK::Affinities oldAffs = vm->setAffinities((void *)sarg);

  // set stdout and stderr redirects
  VMK::Redirects oldReds = VMK::setRedirects((void *)sarg);

  // call the function pointer with the new VMK as its argument
  // this is where we finally enter the user code again...
  if (vmkt->arg==NULL)
    sarg->fctp((void *)vm, sarg->cargo);
  else
    sarg->fctp((void *)vm, vmkt->arg);
  //vmkt->routine(vmkt->arg);

#ifdef VM_PETMANAGEMENTLOG_on
  {
    std::stringstream msg;
    msg << "enter_callback()#" << __LINE__
      << " tid=" << vmkt->tid << ": pthread=" << pthread_self()
      << " has returned from user code callback, pid=" << getpid();
    ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
  }
#endif

  // before pet terminates it must send a signal indicating that core is free
  for (int i=0; i<sarg->ncontributors; i++){
    // send signal to the _other_ process
#ifdef VM_PETMANAGEMENTLOG_on
    {
      std::stringstream msg;
      msg << "enter_callback()#" << __LINE__
        << " tid=" << vmkt->tid << ": pthread=" << pthread_self()
        << " sending kill(VM_SIG1) to PID="
        << sarg->contributors[i].pid;
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
    }
#endif
#if !defined (ESMF_OS_MinGW)
    kill(sarg->contributors[i].pid, VM_SIG1);
#else
    // TODO: Windows equivalent, perhaps using TerminateProcess
#endif
#ifdef VM_PETMANAGEMENTLOG_on
    {
      std::stringstream msg;
      msg << "enter_callback()#" << __LINE__
        << " tid=" << vmkt->tid << ": pthread=" << pthread_self()
        << " done sending kill(VM_SIG1) to PID="
        << sarg->contributors[i].pid;
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
    }
#endif
    // which ever thread of the other process woke up will try to receive tid
#ifndef ESMF_NO_PTHREADS
    if (vm->mpi_thread_level<MPI_THREAD_MULTIPLE)
      pthread_mutex_lock(pmutex);
#endif
    MPI_Send(&(sarg->contributors[i].blocker_vmkt),
      sizeof(vmkt_t *), MPI_BYTE, sarg->contributors[i].mpi_pid,
      VM_TID_MPI_TAG, vm->default_mpi_c);
#ifndef ESMF_NO_PTHREADS
    if (vm->mpi_thread_level<MPI_THREAD_MULTIPLE)
      pthread_mutex_unlock(pmutex);
#endif
  }

  // reset previous stdout and stderr settings
  oldReds.reset();

  // reset previous affinities and OpenMP settings
  oldAffs.reset();

}


static void *vmk_spawn(void *arg){
  // vmkt's first level spawn function, includes the catch/release loop
  // typecast the argument into the type it really is:
  SpawnArg *sarg = (SpawnArg *)arg;
  vmkt_t *vmkt = &(sarg->vmkt);
#ifdef VM_PETMANAGEMENTLOG_on
  {
    std::stringstream msg;
    msg << "vmk_spawn()#" << __LINE__
      << " tid=" << vmkt->tid << ": pthread=" << pthread_self()
      << " pid=" << getpid() << " starting prologue negotiation.";
    ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
  }
#endif
  // now use vmkt features to prepare for catch/release loop (back-sync)
  // - part 1
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_lock(&(vmkt->mut0));        // back-sync #1 ...
  pthread_cond_signal(&(vmkt->cond0));      // . back-sync #1 .
  pthread_mutex_unlock(&(vmkt->mut0));      // ... back-sync #1
#endif
  // now we know that vmkt_create is past pthread_create()
  // ... and we are between back-sync #1 and #2
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_lock(&(vmkt->mut1));        // prepare this thread's mutex
  pthread_mutex_lock(&(vmkt->mut_extra1));  // prepare this thread's mutex
#endif
  // fill in the tid for this thread
  sarg->pthid = sarg->vmkt.tid;
  // obtain reference to the vm instance on heap
  VMK *vm = sarg->myvm;
  // setup the pet section in this vm instance
  vm->construct((void *)sarg);
  // note: The VM above must be constructed _before_ back-sync'ing #2 to
  //       vmkt_create in order to assure that the entries in the VM are valid!
  // now use vmkt features to prepare for catch/release loop (back-sync)
  // - part 2
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_lock(&(vmkt->mut_extra2));    // back-sync #2 ...
  pthread_cond_signal(&(vmkt->cond_extra2));  // . back-sync #2 .
  pthread_mutex_unlock(&(vmkt->mut_extra2));  // ... back-sync #2
#endif
#ifdef VM_PETMANAGEMENTLOG_on
  {
    std::stringstream msg;
    msg << "vmk_spawn()#" << __LINE__
      << " tid=" << vmkt->tid << ": pthread=" << pthread_self()
      << " pid=" << getpid() << " finished prologue negotiation.";
    ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
  }
#endif
  volatile int *f = &(vmkt->flag);
  // now enter the catch/release loop
  for(;;){
    //sleep(2); // put this in the code to verify that earlier received signals
    // will be pending on a per thread basis...
#ifdef VM_PETMANAGEMENTLOG_on
    {
      std::stringstream msg;
      msg << "vmk_spawn()#" << __LINE__
        << " tid=" << vmkt->tid << ": pthread=" << pthread_self()
        << " going to wait for release, pid=" << getpid();
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
    }
#endif
#ifndef ESMF_NO_PTHREADS
    pthread_cond_wait(&(vmkt->cond1), &(vmkt->mut1));
#endif
#ifdef VM_PETMANAGEMENTLOG_on
    {
      std::stringstream msg;
      msg << "vmk_spawn()#" << __LINE__
        << " tid=" << vmkt->tid << ": pthread=" << pthread_self()
        << " was released, pid=" << getpid();
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
    }
#endif

    if (*f==1) break; // check whether this was a wrap up call

    //vm.barrier();

    // call back into user code, and return PEs to their contributing PETs
    enter_callback(sarg, &(vmkt->mut0));

    // now signal to parent thread that child is done with its work
#ifndef ESMF_NO_PTHREADS
    pthread_mutex_lock(&(vmkt->mut0)); // wait until parent has reached "catch"
    pthread_cond_signal(&(vmkt->cond0)); // then signal that child is done
    pthread_mutex_unlock(&(vmkt->mut0)); // release the mutex lock for parent
#endif
  }
  // wrap-up...
  vm->epochFinal();  // close down epoch handling
  vm->destruct();    // destroy this vm instance
  // when returning from this procedure this pet will terminate
  return NULL;
}


static void *vmk_sigcatcher(void *arg){
#undef  ESMC_METHOD
#define ESMC_METHOD "ESMCI::vmk_sigcatcher()"
  // vmkt's first level spawn function, includes the catch/release loop
  // typecast the argument into the type it really is:
  SpawnArg *sarg = (SpawnArg *)arg;
#if (VERBOSITY > 5)
  fprintf(stderr, "hello from within vmk_sigcatcher\n");
#endif
  // need this for waking up blocker
  vmkt_t *blocker_vmkt;
  // now use vmkt features to prepare for catch/release loop (back-sync)
  // - part 1
  vmkt_t *vmkt = &(sarg->vmkt_extra);
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_lock(&(vmkt->mut0));        // back-sync #1 ...
  pthread_cond_signal(&(vmkt->cond0));      // . back-sync #1 .
  pthread_mutex_unlock(&(vmkt->mut0));      // ... back-sync #1
#endif
  // now we know that vmkt_create is past pthread_create()
  // ... and we are between back-sync #1 and #2
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_lock(&(vmkt->mut1));        // prepare this thread's mutex
  pthread_mutex_lock(&(vmkt->mut_extra1));  // prepare this thread's mutex
#endif
  volatile int *f = &(vmkt->flag);
  // since LinuxThreads (pre NPTL) have the problem that each thread reports
  // its own PID instead the same for each thread, which would be the posix
  // behavior, we need to get the sigcatcher's pid to send it over to the
  // other process.
  pid_t pid = getpid();
  vmkt->arg = (void *)&pid;
  // more preparation
  VMK vm;  // need a handle to a VM object to access the static members
  if (!vm.pre_mpi_init){
    int localrc;
    ESMC_LogDefault.MsgFoundError(ESMC_RC_INTNRL_BAD,
      "Cannot safely use signals inside VMKernel because signal handling was "
      "not installed before MPI was initialized.", ESMC_CONTEXT, &localrc);
    throw localrc;  // bail out with exception
  }
#ifdef ESMF_NO_SIGNALS
  int localrc;
  ESMC_LogDefault.MsgFoundError(ESMC_RC_INTNRL_BAD,
    "Need signals to support advanced threading options inside VMKernel",
    ESMC_CONTEXT, &localrc);
  throw localrc;  // bail out with exception
#else
  sigset_t sigs_to_catch;
  sigemptyset(&sigs_to_catch);
  sigaddset(&sigs_to_catch, VM_SIG1);
#endif
  int caught;
  MPI_Status mpi_s;
  // now use vmkt features to prepare for catch/release loop (back-sync)
  // - part 2
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_lock(&(vmkt->mut_extra2));    // back-sync #2 ...
  pthread_cond_signal(&(vmkt->cond_extra2));  // . back-sync #2 .
  pthread_mutex_unlock(&(vmkt->mut_extra2));  // ... back-sync #2
#endif
  // now enter the catch/release loop
  for(;;){
    //sleep(2); // put this in the code to verify that earlier received signals
    // will be pending on a per thread basis...
#ifdef VM_PETMANAGEMENTLOG_on
    {
      std::stringstream msg;
      msg << "vmk_sigcatcher()#" << __LINE__
        << " tid=" << vmkt->tid << ": pthread=" << pthread_self()
        << " going to wait for release, pid=" << getpid();
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
    }
#endif
#ifndef ESMF_NO_PTHREADS
    pthread_cond_wait(&(vmkt->cond1), &(vmkt->mut1));
#endif
#ifdef VM_PETMANAGEMENTLOG_on
    {
      std::stringstream msg;
      msg << "vmk_sigcatcher()#" << __LINE__
        << " tid=" << vmkt->tid << ": pthread=" << pthread_self()
        << " was released, pid=" << getpid();
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
    }
#endif

  if (*f==1) break; // check whether this was a wrap up call

  // This is a signal catcher. Its job is to catch a signal from other processes
  // indicating that cores have become available again. When such a signal has
  // been received the sigcatcher will communicate with the signaling process
  // via MPI and receive the actual pthread_id that need to be awoken on this
  // process, which is actually a blocker thread which then will wrap up and 
  // by that indicate that the resource has been made available again.
  // suspend thread until a signal arrives

#ifdef VM_PETMANAGEMENTLOG_on
    {
      std::stringstream msg;
      msg << "vmk_sigcatcher()#" << __LINE__
        << " tid=" << vmkt->tid << ": pthread=" << pthread_self()
        << " going to sleep in sigwait(), pid=" << getpid();
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
    }
#endif

#ifndef ESMF_NO_SIGNALS
  sigwait(&sigs_to_catch, &caught);
#endif
  
#ifdef VM_PETMANAGEMENTLOG_on
    {
      std::stringstream msg;
      msg << "vmk_sigcatcher()#" << __LINE__
        << " tid=" << vmkt->tid << ": pthread=" << pthread_self()
        << " pid=" << getpid() << " was woken up by signal: " << caught;
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
    }
#endif

  // this signal was received from a thread running under another process
  // receive the thread id of the blocker thread that needs to be woken up
#ifdef VM_PETMANAGEMENTLOG_on
    {
      std::stringstream msg;
      msg << "vmk_sigcatcher()#" << __LINE__
        << " tid=" << vmkt->tid << ": pthread=" << pthread_self()
        << " pid=" << getpid() << " now calling MPI_Recv() for MPI_ANY_SOURCE.";
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
    }
#endif
  MPI_Recv(&blocker_vmkt, sizeof(vmkt_t *), MPI_BYTE, MPI_ANY_SOURCE, 
    VM_TID_MPI_TAG, vm.default_mpi_c, &mpi_s);
#ifdef VM_PETMANAGEMENTLOG_on
    {
      std::stringstream msg;
      msg << "vmk_sigcatcher()#" << __LINE__
        << " tid=" << vmkt->tid << ": pthread=" << pthread_self()
        << " pid=" << getpid() << " returned from MPI_Recv() for MPI_ANY_SOURCE.";
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
    }
#endif
    
#ifdef WITHBLOCKER_on
  // now wake up the correct blocker thread within this pid
#ifdef VM_PETMANAGEMENTLOG_on
    {
      std::stringstream msg;
      msg << "vmk_sigcatcher()#" << __LINE__
        << " tid=" << vmkt->tid << ": pthread=" << pthread_self()
        << " pid=" << getpid() << " waking up blocker thread.";
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
    }
#endif
#ifndef ESMF_NO_PTHREADS
    pthread_mutex_lock(&(blocker_vmkt->mut_extra1));
    pthread_cond_signal(&(blocker_vmkt->cond_extra1));
    pthread_mutex_unlock(&(blocker_vmkt->mut_extra1));
#endif
#ifdef VM_PETMANAGEMENTLOG_on
    {
      std::stringstream msg;
      msg << "vmk_sigcatcher()#" << __LINE__
        << " tid=" << vmkt->tid << ": pthread=" << pthread_self()
        << " pid=" << getpid() << " done waking up blocker thread.";
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
    }
#endif
#endif

    // this sigcatcher has done its job and is allowed to recycle to be caught..
  
    // now signal to parent thread that child is done with its work
#ifndef ESMF_NO_PTHREADS
    pthread_mutex_lock(&(vmkt->mut0)); // wait until parent has reached "catch"
    pthread_cond_signal(&(vmkt->cond0)); // then signal that child is done
    pthread_mutex_unlock(&(vmkt->mut0)); // release the mutex lock for parent
#endif
  }
  return NULL;
}


static void *vmk_block(void *arg){
  // vmkt's first level spawn function, includes the catch/release loop
  // typecast the argument into the type it really is:
  SpawnArg *sarg = (SpawnArg *)arg;
#if (VERBOSITY > 5)
  fprintf(stderr, "hello from within vmk_block\n");
#endif
  // now use vmkt features to prepare for catch/release loop (back-sync)
  // - part 1
  vmkt_t *vmkt = &(sarg->vmkt);
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_lock(&(vmkt->mut0));        // back-sync #1 ...
  pthread_cond_signal(&(vmkt->cond0));      // . back-sync #1 .
  pthread_mutex_unlock(&(vmkt->mut0));      // ... back-sync #1
#endif
  // now we know that vmkt_create is past pthread_create()
  // ... and we are between back-sync #1 and #2
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_lock(&(vmkt->mut1));        // prepare this thread's mutex
  pthread_mutex_lock(&(vmkt->mut_extra1));  // prepare this thread's mutex
#endif
  // fill in the tid for this thread
  sarg->pthid = sarg->vmkt.tid;
  // now use vmkt features to prepare for catch/release loop (back-sync)
  // - part 2
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_lock(&(vmkt->mut_extra2));    // back-sync #2 ...
  pthread_cond_signal(&(vmkt->cond_extra2));  // . back-sync #2 .
  pthread_mutex_unlock(&(vmkt->mut_extra2));  // ... back-sync #2
#endif
#if (VERBOSITY > 5)
  fprintf(stderr, "blocker is past back-sync #2\n");  
#endif
  volatile int *f = &(vmkt->flag);
  // now enter the catch/release loop
  for(;;){
    //sleep(2); // put this in the code to verify that earlier received signals
    // will be pending on a per thread basis...
#if (VERBOSITY > 5)
    fprintf(stderr,"vmk_block: tid=%d: %d going to wait for release, pid: %d\n",
      vmkt->tid, pthread_self(), getpid());
#endif
#ifndef ESMF_NO_PTHREADS
    pthread_cond_wait(&(vmkt->cond1), &(vmkt->mut1));
#endif
#if (VERBOSITY > 5)
    fprintf(stderr,"vmk_block: tid=%d: %d was released, pid:%d\n", vmkt->tid,
      pthread_self(), getpid());
#endif
    
    if (*f==1) break; // check whether this was a wrap up call


    // This blocker thread is responsible for staying alive until resources,
    // i.e. cores, become available to the contributing pet. The contributing
    // pet is blocked (asynchonously) via pthread_cond_wait().
#ifdef VM_PETMANAGEMENTLOG_on
    {
      std::stringstream msg;
      msg << "vmk_block()#" << __LINE__
        << " pthread=" << pthread_self()
        << " going to sleep until sigcatcher wakes me up, pid=" << getpid();
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
    }
#endif

    // suspend this thread until awoken by one of the sigcatcher threads  
#ifndef ESMF_NO_PTHREADS
    pthread_cond_wait(&(vmkt->cond_extra1), &(vmkt->mut_extra1));
#endif

#ifdef VM_PETMANAGEMENTLOG_on
    {
      std::stringstream msg;
      msg << "vmk_block()#" << __LINE__
        << " pthread=" << pthread_self()
        << " awoken by sigcatcher, now get caught..., pid=" << getpid();
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
    }
#endif
    // once the signal has been received from a sigcatcher the blocker can 
    // go into the catch section before returning to wait for release

    // now signal to parent thread that child is done with its work
#ifndef ESMF_NO_PTHREADS
    pthread_mutex_lock(&(vmkt->mut0)); // wait until parent has reached "catch"
    pthread_cond_signal(&(vmkt->cond0)); // then signal that child is done
    pthread_mutex_unlock(&(vmkt->mut0)); // release the mutex lock for parent
#endif
  }
  return NULL;
}


void *VMK::startup(class VMKPlan *vmp, void *(fctp)(void *, void *),
  void *cargo, int *rc){
#if (VERBOSITY > 9)
  vmp->vmkplan_print();
#endif
#ifdef VM_MEMLOG_on
      VM::logMemInfo(std::string("VMK::startup():1.0"));
#endif
  // enter a vm derived from current vm according to the VMKPlan
  // need as many spawn_args as there are threads to be spawned from this pet
  // this is so that each spawned thread does not have to be worried about this
  // info to disappear while still accessing it
  int at_least_1 = vmp->spawnflag[mypet];
  if (at_least_1 < 1)
    at_least_1 = 1;
  SpawnArg *sarg = new SpawnArg[at_least_1];
  // set rc to indicate "no error". pthread_create() (if used) will set rc below
  *rc = 0;
  // first handle the simple case of using the parent VM
  if (vmp->parentVMflag){
    sarg[0].myvm = this;
    sarg[0].fctp = fctp;
    sarg[0].cargo = cargo;
    return sarg;
  }
#ifdef VM_MEMLOG_on
      VM::logMemInfo(std::string("VMK::startup():2.0"));
#endif
  // now:
  //    sarg[] has as many elements as mypet spawns threads, but at least one
  // next, allocate as many vm objects off the heap as there will be spawned
  // next, set pointers in sarg to the VMK instances on the heap
  for (int i=0; i<vmp->spawnflag[mypet]; i++){
    if (vmp->myvms == NULL){
      fprintf(stderr, "VM_ERROR: No vm objects provided.\n");
      MPI_Abort(default_mpi_c, 0);
    }
    sarg[i].myvm = vmp->myvms[i];
  }
#ifdef VM_MEMLOG_on
      VM::logMemInfo(std::string("VMK::startup():3.0"));
#endif
  // next, determine new_npets and new_mypet_base ...
  int new_mypet_base=0;
  volatile int new_npets=0;   // volatile to prevent precompute of log10() below
                              // in optimized mode or else might trigger SIGFPE
                              // for new_npets==1 if user code sets FPE trapping
  int found_my_pet_flag = 0;
  for (int ii=0; ii<npets; ii++){
    int i = vmp->petlist[ii];   // indirection to preserve petlist order
    new_npets += vmp->spawnflag[i];
    if (mypet == i) found_my_pet_flag = 1;
    if (!found_my_pet_flag){
      new_mypet_base += vmp->spawnflag[i];
    }
  }
#ifdef VM_MEMLOG_on
      VM::logMemInfo(std::string("VMK::startup():4.0"));
#endif
  // now:
  //    new_npets is equal to the total number of pets in the new VMK
  //    new_mypet_base is the index of the first new pet that mypet will spawn
  // next, allocate temporary arrays ...
  vector<int> new_lpid(new_npets);
  vector<int> new_pid(new_npets);
  vector<int> new_tid(new_npets);
  vector<int> new_ncpet(new_npets);
  vector<int> new_nadevs(new_npets);
  vector<int> new_ncontributors(new_npets);
  int **new_cid = new int*[new_npets];
  contrib_id **new_contributors = new contrib_id*[new_npets];
  // local variables, unallocated yet...
  esmf_pthread_mutex_t *new_pth_mutex2;
  esmf_pthread_mutex_t *new_pth_mutex;
  int *new_pth_finish_count;
  ipshmAlloc **new_ipshmTop;
  esmf_pthread_mutex_t *new_ipshmMutex;
  esmf_pthread_mutex_t *new_ipSetupMutex;
  // utility variables that will be used beyond the next i-loop
  int num_diff_pids=0;  // total number of different pids/lpids in new VMK
  // utility arrays and variables used only during the next i-loop
  int *keep_max_tid = new int[npets]; // sum of threads that will be spawned
  int new_petid=0;      // used for keeping track of new_petid in loop
  // next, run through all current pets and check the VMKPlan ...
  // inside the following loop pet "i" will be refered to as "this pet"
#ifdef VM_MEMLOG_on
      VM::logMemInfo(std::string("VMK::startup():5.0"));
#endif
  for (int ii=0; ii<npets; ii++){
    int i = vmp->petlist[ii];   // indirection to preserve petlist order
    // get the last max_tid count of a pet with same pid
    int local_tid = 0;
    for (int j=0; j<ii; j++)
      if (pid[j]==pid[ii])
        local_tid = keep_max_tid[j];
    // now:
    //    local_tid is the tid index for the first pet this pet might spawn
    // next, if this pet spawns determine whether a previous spaner had same pid
    int temp_lpid;
    if (vmp->spawnflag[i]){
      int j;
      // check all previous spawners
      for (j=0; j<new_petid; j++)
        if (new_pid[j]==pid[i]) break;
      if (j<new_petid){
        // found same pid in previous spawner
        temp_lpid=new_lpid[j];   // carry over the new_lpid determined before
      }else{
        // no previous spawner had that same pid
        temp_lpid=num_diff_pids; // new lpid for spawned pets from this pet
        ++num_diff_pids;  // increment the total number of different pids in new
      }
    }
    // now:
    //    temp_lpid is the lpid for all new pets spawned by this pet
    // next, handle this pet amd fill in info for _all_ the threads it spawns
    for (int j=0; j<vmp->spawnflag[i]; j++){
      // here j is the counter over threads this pet spawns
      new_lpid[new_petid]=temp_lpid;  // new lpid is that previously determined
      new_pid[new_petid]=pid[i];      // new pid is equal to that of this pet
      new_tid[new_petid]=local_tid;   // new tid is continuous count per pid
      // next, determine how many cores the new pet will have & its contributors
      new_ncpet[new_petid]=0;         // reset the counter
      new_nadevs[new_petid]=nadevs[i];         // copy the number of acc devs
      new_ncontributors[new_petid]=0; // reset the counter
      for (int kk=0; kk<npets; kk++){
        int k = vmp->petlist[kk];   // indirection to preserve petlist order
        if (vmp->contribute[k]==i && vmp->cspawnid[k]==j){
          // pet k contributes to this pet's spawned thread number j
          new_ncpet[new_petid]+=ncpet[k]; // add in all the cores from pet k
          if (k!=i){
            // this contribution came from _another_ pet
            ++new_ncontributors[new_petid]; // increase count of contributors
          }
        }
      }
      // now:
      //    new_lpid[new_petid] is valid lpid of new pet
      //    new_pid[new_petid] is valid pid of new pet
      //    new_tid[new_petid] is valid tid of new pet
      //    new_ncpet[new_petid] is valid number of cores held by new pet
      //    new_ncontributors[new_petid] is valid number of contributor pets
      // next, allocate and fill 2nd dimension of new_cid and new_contributors
      new_cid[new_petid] = new int[new_ncpet[new_petid]];
      new_contributors[new_petid] = 
        new contrib_id[new_ncontributors[new_petid]];
      int ncpet_counter=0;      // reset core counter
      int ncontrib_counter=0;   // reset contributor counter
      // loop over all current pets and see how they contribute to this pet
      for (int kk=0; kk<npets; kk++){
        int k = vmp->petlist[kk];  // indirection to preserve petlist order
        if (vmp->contribute[k]==i && vmp->cspawnid[k]==j){
          // found a contributor pet (k) which contributes cores
          // to this pet (i) for its spawned thread (j)
          // next, determine the contrib_id of pet (k) and share info with pet i
          if (k!=i){
            // only if contributor pet (k) is different from this receiver pet i
            if (mypet==k){
              // mypet (k) contributes to this pet (i)
              // mypet does not spawn but contributes -> spawn blocker thread
              *rc = vmkt_create(&(sarg[0].vmkt), vmk_block, (void *)&sarg[0],
                true, vmp->minStackSize);  // service thread
              if (*rc) return NULL;  // could not create pthread -> bail out
              // also spawn sigcatcher thread
              *rc = vmkt_create(&(sarg[0].vmkt_extra), vmk_sigcatcher,
                (void *)&sarg[0], true, vmp->minStackSize);  // service thread
#if (VERBOSITY > 5)
              fprintf(stderr, "parent thread is back from vmkt_create()s "
                "for vmk_block and vmk_sigcatcher\n");
#endif
              if (*rc) return NULL;  // could not create pthread -> bail out
              // fill in the info about mypet contibuting...
              new_contributors[new_petid][ncontrib_counter].blocker_tid =
                sarg[0].pthid;
              new_contributors[new_petid][ncontrib_counter].blocker_vmkt =
                &(sarg[0].vmkt);
              new_contributors[new_petid][ncontrib_counter].mpi_pid =
                pid[mypet];
              new_contributors[new_petid][ncontrib_counter].pid = getpid();
#ifndef ESMF_NO_PTHREADS
              new_contributors[new_petid][ncontrib_counter].tid =
                pthread_self();
#else
              new_contributors[new_petid][ncontrib_counter].tid = 0;
#endif
              // LinuxThreads (pre-NPTL) have the problem that each thread
              // comes with a different PID. Thus the stored PID must be that
              // of the sigcatcher in order to work. For correct posix behavior
              // that pid will be identical to the one out of the parent thread
              // obtained with getpid() above, and thus won't break posix!
              // The sigcatcher will have put the correct pid into the
              // vmkt_extra.arg during its vmkt_create() call, so we can
              // simply pull it out of there and replace the pid member.
              new_contributors[new_petid][ncontrib_counter].pid = 
                *(pid_t *)sarg[0].vmkt_extra.arg;
              // send contributor info over to this pet (i) that receives cores
#if (VERBOSITY > 5)
              fprintf(stderr, "sending...\n");
#endif
              send(&new_contributors[new_petid][ncontrib_counter],
                sizeof(contrib_id), i);
#if (VERBOSITY > 5)
              fprintf(stderr, "send off contrib_id for later wake-up signal: \n"
                " blocker_tid: %d\n mpi_pid: %d\n pid: %d\n tid: %d\n",
                new_contributors[new_petid][ncontrib_counter].blocker_tid,
                new_contributors[new_petid][ncontrib_counter].mpi_pid,
                new_contributors[new_petid][ncontrib_counter].pid,
                new_contributors[new_petid][ncontrib_counter].tid);
#endif
            }else if (mypet==i){
              // mypet is this pet (i)-> receiver of cores from _another_ pet k
#if (VERBOSITY > 5)
              fprintf(stderr, "receiving...\n");
#endif
              recv(&new_contributors[new_petid][ncontrib_counter],
                sizeof(contrib_id), k);
#if (VERBOSITY > 5)
              fprintf(stderr, "received contrib_id for later wake-up signal: \n"
                " blocker_tid: %d\n mpi_pid: %d\n pid: %d\n tid: %d\n",
                new_contributors[new_petid][ncontrib_counter].blocker_tid,
                new_contributors[new_petid][ncontrib_counter].mpi_pid,
                new_contributors[new_petid][ncontrib_counter].pid,
                new_contributors[new_petid][ncontrib_counter].tid);
#endif
            }else{
              // mypet has nothing to do with this contribution of cores
            }
            ++ncontrib_counter;   // increment the counter of contributing pets
          }
          // fill in the cores that will be contributed
          for (int l=0; l<ncpet[k]; l++){
            new_cid[new_petid][ncpet_counter]=cid[k][l];
            ++ncpet_counter;
          }
        }
      }
      // advance to the next new thread (j) spawned by current pet (i)
      // which will become pet (new_petid) in new VMK
      ++new_petid;
      ++local_tid;
    }
    // keep record of how high local_tid counted for the pid of this pet
    keep_max_tid[i] = local_tid;
  }
#ifdef VM_MEMLOG_on
      VM::logMemInfo(std::string("VMK::startup():6.0"));
#endif
  // collect garbage of temporary arrays from previous i-loop
  delete [] keep_max_tid;
  // now:
  //    new_lpid[new_petid] is valid lpid of new pet
  //    new_pid[new_petid] is valid pid of new pet
  //    new_tid[new_petid] is valid tid of new pet
  //    new_ncpet[new_petid] is valid number of cores held by new pet
  //    new_ncontributors[new_petid] is valid number of contributor pets
  //    new_cid[new_petid][] holds valid core indices
  //    new_contributors[new_petid][] holds valid contrib_id's for pairs
  //    num_diff_pids is the total number of different pids in new VMK
#if (VERBOSITY > 9)
  printf(">>>>>>>>> num_diff_pids for new VMK = %d\n", num_diff_pids);
#endif
  //
#ifdef VM_MEMLOG_on
      VM::logMemInfo(std::string("VMK::startup():7.0"));
#endif
  // next, set up temporary arrays lpid_list and pet_list to facilitate 
  // MPI_Comm creation and shared memory allocation for new VMK
  vector<vector<int>> lpid_list(2);
  lpid_list[0].resize(num_diff_pids); // this dimension holds lpids
  lpid_list[1].resize(num_diff_pids); // this dimension holds number of pets
  vector<vector<int>> pet_list(num_diff_pids);
  for (int i=0; i<num_diff_pids; i++){
    lpid_list[0][i] = -1;  // invalidate the lpid entry
    lpid_list[1][i] = 0;   // no pets associated yet
  }
  for (int ii=0; ii<npets; ii++){
    int i = vmp->petlist[ii];     // indirection to preserve petlist order
    if (vmp->spawnflag[i]){
      // this pet will spawn, so look if its lpid has already been recorded
      int j;
      for (j=0; j<num_diff_pids; j++)
        if (lpid_list[0][j]==lpid[i] || lpid_list[0][j]==-1) break;
      lpid_list[0][j] = lpid[i];  // store lpid (does not matter to overwrite)
      pet_list[j].push_back(i);
      ++lpid_list[1][j];         // increment pet count for this pid
    }
  }

#ifdef VM_MEMLOG_on
      VM::logMemInfo(std::string("VMK::startup():8.0"));
#endif
#if (VERBOSITY > 9)
  printf("finished setting up lpid_list and pet_list\n");
#endif
  
  // now:
  //    lpid_list[0][] list of current lpids with at least one pet spawning
  //    lpid_list[1][] associated list indicating how many pets will spawn
  //    pet_list[][] list of current pets that spawn
  
  // next, determine how many new pets are going to be in the thread group
  // that's running in the same VAS as mypet
  int mypetNewThreadGroupSize = 0;  // reset
  for (int i=0; i<num_diff_pids; i++){
    for (int j=0; j<lpid_list[1][i]; j++){
      if (mypet == pet_list[i][j]){
        for (int pet=0; pet<new_npets; pet++)
          if (pid[mypet] == new_pid[pet])
            ++mypetNewThreadGroupSize;
      }
    }
  }
  
#ifdef VM_MEMLOG_on
      VM::logMemInfo(std::string("VMK::startup():9.0"));
#endif
  // A new_commarray will be allocated for every PET that runs in a VAS
  // that is going to have threads in the new VMK.
  // The new_commarray is a temporary data structure that will be deleted
  // for every PET at the end of this routine.
  comminfo **new_commarray;
  bool new_commarray_delete_flag = false; // reset
  if (mypetNewThreadGroupSize){
    new_commarray_delete_flag = true; // set
    new_commarray = new comminfo*[mypetNewThreadGroupSize];
    for (int i=0; i<mypetNewThreadGroupSize; i++){
      new_commarray[i] = new comminfo[mypetNewThreadGroupSize];
    }
  }
  
#ifdef VM_MEMLOG_on
      VM::logMemInfo(std::string("VMK::startup():10.0"));
#endif
  // the new MPI group will be derived from the mpi_g_part group so there is an
  // additional level of indirection here
  int *grouplist = new int[num_diff_pids];
  for (int i=0; i<num_diff_pids; i++){
    grouplist[i] = vmp->lpid_mpi_g_part_map[lpid_list[0][i]];
  }
  
#ifdef VM_MEMLOG_on
      VM::logMemInfo(std::string("VMK::startup():11.0"));
#endif
  // setting up MPI communicators is a collective MPI communication call
  // thus it requires that exactly one pet of each process running in the 
  // current VMK makes that call, even if this process will not participate
  // in the new VMK...
  MPI_Comm new_mpi_c;
  MPI_Comm new_mpi_c_ssi;
  MPI_Comm new_mpi_c_ssi_roots;
  int devCount;
  int ssiLocalDevCount;
  int *ssiLocalDevList;

#ifdef VM_MEMLOG_on
      VM::logMemInfo(std::string("VMK::startup():12.0"));
#endif
  int foundfirstflag=0;
  int foundfirstpet;
  int mylpid = lpid[mypet];
  sarg[0].mpi_c_freeflag = 0; // invalidate on all PETs
  for (int ii=0; ii<vmp->nplist; ii++){
    int i = vmp->petlist[ii];     // indirection to preserve petlist order
    if (mylpid == lpid[i]){
      // found lpid match
      if (!foundfirstflag){
        // found first pet with that spawns under this lpid
        foundfirstflag = 1;
        foundfirstpet = i;
      }
      if (mypet == i){
        // I am this pet
        if (foundfirstpet == i){
          // I am the first under this lpid and must create communicator
          MPI_Group mpi_g_part;
          MPI_Comm_group(vmp->mpi_c_part, &mpi_g_part);     // plan's group
          MPI_Group new_mpi_g;
          MPI_Group_incl(mpi_g_part, num_diff_pids, grouplist, &new_mpi_g);
          MPI_Comm_create(vmp->mpi_c_part, new_mpi_g, &new_mpi_c);
          MPI_Group_free(&new_mpi_g);
          MPI_Group_free(&mpi_g_part);
          // store the communicator on this PET with info to free
          sarg[0].mpi_c = new_mpi_c;
          sarg[0].mpi_c_freeflag = 1; // responsible to free the communicator
#if (MPI_VERSION >= 3)
          // set up communicator across single-system-images SSIs
          if (new_mpi_c != MPI_COMM_NULL)
            MPI_Comm_split_type(new_mpi_c, MPI_COMM_TYPE_SHARED, 0,
              MPI_INFO_NULL, &new_mpi_c_ssi);
          else
            new_mpi_c_ssi = MPI_COMM_NULL;
          // set up communicator across root pets of each SSI
          if (new_mpi_c != MPI_COMM_NULL){
            int color;
            MPI_Comm_rank(new_mpi_c_ssi, &color);
            if (color>0) color = MPI_UNDEFINED; // only root PETs on each SSI
            MPI_Comm_split(new_mpi_c, color, 0, &new_mpi_c_ssi_roots);
          }else{
            new_mpi_c_ssi_roots = MPI_COMM_NULL;
          }
#ifdef VM_SSISHMLOG_on
          {
            std::stringstream msg;
            int sz1, sz2, sz3, sz4;
            sz1 = sz2 = sz3 = sz4 = -1;
            MPI_Comm_size(vmp->mpi_c_part, &sz1);
            if (new_mpi_c != MPI_COMM_NULL)
              MPI_Comm_size(new_mpi_c, &sz2);
            if (new_mpi_c_ssi != MPI_COMM_NULL)
              MPI_Comm_size(new_mpi_c_ssi, &sz3);
            if (new_mpi_c_ssi_roots != MPI_COMM_NULL)
              MPI_Comm_size(new_mpi_c_ssi_roots, &sz4);
            msg << "VMK::startup()#" << __LINE__
              << ", mpi_c_part of size=" << sz1
              << ", new_mpi_c of size=" << sz2
              << ", created new_mpi_c_ssi of size=" << sz3
              << ", created new_mpi_c_ssi_roots of size=" << sz4;
            ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
          }
#endif
#else
          new_mpi_c_ssi = MPI_COMM_NULL;
          new_mpi_c_ssi_roots = MPI_COMM_NULL;
#endif
          sarg[0].mpi_c_ssi = new_mpi_c_ssi;
          sarg[0].mpi_c_ssi_roots = new_mpi_c_ssi_roots;
          // device management
          if (vmp->ndevlist > 0){
            std::vector<int> devListVec(vmp->devlist,
              vmp->devlist + vmp->ndevlist);
            std::sort(devListVec.begin(), devListVec.end());
            std::vector<int> ssiLocalDevListVec;
            auto it=devListVec.begin();
            for (auto i=0; i<VMK::ndevsSSI; i++){
              // loop over DEVs on the local SSI
              int gDev = VMK::ssidevs[i]; // global id of SSI local device
              for (; (it!=devListVec.end()) && (*it<=gDev); ++it){
                if (*it == gDev){
                  // found matching entry in devListVec
                  ssiLocalDevListVec.push_back(i);
                }
              }
            }
            ssiLocalDevCount = ssiLocalDevListVec.size();
            ssiLocalDevList = new int[ssiLocalDevCount];
            for (auto i=0; i<ssiLocalDevCount; i++)
              ssiLocalDevList[i] = ssiLocalDevListVec[i];
            if (new_mpi_c_ssi_roots != MPI_COMM_NULL)
              MPI_Allreduce(&ssiLocalDevCount, &devCount, 1, MPI_INTEGER, MPI_SUM,
                new_mpi_c_ssi_roots);
            if (new_mpi_c_ssi != MPI_COMM_NULL)
              MPI_Bcast(&devCount, 1, MPI_INTEGER, 0, new_mpi_c_ssi);
          }else{
            devCount = 0;
            ssiLocalDevCount = 0;
            ssiLocalDevList = NULL;
          }
          sarg[0].devCount = devCount;
          sarg[0].ssiLocalDevCount = ssiLocalDevCount;
          sarg[0].ssiLocalDevList = ssiLocalDevList;
        }else{
          // I am not the first under this lpid and must receive 
#if (VERBOSITY > 9)
          printf("mypet %d recvs new_mpi_c from %d\n", mypet, foundfirstpet);
#endif
          recv(&new_mpi_c, sizeof(MPI_Comm), foundfirstpet);
          sarg[0].mpi_c_freeflag = 0; // not responsible to free the communicat.
#if (MPI_VERSION >= 3)
          recv(&new_mpi_c_ssi, sizeof(MPI_Comm), foundfirstpet);
          recv(&new_mpi_c_ssi_roots, sizeof(MPI_Comm), foundfirstpet);
#ifdef VM_SSISHMLOG_on
          {
            std::stringstream msg;
            int sz;
            MPI_Comm_size(new_mpi_c_ssi, &sz);
            msg << "VMK::startup()#" << __LINE__
              << " received mpi_c_ssi of size=" << sz
              << " from PET: " << foundfirstpet;
            ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
          }
#endif
#endif
          recv(&devCount, sizeof(int), foundfirstpet);
          recv(&ssiLocalDevCount, sizeof(int), foundfirstpet);
          recv(ssiLocalDevList, devCount*sizeof(int), foundfirstpet);
        }
      }else if (mypet == foundfirstpet){
        // I am the master and must send the communicator
#if (VERBOSITY > 9)
        printf("mypet %d sends new_mpi_c to pet %d\n", mypet, i);
#endif
        send(&new_mpi_c, sizeof(MPI_Comm), i);
#if (MPI_VERSION >= 3)
        send(&new_mpi_c_ssi, sizeof(MPI_Comm), i);
        send(&new_mpi_c_ssi_roots, sizeof(MPI_Comm), i);
#ifdef VM_SSISHMLOG_on
        {
          std::stringstream msg;
          int sz;
          MPI_Comm_size(new_mpi_c_ssi, &sz);
          msg << "VMK::startup()#" << __LINE__
            << " sent mpi_c_ssi of size=" << sz
            << " to PET: " << i;
          ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
        }
#endif
#endif
        send(&devCount, sizeof(int), i);
        send(&ssiLocalDevCount, sizeof(int), i);
        send(ssiLocalDevList, devCount*sizeof(int), i);
      }
    }
  }
  delete [] grouplist;

#if (VERBOSITY > 9)
  printf("now valid new_mpi_c exists\n");
#endif

#ifdef VM_MEMLOG_on
      VM::logMemInfo(std::string("VMK::startup():13.0"));
#endif
  // now:
  //    new_mpi_c is the valid MPI_Comm for the new VMK
  // Next, setting up intra-process shared memory connection between
  // qualifying pets of the new VMK. Only one of the current pets that
  // spawn for a certain lpid must allocate memory for the shared variables 
  // and then send this info to all the associated intra- or inter-process pets
  // of the current VMK which also spawn threads.
  for (int i=0; i<num_diff_pids; i++){
    // consider all of the different lpids of current VMK
    if (lpid_list[0][i]>-1){
      // at least one pet of the current VMK with this lpid will spawn
#if (VERBOSITY > 9)
      printf("setting up shared memory variables for lpid=%d\n", 
        lpid_list[0][i]);
#endif
      // allocate and initialize memory 
      if (mypet==pet_list[i][0]){
        // mypet is the first in the list of those that spawn from this lpid
        // -> will allocate shared memory variables
#if (VERBOSITY > 9)
        printf("mypet is first one for lpid -> allocating shared memory\n");
#endif
        // initialize shared variables
        new_pth_mutex2 = new esmf_pthread_mutex_t;
#ifndef ESMF_NO_PTHREADS
        pthread_mutex_init(new_pth_mutex2, NULL);
#endif
        new_pth_mutex = new esmf_pthread_mutex_t;
#ifndef ESMF_NO_PTHREADS
        pthread_mutex_init(new_pth_mutex, NULL);
#endif
        new_pth_finish_count = new int;
        *new_pth_finish_count = 0;
        // initialize the IntraProcessSharedMemoryAllocation Table
        new_ipshmTop = new ipshmAlloc*;
        *new_ipshmTop = NULL;  // reset
        new_ipshmMutex = new esmf_pthread_mutex_t;
#ifndef ESMF_NO_PTHREADS
        pthread_mutex_init(new_ipshmMutex, NULL);
#endif
        new_ipSetupMutex = new esmf_pthread_mutex_t;
#ifndef ESMF_NO_PTHREADS
        pthread_mutex_init(new_ipSetupMutex, NULL);
#endif
        // set up the shared_mp structure within the new_commarray
        int pet1Index = 0;  // reset
        for (int pet1=0; pet1<new_npets; pet1++){
          if (new_pid[pet1]==pid[mypet]){
            int pet2Index = 0;  // reset
            for (int pet2=0; pet2<new_npets; pet2++){
              if (new_pid[pet2]==pid[mypet]){
                new_commarray[pet1Index][pet2Index].shmp = NULL; // detectable
#ifdef ESMF_MPIUNI
                // pet1==pet2==mypet==0
                // -> allocate shared_mp structure for such PETs
                new_commarray[pet1Index][pet2Index].shmp = new shared_mp;
                // reset the shms structure in shared_mp preparing for use
                sync_reset(&(new_commarray[pet1Index][pet2Index].shmp->shms));
                new_commarray[pet1Index][pet2Index].comm_type =
                  VM_COMM_TYPE_MPIUNI;
                new_commarray[pet1Index][pet2Index].shmp->tcounter = 0;
                new_commarray[pet1Index][pet2Index].shmp->recvCount = 0;
                new_commarray[pet1Index][pet2Index].shmp->sendCount = 0;
                for (int i=0; i<SHARED_NONBLOCK_CHANNELS; i++){
                  new_commarray[pet1Index][pet2Index].shmp->ptr_src_nb[i]
                    = NULL;
                  new_commarray[pet1Index][pet2Index].shmp->ptr_dst_nb[i]
                    = NULL;
                }
#else
                if (pet1 != pet2){
                  // pet1 and pet2 are different PETs that run in mypet's VAS
                  // -> allocate shared_mp structure for such PETs
                  new_commarray[pet1Index][pet2Index].shmp = new shared_mp;
                  // reset the shms structure in shared_mp preparing for use
                  sync_reset(&(new_commarray[pet1Index][pet2Index].shmp->shms));
                  // don't modify intra-PET comm_type
                  if (vmp->pref_intra_process == PREF_INTRA_PROCESS_SHMHACK){
                    new_commarray[pet1Index][pet2Index].comm_type =
                      VM_COMM_TYPE_SHMHACK;
                  }else if(vmp->pref_intra_process==PREF_INTRA_PROCESS_PTHREAD){
                    new_commarray[pet1Index][pet2Index].comm_type =
                      VM_COMM_TYPE_PTHREAD;
                    // initialize pthread variables in shared_mp
#ifndef ESMF_NO_PTHREADS
                    pthread_mutex_init(
                      &(new_commarray[pet1Index][pet2Index].shmp->mutex1),
                      NULL);
                    pthread_cond_init(
                      &(new_commarray[pet1Index][pet2Index].shmp->cond1),
                      NULL);
                    pthread_mutex_init(
                      &(new_commarray[pet1Index][pet2Index].shmp->mutex2),
                      NULL);
                    pthread_cond_init(
                      &(new_commarray[pet1Index][pet2Index].shmp->cond2),
                      NULL);
#endif
                    new_commarray[pet1Index][pet2Index].shmp->tcounter = 0;
                    new_commarray[pet1Index][pet2Index].shmp->recvCount = 0;
                    new_commarray[pet1Index][pet2Index].shmp->sendCount = 0;
                    for (int i=0; i<SHARED_NONBLOCK_CHANNELS; i++){
                      new_commarray[pet1Index][pet2Index].shmp->ptr_src_nb[i]
                        = NULL;
                      new_commarray[pet1Index][pet2Index].shmp->ptr_dst_nb[i]
                        = NULL;
                    }
                  }
                }else{
                  new_commarray[pet1Index][pet2Index].comm_type =
                    VM_COMM_TYPE_MPI1;  // default for selfcommunication
                }
#endif
                ++pet2Index;
              }
            }
            ++pet1Index;
          }
        }
      }
      // share pointers with all current pets that also spawn for same pid/lpid
      for (int j=1; j<lpid_list[1][i]; j++){
        int pet_dest = pet_list[i][j];
        int pet_src = pet_list[i][0];
        if (mypet==pet_src){
          // mypet is the first pet in the list -> mypet allocated -> must send
          send(&new_pth_mutex2, sizeof(esmf_pthread_mutex_t*), pet_dest);
          send(&new_pth_mutex, sizeof(esmf_pthread_mutex_t*), pet_dest);
          send(&new_pth_finish_count, sizeof(int*), pet_dest);
          send(&new_ipshmTop, sizeof(ipshmAlloc*), pet_dest);
          send(&new_ipshmMutex, sizeof(esmf_pthread_mutex_t*), pet_dest);
          send(&new_ipSetupMutex, sizeof(esmf_pthread_mutex_t*), pet_dest);
          send(&new_commarray, sizeof(comminfo**), pet_dest);
        }else if(mypet==pet_dest){
          // mypet is one of the pets that also spawn for this lpid -> receive
          // before this PETs new_commarray is overridden it must be deleted
          for (int ii=0; ii<mypetNewThreadGroupSize; ii++)
            delete [] new_commarray[ii];
          delete [] new_commarray;
          new_commarray_delete_flag = false;  // mypet doesn't delete again
          // now this PET is ready to receive the pointers for shared variables
          recv(&new_pth_mutex2, sizeof(esmf_pthread_mutex_t*), pet_src);
          recv(&new_pth_mutex, sizeof(esmf_pthread_mutex_t*), pet_src);
          recv(&new_pth_finish_count, sizeof(int*), pet_src);
          recv(&new_ipshmTop, sizeof(ipshmAlloc*), pet_src);
          recv(&new_ipshmMutex, sizeof(esmf_pthread_mutex_t*), pet_src);
          recv(&new_ipSetupMutex, sizeof(esmf_pthread_mutex_t*), pet_src);
          recv(&new_commarray, sizeof(comminfo**), pet_src);
        }
      }
    } // at least one PET of the current VMK will spawn from this lpid
  } // i
#ifdef VM_MEMLOG_on
      VM::logMemInfo(std::string("VMK::startup():14.0"));
#endif
  // now:
  //    new_pth_mutex2 is valid pthread_mutex
  //    new_pth_mutex is valid pthread_mutex
  //    new_pth_finish_count is valid shared memory counter
  //    new_commarray now holds valid shared memory shared_mp objects
  //
  // next, enter the spawn-loop for mypet
  for (int i=0; i<vmp->spawnflag[mypet]; i++){
    // copy this threads information into the sarg structure
    sarg[i].fctp = fctp;
    sarg[i].mypet = new_mypet_base + i;   // different for each thread spawned
    sarg[i].npets = new_npets;
    sarg[i].lpid = new int[new_npets];
    sarg[i].pid = new int[new_npets];
    sarg[i].tid = new int[new_npets];
    sarg[i].ncpet = new int[new_npets];
    sarg[i].nadevs = new int[new_npets];
    sarg[i].cid = new int*[new_npets];
    sarg[i].ncontributors = new_ncontributors[sarg[i].mypet];
    sarg[i].contributors = new contrib_id[sarg[i].ncontributors];
    for (int k=0; k<sarg[i].ncontributors; k++)
      sarg[i].contributors[k] = new_contributors[sarg[i].mypet][k];
    for (int j=0; j<new_npets; j++){
      sarg[i].lpid[j] = new_lpid[j];
      sarg[i].pid[j] = new_pid[j];
      sarg[i].tid[j] = new_tid[j];
      sarg[i].ncpet[j] = new_ncpet[j];
      sarg[i].nadevs[j] = new_nadevs[j];
      sarg[i].cid[j] = new int[new_ncpet[j]];
      for (int k=0; k<new_ncpet[j]; k++)
        sarg[i].cid[j][k] = new_cid[j][k];
    }
    sarg[i].mpi_c = new_mpi_c;
#if (MPI_VERSION >= 3)
    sarg[i].mpi_c_ssi = new_mpi_c_ssi;
    sarg[i].mpi_c_ssi_roots = new_mpi_c_ssi_roots;
#ifdef VM_SSISHMLOG_on
    {
      std::stringstream msg;
      int sz;
      MPI_Comm_size(sarg[i].mpi_c_ssi, &sz);
      msg << "VMK::startup()#" << __LINE__
        << " copied mpi_c_ssi of size=" << sz
        << " into sarg[" << i << "]";
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
    }
#endif
#endif
    sarg[i].pth_mutex2 = new_pth_mutex2;
    sarg[i].pth_mutex = new_pth_mutex;
    sarg[i].pth_finish_count = new_pth_finish_count;
    sarg[i].sendChannel = new comminfo[new_npets];
    sarg[i].recvChannel = new comminfo[new_npets];
    int new_mypet = sarg[i].mypet;
    int new_mypetIndex = 0; // reset
    for (int j=0; j<new_mypet; j++)
      if (new_pid[j] == new_pid[new_mypet])
        ++new_mypetIndex;
    int petIndex = 0; // reset
    for (int j=0; j<new_npets; j++){
      if (new_pid[j] == new_pid[new_mypet]){
        // new pet j and new_mypet will run in the same VAS
        // -> copy new_commarray entry into sendChannel and recvChannel
        sarg[i].sendChannel[j] = new_commarray[new_mypetIndex][petIndex];
        sarg[i].recvChannel[j] = new_commarray[petIndex][new_mypetIndex];
        ++petIndex;
      }else{
        // default inter-process communication via MPI1
        sarg[i].recvChannel[j].comm_type = VM_COMM_TYPE_MPI1;
        sarg[i].sendChannel[j].comm_type = VM_COMM_TYPE_MPI1;
      }
    }
    sarg[i].ipshmTop = new_ipshmTop;
    sarg[i].ipshmMutex = new_ipshmMutex;
    sarg[i].ipSetupMutex = new_ipSetupMutex;
    sarg[i].pref_intra_ssi = vmp->pref_intra_ssi;
    // device management
    sarg[i].devCount = devCount;
    sarg[i].ssiLocalDevCount = ssiLocalDevCount;
    sarg[i].ssiLocalDevList = ssiLocalDevList;
    // cargo
    sarg[i].cargo = cargo;
    // stdout and stderr redirect
    string stdTemp; size_t pos; // temp helpers
    if (vmp->stdoutName)
      stdTemp = string(vmp->stdoutName);
    else 
      stdTemp = string("");
    pos = stdTemp.rfind('*');  // right most asterisk
    if (pos != string::npos){
      // found wildcard -> replace with local pet number
      int digits = 1; // default number of digits needed
      if (new_npets>1) digits = (int) log10(new_npets-1) + 1;
      std::stringstream label;                    // fill with zeros from left
      label << setw(digits) << setfill('0') << to_string(sarg[i].mypet);
      sarg[i].stdoutName = stdTemp.substr(0, pos) + label.str()
        + stdTemp.substr(pos+1, string::npos);    // store the concretized name
    }else{
      // no wildcard -> use incoming string verbatim
      sarg[i].stdoutName = stdTemp;
    }
    if (vmp->stderrName)
      stdTemp = string(vmp->stderrName);
    else 
      stdTemp = string("");
    pos = stdTemp.rfind('*');  // right most asterisk
    if (pos != string::npos){
      // found wildcard -> replace with local pet number
      int digits = 1; // default number of digits needed
      if (new_npets>1) digits = (int) log10(new_npets-1) + 1;
      std::stringstream label;                    // fill with zeros from left
      label << setw(digits) << setfill('0') << to_string(sarg[i].mypet);
      sarg[i].stderrName = stdTemp.substr(0, pos) + label.str()
        + stdTemp.substr(pos+1, string::npos);    // store the concretized name
    }else{
      // no wildcard -> use incoming string verbatim
      sarg[i].stderrName = stdTemp;
    }
    if (i==0){
      // stdout and stderr redirect is a per process feature
      // only do this for the first PET, even if potentially multiple spawned
      if (sarg[i].stdoutName.length()){
        // create stdout file and possibly truncate if already there
        int fd = open(sarg[i].stdoutName.c_str(),
          O_WRONLY | O_CREAT | O_TRUNC, 0644);
        if (fd == -1){
          ESMC_LogDefault.MsgFoundError(ESMC_RC_INTNRL_BAD,
            "Did not obtain a valid file descriptor", ESMC_CONTEXT, rc);
          return NULL; // bail out
        }
        close(fd);  // free up file descriptor, file will be opened again later
      }
      if (sarg[i].stderrName.length()){
        // create stderr file and possibly truncate if already there
        int fd = open(sarg[i].stderrName.c_str(),
          O_WRONLY | O_CREAT | O_TRUNC, 0644);
        if (fd == -1){
          ESMC_LogDefault.MsgFoundError(ESMC_RC_INTNRL_BAD,
            "Did not obtain a valid file descriptor", ESMC_CONTEXT, rc);
          return NULL; // bail out
        }
        close(fd);  // free up file descriptor, file will be opened again later
      }
    }
    // threading stuff
    sarg[i].openmphandling = vmp->openmphandling;
    sarg[i].openmpnumthreads = vmp->openmpnumthreads;
    sarg[i].threadsflag = vmp->eachChildPetOwnPthread;
    if (!vmp->eachChildPetOwnPthread){
      // For a VM that does not create new Pthreads, the VM can already be
      // constructed.
      // obtain reference to the vm instance on heap
      VMK &vm = *(sarg[0].myvm);
      // setup the pet section in this vm instance
#ifndef ESMF_NO_PTHREADS
      sarg[0].pthid = pthread_self();
#else
      sarg[0].pthid = 0;
#endif
      vm.construct((void *)&sarg[0]);
    }else{
      // For a VM that creates new Pthreads...
      // ... spawn threads from this pet ...
      // ... cannot construct new VM here ...
      // ... must construct under the new Pthread where pthreadID is known!
      *rc = vmkt_create(&(sarg[i].vmkt), vmk_spawn, (void *)&sarg[i],
        false, vmp->minStackSize); // not a service thread
      if (*rc) return NULL;  // could not create pthread -> bail out
    }
  }
#ifdef VM_MEMLOG_on
      VM::logMemInfo(std::string("VMK::startup():15.0"));
#endif
  // free all the temporary arrays.... (not sarg array!!!)
  for (int i=0; i<new_npets; i++){
    delete [] new_cid[i];
    delete [] new_contributors[i];
  }
  delete [] new_cid;
  delete [] new_contributors;
  if (new_commarray_delete_flag){
    // mypet must deallocate its new_commarray
    for (int ii=0; ii<mypetNewThreadGroupSize; ii++)
      delete [] new_commarray[ii];
    delete [] new_commarray;
  }
#ifdef VM_MEMLOG_on
      VM::logMemInfo(std::string("VMK::startup():16.0"));
#endif
  // return info that is associated with the new VMK...
  return sarg;
}


void VMK::enter(class VMKPlan *vmp, void *arg, void *argvmkt){
  // Enter into VMK by its registered function, i.e. release vmkt
  // First need to cast arg into its correct type
  SpawnArg *sarg = (SpawnArg *)arg;
  bool simpleBlockingCallback = false;
  // simple case is that where the child runs in the parent VM, then all this
  // degenerates into a simple blocking callback
  simpleBlockingCallback |= vmp->parentVMflag;
  // the non-thread based VMs simply do a blocking callback for all the 
  // spawning PETs.
  simpleBlockingCallback |= 
    (!vmp->supportContributors && vmp->spawnflag[mypet]==1);
  // finally execute the simple blocking callback
  if (simpleBlockingCallback){
#ifdef VM_PETMANAGEMENTLOG_on
      {
        std::stringstream msg;
        msg << "VMK::enter()#" << __LINE__
          << " simpleBlockingCallback to user code ...";
        ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
      }
#endif
    // set stdout and stderr redirects
    VMK::Redirects oldReds = VMK::setRedirects((void *)sarg);
    // callback
    if (argvmkt==NULL)
      sarg[0].fctp((void *)sarg[0].myvm, sarg[0].cargo);
    else
      sarg[0].fctp((void *)sarg[0].myvm, argvmkt);
    // reset previous stdout and stderr settings
    oldReds.reset();
#ifdef VM_PETMANAGEMENTLOG_on
      {
        std::stringstream msg;
        msg << "VMK::enter()#" << __LINE__
          << " simpleBlockingCallback to user code returned.";
        ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
      }
#endif
    return;
  }
  // continue with the more complicated case, where threads must be released...
  // pets that do not spawn but contribute need to release their blocker and
  // sigcatcher _before_ the actual spawner threads get released 
  // (this is so that no signals get missed!)
  if (vmp->spawnflag[mypet]==0){
    // mypet is a contribtor, not a spawner
    if (vmp->contribute[mypet]>-1){

#ifdef WITHBLOCKER_on

#ifdef VM_PETMANAGEMENTLOG_on
      {
        std::stringstream msg;
        msg << "VMK::enter()#" << __LINE__
          << " call vmkt_release on blocker thread for contributing PET.";
        ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
      }
#endif
      vmkt_release(&(sarg[0].vmkt), NULL);          // release blocker
#endif
      
      
#ifdef VM_PETMANAGEMENTLOG_on
      {
        std::stringstream msg;
        msg << "VMK::enter()#" << __LINE__
          << " call vmkt_release on sigcatcher thread for contributing PET.";
        ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
      }
#endif
      vmkt_release(&(sarg[0].vmkt_extra), NULL);    // release sigcatcher
#ifdef VM_PETMANAGEMENTLOG_on
      {
        std::stringstream msg;
        msg << "VMK::enter()#" << __LINE__
          << " sending handshake to spawning PET: "
          << vmp->contribute[mypet];
        ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
      }
#endif
      send(NULL, 0, vmp->contribute[mypet]);     // tell spawner about me
#ifdef VM_PETMANAGEMENTLOG_on
      {
        std::stringstream msg;
        msg << "VMK::enter()#" << __LINE__
          << " done sending handshake to spawning PET: "
          << vmp->contribute[mypet];
        ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
      }
#endif
    }
  }else{
    // mypet is a spawner, wait on all the contributors to this spawner
    for (int i=0; i<npets; i++){
      if (vmp->contribute[i]==mypet && i!=mypet){
#ifdef VM_PETMANAGEMENTLOG_on
        {
          std::stringstream msg;
          msg << "VMK::enter()#" << __LINE__
            << " receiving handshake from contributing PET: " << i;
          ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
        }
#endif
        recv(NULL, 0, i); // listen if contributor has released its threads
#ifdef VM_PETMANAGEMENTLOG_on
        {
          std::stringstream msg;
          msg << "VMK::enter()#" << __LINE__
            << " done receiving handshake from contributing PET: " << i;
          ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
        }
#endif
      }
    }
    // now all contributors are in, pets that spawn need to release their vmkts
    if (vmp->eachChildPetOwnPthread){
      // Each child PET in its own Pthread
      for (int i=0; i<vmp->spawnflag[mypet]; i++){
#ifdef VM_PETMANAGEMENTLOG_on
        {
          std::stringstream msg;
          msg << "VMK::enter()#" << __LINE__
            << ": pthread=" << pthread_self() << " pid=" << getpid()
            << " call vmkt_release on spawned thread: " << i;
          ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
        }
#endif
        vmkt_release(&(sarg[i].vmkt), argvmkt);
      }
#ifdef VM_PETMANAGEMENTLOG_on
      {
        std::stringstream msg;
        msg << "VMK::enter()#" << __LINE__
          << ": pthread=" << pthread_self() << " pid=" << getpid()
          << " done calling vmkt_release for all spawned threads";
        ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
      }
#endif
    }else{
      // Each child PET in its parent thread (might be MPI process or Pthread)
      if (vmp->spawnflag[mypet]==1){
        // call back into user code, and return PEs to their contributing PETs
        (sarg->vmkt).arg = argvmkt;
        (sarg->vmkt).tid = 0; // mark as an invalid tid
        enter_callback(sarg, pth_mutex);
      }else if (vmp->spawnflag[mypet]==0){
        // not a spawning PET -> NoOp
      }else{
        int localrc;
        ESMC_LogDefault.MsgFoundError(ESMC_RC_INTNRL_BAD,
          "Cannot spawn multiple child PETs without eachChildPetOwnPthread!",
          ESMC_CONTEXT, &localrc);
        throw localrc;  // bail out with exception
      }
    }
  }
}


void VMK::exit(class VMKPlan *vmp, void *arg){
  // Exit from VMK's registered function, i.e. catch the vmkts
  // First need to cast arg into its correct type
  SpawnArg *sarg = (SpawnArg *)arg;
  // simple case is that where the child runs in the parent VM, then there is
  // nothing to catch on exit.
  if (vmp->parentVMflag) return;
  // check if this is a thread-based VM
  if (vmp->supportContributors){
#ifdef VM_PETMANAGEMENTLOG_on
    {
      std::stringstream msg;
      msg << "VMK::exit()#" << __LINE__
        << " for a threaded component.";
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
    }
#endif
    // pets that spawn in a thread-based VM need to catch their vmkts
    if (vmp->eachChildPetOwnPthread){
      // Each child PET in its own Pthread
      for (int i=0; i<vmp->spawnflag[mypet]; i++){
#ifdef VM_PETMANAGEMENTLOG_on
        {
          std::stringstream msg;
          msg << "VMK::exit()#" << __LINE__
          << " call vmkt_catch on spawned thread: " << i;
          ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
        }
#endif
        vmkt_catch(&(sarg[i].vmkt));
      }
    }else{
      // Each child PET in its parent thread (might be MPI process or Pthread)
      if (vmp->spawnflag[mypet]>1){
        // TODO: implemnet error handling here!!!
      }
    }
    // pets that did not spawn but contributed need to catch their blocker and
    // sigcatcher
    if (vmp->spawnflag[mypet]==0 && vmp->contribute[mypet]>-1){
#ifdef VM_PETMANAGEMENTLOG_on
      {
        std::stringstream msg;
        msg << "VMK::exit()#" << __LINE__
          << " call vmkt_catch on sigcatcher thread for contributor PET.";
        ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
      }
#endif
      vmkt_catch(&(sarg[0].vmkt_extra));
      
#ifdef WITHBLOCKER_on
#ifdef VM_PETMANAGEMENTLOG_on
      {
        std::stringstream msg;
        msg << "VMK::exit()#" << __LINE__
          << " call vmkt_catch on blocker thread for contributor PET.";
        ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
      }
#endif
      vmkt_catch(&(sarg[0].vmkt));
#endif

    }
#ifdef VM_PETMANAGEMENTLOG_on
    {
      std::stringstream msg;
      msg << "VMK::exit()#" << __LINE__
        << " done calling vmkt_catch for any threads";
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
    }
#endif
  }
  // The following threadbarrier ensures that each parent PET blocks until 
  // all threads that work in the PET-local VAS have completed.
  threadbarrier();
}


void VMK::shutdown(class VMKPlan *vmp, void *arg){
  // Block all pets of the current VMK until their individual resources,
  // i.e. cores, become available. This means:
  //  1) pets which did not spawn nor contribute in the VMKPlan are not
  //     blocked
  //  2) pets which did not spawn but contributed in the VMKPlan block
  //     until their cores have been returned by spwawned VMK
  //  3) pets that spwan will be blocked until _all_ of _their_ spawned threads
  //     exit in order to delete allocated info structure
  // First need to cast arg into its correct type
  SpawnArg *sarg = (SpawnArg *)arg;
  // simple case is that where the child runs in the parent VM, then there is
  // nothing to clean up here except to free sarg;
  if (vmp->parentVMflag){
    delete [] sarg;
    return;
  }
  for (int i=0; i<vmp->spawnflag[mypet]; i++){
    if (!vmp->eachChildPetOwnPthread){
      // obtain reference to the vm instance on heap
      VMK &vm = *(sarg[0].myvm);
      vm.epochFinal();  // close down epoch handling
      vm.destruct();    // destroy this vm instance
    }else{
      // thread-based VM pets must be joined
      vmkt_join(&(sarg[i].vmkt));
    }
    // free arrays in sarg[i]
    delete [] sarg[i].lpid;
    delete [] sarg[i].pid;
    delete [] sarg[i].tid;
    delete [] sarg[i].ncpet;
    delete [] sarg[i].nadevs;
    delete [] sarg[i].contributors;
    for (int j=0; j<sarg[i].npets; j++){
      delete [] sarg[i].cid[j];
    }
    delete [] sarg[i].cid;
    delete [] sarg[i].sendChannel;
    delete [] sarg[i].recvChannel;
  }
  // need to block pets that did not spawn but contributed in the VMKPlan
  if (vmp->spawnflag[mypet]==0 && vmp->contribute[mypet]>-1){
    // wait for the blocker thread to return, maybe already is done...
#if (VERBOSITY > 9)
    printf("I am pet, pid %d, tid %d. I'll block on a vmkt_join until "
      "blocker returns.\n", getpid(), pthread_self());
#endif
    vmkt_join(&(sarg[0].vmkt));         // vmk_block
    vmkt_join(&(sarg[0].vmkt_extra));   // vmk_sigcatcher
#if (VERBOSITY > 9)
    printf("I am pet, pid %d, tid %d. My blocker returned\n", 
      getpid(), pthread_self());
#endif
  }
  // now free up the MPI communicators that were associated with the VMK
  if (sarg[0].mpi_c_freeflag){
    if (sarg[0].mpi_c != MPI_COMM_NULL){
      MPI_Comm_free(&(sarg[0].mpi_c));
    }
#if (MPI_VERSION >= 3)
    if (sarg[0].mpi_c_ssi != MPI_COMM_NULL){
#ifdef VM_SSISHMLOG_on
      {
        std::stringstream msg;
        if (sarg[0].mpi_c_ssi != MPI_COMM_NULL){
          int sz;
          MPI_Comm_size(sarg[0].mpi_c_ssi, &sz);
          msg << "VMK::shutdown()#" << __LINE__
            << " about to call MPI_Comm_free() on mpi_c_ssi of size=" << sz;
        }else{
          msg << "VMK::shutdown()#" << __LINE__
            << " about to call MPI_Comm_free() on mpi_c_ssi == MPI_COMM_NULL";
        }
        ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
      }
      MPI_Barrier(sarg[0].mpi_c_ssi);
#endif
      MPI_Comm_free(&(sarg[0].mpi_c_ssi));
    }
    if (sarg[0].mpi_c_ssi_roots != MPI_COMM_NULL){
      MPI_Comm_free(&(sarg[0].mpi_c_ssi_roots));
    }
#endif
    if (sarg[0].ssiLocalDevList)
      delete [] sarg[0].ssiLocalDevList;
  }
  // done holding info in SpawnArg array -> delete now
  delete [] sarg;
  // done blocking...
}


int VMK::checkPetList(int *petList, int count){
#undef  ESMC_METHOD
#define ESMC_METHOD "ESMCI::VMK::checkPetList()"
  int rc;
  if (!petList){
    ESMC_LogDefault.MsgFoundError(ESMC_RC_PTR_NULL,
      "petList must not be NULL.",
      ESMC_CONTEXT, &rc);
    return rc;
  }
  std::vector<int> petListVec(petList, petList + count);
  std::set<int> petListSet(petListVec.begin(), petListVec.end());
  if (petListSet.size() != petListVec.size()){
    // there must be at least one duplicate element in petList
    auto it=petListVec.begin();
    for (int i=0; i<count/10; i++){
      std::stringstream msg;
      msg << "{ ";
      for (int j=0; j<9; j++){
        msg << *it << ", ";
        ++it;
      }
      msg << *it << "}";
      ++it;
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_ERROR);
    }
    std::stringstream msg;
    msg << "{ ";
    for (int j=0; j<count%10-1; j++){
      msg << *it << ", ";
      ++it;
    }
    msg << *it << "}";
    ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_ERROR);
    ESMC_LogDefault.MsgFoundError(ESMC_RC_ARG_DUP,
      "There appear to be duplicate PETs in the petList.",
      ESMC_CONTEXT, &rc);
    return rc;
  }
  return ESMF_SUCCESS;
}


int VMK::checkDevList(int *devList, int count){
#undef  ESMC_METHOD
#define ESMC_METHOD "ESMCI::VMK::checkDevList()"
  int rc;
  if (!devList){
    ESMC_LogDefault.MsgFoundError(ESMC_RC_PTR_NULL,
      "devList must not be NULL.",
      ESMC_CONTEXT, &rc);
    return rc;
  }
  std::vector<int> devListVec(devList, devList + count);
  std::set<int> devListSet(devListVec.begin(), devListVec.end());
  if (devListSet.size() != devListVec.size()){
    // there must be at least one duplicate element in devList
    auto it=devListVec.begin();
    for (int i=0; i<count/10; i++){
      std::stringstream msg;
      msg << "{ ";
      for (int j=0; j<9; j++){
        msg << *it << ", ";
        ++it;
      }
      msg << *it << "}";
      ++it;
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_ERROR);
    }
    std::stringstream msg;
    msg << "{ ";
    for (int j=0; j<count%10-1; j++){
      msg << *it << ", ";
      ++it;
    }
    msg << *it << "}";
    ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_ERROR);
    ESMC_LogDefault.MsgFoundError(ESMC_RC_ARG_DUP,
      "There appear to be duplicate DEVs in the devList.",
      ESMC_CONTEXT, &rc);
    return rc;
  }
  return ESMF_SUCCESS;
}


void VMK::print()const{
  // print info about the VMK object
  printf("--- VMK::print() start ---\n");
  printf("vm located at: %p\n", this);
  printf("npets = %d, mypet=%d\n", npets, mypet);
  printf("  pth_mutex =\t\t %p\n"
         "  pth_finish_count =\t %p\n",
    pth_mutex, pth_finish_count);
  int size, rank;
  MPI_Comm_size(mpi_c, &size);
  printf("MPI_Comm_size: %d\n", size);
  MPI_Comm_rank(mpi_c, &rank);
  printf("MPI_Comm_rank in local MPI communicator: %d\n", rank);
  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
  printf("MPI_Comm_rank in MPI_COMM_WORLD: %d\n", rank);
  printf("MPI thread level support: %d\n", mpi_thread_level);
  printf("mpi_mutex_flag: %d\n", mpi_mutex_flag);
  printf("mpionly: %d\n", mpionly);
  printf("threadsflag: %d\n", threadsflag);
  for (int i=0; i<npets; i++){
    printf("  lpid[%d]=%d, tid[%d]=%d, vas[%d]=%d, ncpet[%d]=%d, nadevs[%d]=%d",
      i, lpid[i], i, tid[i], i, pid[i], i, ncpet[i], i, nadevs[i]);
    for (int j=0; j<ncpet[i]; j++)
      printf(", cid[%d][%d]=%d", i, j, cid[i][j]);
    printf("\n");
  }
  printf("ncores = %d\n", ncores);
  for (int i=0; i<ncores; i++)
    printf("  cpuid[%d]=%d, ssiid[%d]=%d\n", i, cpuid[i], i, ssiid[i]);
  printf("--- VMK::print() end ---\n");
}


void VMK::log(std::string prefix, ESMC_LogMsgType_Flag msgType)const{
  std::stringstream msg;
  msg << prefix << "--- VMK::log() start -------------------------------------";
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
  msg << prefix << "vm located at: " << this;
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
  msg << prefix << "mpionly=" << mpionly << " threadsflag=" << threadsflag;
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
  msg << prefix << "ssiCount=" << getSsiCount()
    << " localSsi=" << ssiid[cid[mypet][0]];
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
  msg << prefix << "devCount=" << getDevCount()
    << " ssiLocalDevCount=" << getSsiLocalDevCount();
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
  if (getSsiLocalDevCount() > 0){
    msg << prefix;
    for (auto i=0; i<getSsiLocalDevCount(); i++)
      msg << " ssiLocalDevList[" << i << "]=" << ssiLocalDevList[i]
        << "->" << ssidevs[ssiLocalDevList[i]];
    ESMC_LogDefault.Write(msg.str(), msgType);
  }
  msg.str("");  // clear
  msg << prefix << "petCount=" << getPetCount()
    << " ssiLocalPetCount=" << getSsiLocalPetCount();
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
  msg << prefix << "localPet=" << getLocalPet()
    << " mypthid=" << mypthid
    << " ssiLocalPet=" << getSsiLocalPet()
    << " currentSsiPe=" << getCurrentSsiPe();
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
#ifndef ESMF_NO_PTHREADS
#if !defined(ESMF_OS_Darwin) && !defined(ESMF_OS_Cygwin)
  // output thread affinity
  cpu_set_t cpuset;
  pthread_getaffinity_np(mypthid, sizeof(cpu_set_t), &cpuset);
  msg << prefix << "Current system level affinity pinning for local PET:";
  ESMC_LogDefault.Write(msg.str(), msgType);
  for (auto i=0; i<CPU_SETSIZE; i++){
    if (CPU_ISSET(i, &cpuset)){
      msg.str("");  // clear
      msg << prefix << " SSIPE=" << i;
      ESMC_LogDefault.Write(msg.str(), msgType);
    }
  }
  msg.str("");  // clear
#endif
#endif
#ifndef ESMF_NO_OPENMP
  // output OpenMP info
  msg << prefix << "Current system level OMP_NUM_THREADS setting for local PET: "
    << omp_get_max_threads();
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
  msg << prefix << "Current system level OMP_NUM_PROCS setting for local PET: "
    << omp_get_num_procs();
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
#ifdef ESMF_OPENMP4
  msg << prefix << "Current system level OMP_NUM_DEVICES setting for local PET: "
    << omp_get_num_devices();
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
  msg << prefix << "Current system level OMP_DEVICE_NUM setting for local PET: "
    << omp_get_device_num();
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
  msg << prefix << "Current system level OMP_DEFAULT_DEVICE setting for local PET: "
    << omp_get_default_device();
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
#endif
#endif
#ifndef ESMF_NO_OPENACC
  // output OpenACC info
#ifdef __NVCOMPILER
  // under NVHPC, multi-cpu-core threading (for OpenACC and standard language
  // constructs) is implemented based on OpenACC: NVHPC extended OpenACC API
  msg << prefix << "Current system level ACC_NUM_CORES setting for local PET: "
    << acc_get_num_cores();
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
#endif
  msg << prefix << "Current system level ACC_NUM_DEVICES (acc_device_not_host)";
  msg << " setting for local PET: ";
  msg << acc_get_num_devices(acc_device_not_host);
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
  msg << prefix << "Current system level ACC_DEVICE_NUM (acc_device_not_host)";
  msg << " setting for local PET: ";
  msg << acc_get_device_num(acc_device_not_host);
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
  msg << prefix << "Current system level ACC_DEVICE_TYPE";
  msg << " setting for local PET: ";
  msg << acc_get_device_type();
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
#endif

#ifdef ESMF_NVML
  nvmlReturn_t nvlmRC;
  nvlmRC = nvmlInit_v2();
  msg << prefix << "nvmlInit_v2: " << nvlmRC;
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear

  unsigned int nvmlDeviceCount;
  nvlmRC = nvmlDeviceGetCount_v2(&nvmlDeviceCount);
  msg << prefix << "nvmlDeviceGetCount_v2: " << nvlmRC;
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
  msg << prefix << "nvmlDeviceCount: " << nvmlDeviceCount;
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
  
  char nvmlVersion[80];
  nvlmRC = nvmlSystemGetNVMLVersion(nvmlVersion, 80);
  msg << prefix << "nvmlSystemGetNVMLVersion: " << nvlmRC;
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
  msg << prefix << "nvmlVersion: " << nvmlVersion;
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear

  nvlmRC = nvmlShutdown();
  msg << prefix << "nvmlShutdown: " << nvlmRC;
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
#endif

  msg << prefix << "ESMF level logical PET->PE association "
   << "(but consider system level affinity pinning!):";
  ESMC_LogDefault.Write(msg.str(), msgType);
  for (auto i=0; i<npets; i++){
    msg.str("");  // clear
    msg << prefix << "PET=" << i << " lpid=" << lpid[i] << " tid=" << tid[i]
      << " pid=" << pid[i] << " peCount=" << ncpet[i]
      << " accCount=" << nadevs[i];
    ESMC_LogDefault.Write(msg.str(), msgType);
    for (auto j=0; j<ncpet[i]; j++){
      msg.str("");  // clear
      msg << prefix << " PE=" << cid[i][j] << " SSI=" << ssiid[cid[i][j]]
        << " SSIPE=" << ssipe[cid[i][j]];
      ESMC_LogDefault.Write(msg.str(), msgType);
    }
  }
  msg.str("");  // clear
  msg << prefix << "--- VMK::log() end ---------------------------------------";
  ESMC_LogDefault.Write(msg.str(), msgType);
}


void VMK::logSystem(std::string prefix, ESMC_LogMsgType_Flag msgType){
#undef  ESMC_METHOD
#define ESMC_METHOD "ESMCI::VMK::logSystem()"
  std::stringstream msg;
  msg << prefix << "--- VMK::logSystem() start -------------------------------";
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
  msg << prefix << "esmfComm=" << getEsmfComm();
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
  msg << prefix << "isPthreadsEnabled=" << isPthreadsEnabled();
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
  msg << prefix << "isOpenMPEnabled=" << isOpenMPEnabled();
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
  msg << prefix << "isOpenACCEnabled=" << isOpenACCEnabled();
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
  msg << prefix << "isSsiSharedMemoryEnabled=" << isSsiSharedMemoryEnabled();
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
  msg << prefix << "isNvmlEnabled=" << isNvmlEnabled();
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
  msg << prefix << "isNumaEnabled=" << isNumaEnabled();
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
  msg << prefix << "ssiCount=" << nssiid << " peCount=" << ncores;
  ESMC_LogDefault.Write(msg.str(), msgType);
  for (int i=0; i<ncores; i++){
    msg.str("");  // clear
    msg << prefix << "PE=" << i << " SSI=" << ssiid[i]
      << " SSIPE=" << ssipe[i];
    ESMC_LogDefault.Write(msg.str(), msgType);
  }
  msg.str("");  // clear
  msg << prefix << "ndevs=" << ndevs << " ndevsSSI=" << ndevsSSI;
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
  msg << prefix;
  for (auto i=0; i<ndevsSSI; i++)
    msg << " ssidevs[" << i << "]=" << ssidevs[i];
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
#ifndef ESMF_MPIUNI
  bool mpi_t_okay = true;
  // information about MPI implementation
  msg << prefix << "--- VMK::logSystem() MPI Layer ---------------------------";
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
  msg << prefix << "MPI_VERSION=" << XSTR(MPI_VERSION);
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
  msg << prefix << "MPI_SUBVERSION=" << XSTR(MPI_SUBVERSION);
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
#ifdef MPICH_VERSION
  std::string mpich_version(XSTR(MPICH_VERSION));
  mpich_version.erase(
    std::remove(mpich_version.begin(), mpich_version.end(),
    '"'), mpich_version.end());
  msg << prefix << "MPICH_VERSION=" << mpich_version;
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
  if (mpich_version.rfind("3.3", 0) == 0)  // broken mpi tools interface
    mpi_t_okay = false;
#endif
#ifdef CRAY_MPICH_VERSION
  std::string cray_mpich_version(XSTR(CRAY_MPICH_VERSION));
  cray_mpich_version.erase(
    std::remove(cray_mpich_version.begin(), cray_mpich_version.end(),
    '"'), cray_mpich_version.end());
  msg << prefix << "CRAY_MPICH_VERSION=" << cray_mpich_version;
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
  if (cray_mpich_version.rfind("8.1", 0) == 0)  // broken mpi tools interface
    mpi_t_okay = false;
#endif
  // initialize the MPI tool interface
  msg << prefix << "mpi_t_okay=" << mpi_t_okay;
  ESMC_LogDefault.Write(msg.str(), msgType);
  msg.str("");  // clear
  if (mpi_t_okay){
    int mpi_rc;
    int mpi_thread_level_provided;
    mpi_rc = MPI_T_init_thread(mpi_thread_level_requested,
      &mpi_thread_level_provided);
    if (mpi_rc != MPI_SUCCESS){
      int localrc;
      ESMC_LogDefault.MsgFoundError(ESMC_RC_INTNRL_BAD,
        "MPI_T_init_thread() did not return MPI_SUCCESS.",
        ESMC_CONTEXT, &localrc);
      throw localrc;  // bail out with exception
    }
    msg << prefix << "--- VMK::logSystem() MPI Tool Interface Control Vars ---";
    ESMC_LogDefault.Write(msg.str(), msgType);
    int num_cvar;
    mpi_rc = MPI_T_cvar_get_num(&num_cvar);
    if (mpi_rc != MPI_SUCCESS){
      int localrc;
      ESMC_LogDefault.MsgFoundError(ESMC_RC_INTNRL_BAD,
        "MPI_T_cvar_get_num() did not return MPI_SUCCESS.",
        ESMC_CONTEXT, &localrc);
      throw localrc;  // bail out with exception
    }
    char name[128], desc[1024];
    int nameLen, descLen, verbosity, binding, scope;
    MPI_T_enum enumtype;
    MPI_Datatype datatype;
    for (int i=0; i<num_cvar; i++){
      nameLen = sizeof(name);
      descLen = sizeof(desc);
      mpi_rc = MPI_T_cvar_get_info(i, name, &nameLen, &verbosity, &datatype,
        &enumtype, desc, &descLen, &binding, &scope);
      if (mpi_rc == MPI_SUCCESS){
        msg.str("");  // clear
        msg << prefix << "index=" << std::setw(4) << i << std::setw(60) << name
          << " : " << desc;
        ESMC_LogDefault.Write(msg.str(), msgType);
      }else if (mpi_rc != MPI_T_ERR_INVALID_INDEX){
        int localrc;
        ESMC_LogDefault.MsgFoundError(ESMC_RC_INTNRL_BAD,
          "Call to MPI_T_cvar_get_info() failed in unsupported way.",
          ESMC_CONTEXT, &localrc);
        throw localrc;  // bail out with exception
      }
    }
#if 0
    // testing to change the MPICH EAGER limit for the shared memory channel
    int ci;
    MPI_T_cvar_get_index("MPIR_CVAR_NEMESIS_SHM_EAGER_MAX_SZ", &ci);
    MPI_T_cvar_handle chandle; int nvals, eagersize;
    MPI_T_cvar_handle_alloc(ci, NULL, &chandle, &nvals);
    MPI_T_cvar_read(chandle, &eagersize);
    msg.str("");  // clear
    msg << prefix << "old MPIR_CVAR_NEMESIS_SHM_EAGER_MAX_SZ=" << eagersize;
    ESMC_LogDefault.Write(msg.str(), msgType);
    eagersize = 4000;
    MPI_T_cvar_write(chandle, &eagersize);
    MPI_T_cvar_read(chandle, &eagersize);
    msg.str("");  // clear
    msg << prefix << "new MPIR_CVAR_NEMESIS_SHM_EAGER_MAX_SZ=" << eagersize;
    ESMC_LogDefault.Write(msg.str(), msgType);
#endif
    mpi_rc = MPI_T_finalize();
    if (mpi_rc != MPI_SUCCESS){
      int localrc;
      ESMC_LogDefault.MsgFoundError(ESMC_RC_INTNRL_BAD,
        "MPI_T_finalize() did not return MPI_SUCCESS.",
        ESMC_CONTEXT, &localrc);
      throw localrc;  // bail out with exception
    }
  }
#endif
  msg.str("");  // clear
  msg << prefix << "--- VMK::logSystem() end ---------------------------------";
  ESMC_LogDefault.Write(msg.str(), msgType);
}


int VMK::getCurrentSsiPe()const{
#if (defined ESMF_OS_Linux || defined ESMF_OS_Unicos)
  return sched_getcpu();
#else
  return -1;
#endif
}

int VMK::getNpets(){
  return npets;
}


int VMK::getMypet(){
  return mypet;
}


esmf_pthread_t VMK::getMypthid(){
  return mypthid;
}


int VMK::getNcpet(int i){
  return ncpet[i];
}


int VMK::getNadevs(int i){
  return nadevs[i];
}


int VMK::getSsi(int i){
  return ssiid[cid[i][0]];
}


MPI_Comm VMK::getMpi_c(){
  return mpi_c;
}


int VMK::getNthreads(int i){
  int n=0;
  for (int j=0; j<npets; j++)
    if (pid[j]==pid[i]) ++n;
  return n;
}


int VMK::getTid(int i){
  return tid[i];
}

int VMK::getVas(int i){
  return pid[i];
}

int VMK::getLpid(int i){
  return lpid[i];
}

int VMK::getMaxTag(){
  int *value;
  int flag;
#if MPI_VERSION >= 2
  MPI_Comm_get_attr(MPI_COMM_WORLD, MPI_TAG_UB, &value, &flag);
#else
  // MPI_Attr_get is deprecated in MPI 2.0
  MPI_Attr_get(MPI_COMM_WORLD, MPI_TAG_UB, &value, &flag);
#endif
  if (flag)
    return *value;
  else
    return 0;
}

// --- VMKPlan methods ---


VMKPlan::VMKPlan(int _ndevlist, int *_devlist){
  // native constructor
  supportContributors = false; // by default do not support contributors
  eachChildPetOwnPthread = false; // by default do not create new Pthreads
  parentVMflag = 0; // default is to create a new VM for every child
  openmphandling = 0; // default to no OpenMP handling if not explicitly set
  openmpnumthreads = -1; // default to local peCount
  // invalidate the arrays
  spawnflag = NULL;
  contribute = NULL;
  cspawnid = NULL;
  // invalidate the VMK pointer array
  myvms = NULL;
  // set the default pthread specification
  minStackSize = VM_PTHREAD_STACKSIZE_USER;
  // set the default communication preferences
  pref_intra_process = PREF_INTRA_PROCESS_SHMHACK;
  pref_intra_ssi = PREF_INTRA_SSI_MPI1;
  pref_inter_ssi = PREF_INTER_SSI_MPI1;
  // invalidate members that deal with communicator of participating PETs
  lpid_mpi_g_part_map = NULL;
  commfreeflag = 0;
  // devlist
  ndevlist = _ndevlist;
  devlist = _devlist;
  // stdout and stderr redirect
  stdoutName = NULL;
  stderrName = NULL;
}


VMKPlan::~VMKPlan(){
  // native destructor
  vmkplan_garbage();
  if (lpid_mpi_g_part_map){
    delete [] lpid_mpi_g_part_map;
    lpid_mpi_g_part_map = NULL;
  }
  if (commfreeflag){
    MPI_Comm_free(&mpi_c_part);
    commfreeflag = 0;
  }
  if (stdoutName){
    delete [] stdoutName;
    stdoutName = NULL;
  }
  if (stderrName){
    delete [] stderrName;
    stderrName = NULL;
  }
}


void VMKPlan::vmkplan_garbage(){
  // perform garbage collection within a VMKPlan object
  if (spawnflag != NULL){
    delete [] spawnflag;
    delete [] contribute;
    delete [] cspawnid;
    spawnflag = NULL;
    contribute = NULL;
    cspawnid = NULL;
    myvms = NULL;     // this does NOT deallocate VM objects!
    if (!parentVMflag)
      delete [] petlist;
  }
}


int VMKPlan::vmkplan_nspawn(){
  // return number of PETs that are being spawned out of current PET
  return nspawn;
}


void VMKPlan::vmkplan_myvms(VMK **myvms){
  // set the internal myvms pointer array
  this->myvms = myvms;
}


void VMKPlan::vmkplan_mpi_c_part(VMK &vm){
  // set up the communicator of participating PETs
  int *grouplist = new int[nplist];     // that's big enough
  int *grouppetlist = new int[nplist];  // associated list of pets
  int n=0;  // counter
  for (int i=0; i<nplist; i++){
    int pet = petlist[i];
    int lpid = vm.getLpid(pet);
    int j;
    for (j=0; j<n; j++)
      if (grouplist[j] == lpid) break;
    if (j==n){
      grouplist[n] = lpid;
      grouppetlist[n] = pet;
      ++n;
    }
  }
  
  // all master PETs of the current vm must create the communicator
  int mypet = vm.getMypet();
  if (vm.getTid(mypet) == 0){
    // master PET in this VAS
    MPI_Group mpi_g;
    MPI_Comm_group(vm.mpi_c, &mpi_g);                 // parent's goup
    MPI_Group mpi_g_part;
    MPI_Group_incl(mpi_g, n, grouplist, &mpi_g_part); // child's group
    MPI_Comm_create(vm.mpi_c, mpi_g_part, &mpi_c_part);
    commfreeflag = 1;   // this PET is responsible for freeing the communicator
    MPI_Group_free(&mpi_g);
    MPI_Group_free(&mpi_g_part);
  }else{
    commfreeflag = 0;   // this PET is _not_ responsible for freeing the commu.
  }

  // reset those commfreeflags for PETs outside the group of participants
  int j;
  for (j=0; j<n; j++)
    if (mypet == grouppetlist[j]) break;
  if (j == n)
    commfreeflag = 0;

  lpid_mpi_g_part_map = new int[vm.getNpets()];
  for (int i=0; i<n; i++){
    lpid_mpi_g_part_map[grouplist[i]] = i;
  }
  
  delete [] grouppetlist;
  delete [] grouplist;
}


void VMKPlan::vmkplan_useparentvm(VMK &vm){
  // set up a VMKPlan that will run inside of parent VM
  parentVMflag = 1;
  npets = vm.npets;
  spawnflag = new int[npets];
  contribute = new int[npets];
  cspawnid = new int[npets];
  for (int i=0; i<npets; i++){
    spawnflag[i]=1;
    contribute[i]=0;
    cspawnid[i]=0;
  }
  // now deal with mypet specific members
  nspawn = spawnflag[vm.mypet];
}


void VMKPlan::vmkplan_maxthreads(VMK &vm){
  // set up a VMKPlan that will maximize the number of thread-pets
  vmkplan_maxthreads(vm, 0);
}


void VMKPlan::vmkplan_maxthreads(VMK &vm, int max){
  // set up a VMKPlan that will max. the number of thread-pets up to max
  vmkplan_maxthreads(vm, max, NULL, 0);
}


void VMKPlan::vmkplan_maxthreads(VMK &vm, int max,
  int pref_intra_process, int pref_intra_ssi, int pref_inter_ssi){
  // set up a VMKPlan that will max. the number of thread-pets up to max
  vmkplan_maxthreads(vm, max, NULL, 0, 
    pref_intra_process, pref_intra_ssi, pref_inter_ssi);
}


void VMKPlan::vmkplan_maxthreads(VMK &vm, int max, int *plist, int nplist){
  // set up a VMKPlan that will max. the number of thread-pets up to max
  // but only allow PETs listed in plist to participate
  // first do garbage collection on current object
  vmkplan_garbage();
  // now set stuff up...
  supportContributors = true; // this plan will allow contributors
  openmphandling = 3; // default to pin OpenMP threads
  openmpnumthreads = -1; // default to local peCount
  npets = vm.npets;
  if (nplist != 0)
    this->nplist = nplist;
  else
    this->nplist = npets;
  petlist = new int[npets];
  spawnflag = new int[npets];
  contribute = new int[npets];
  cspawnid = new int[npets];
  // setup petlist
  for (int i=0; i<npets; i++)
    petlist[i] = i;   //default sequence
  if (nplist != 0){
    // an explicit petlist was provided
    for (int i=0; i<nplist; i++){
      int pet = plist[i];
      int j;
      for (j=0; j<npets; j++){
        // search for 'pet'
        if (petlist[j] == pet) break;
      }
      // j is position that holds 'pet' -> swap elements
      petlist[j] = petlist[i];
      petlist[i] = pet;
    }
  }
  // set up a table for ssiid-to-petid mapping and invalidate all entries
  int *issiid = new int[npets]; // can only have as many different ssi as pets
  int *nssiid = new int[npets]; // can only have as many different ssi as pets
  for (int i=0; i<npets; i++)
    issiid[i]=-1;
  // loop over pets and set spawnflag
  for(int i=0; i<npets; i++){
    if (nplist != 0){
      int j;
      for (j=0; j<nplist; j++)
        if (plist[j]==i) break;
      if (j==nplist){
        spawnflag[i]=0;     // this PET is not spawn into new VMK
        contribute[i]=-1;   // don't contribute any cores
        cspawnid[i]=-1;     // invalidate
        continue;
      }
    }
    if (issiid[vm.ssiid[vm.cid[i][0]]]==-1){
      // new ssiid
      spawnflag[i]=1;     // spawn at least one thread in new virtual machine   
      contribute[i]=i;    // contribute cores to itself
      cspawnid[i]=0;      // contribute cores to itself
      issiid[vm.ssiid[vm.cid[i][0]]]=i;   // need this pets id for later
      nssiid[vm.ssiid[vm.cid[i][0]]]=1;   // count how many threads for this ssi
    }else{
      // same ssiid
      if (max <= 0 || nssiid[vm.ssiid[vm.cid[i][0]]] < max){
        // maximum degree of process multi-threading not yet reached
        ++nssiid[vm.ssiid[vm.cid[i][0]]]; // increment counter
        if (vm.pid[i]==vm.pid[issiid[vm.ssiid[vm.cid[i][0]]]]){
          // a previous pet with the same pid has been found before, but in 
          // order to minimize communications required on exit we'll have each
          // thread spawn...
          spawnflag[i]=1;
          contribute[i]=i;   // contribute cores to itself
          cspawnid[i]=0;     // contribute cores to itself
        }else{
          // so this was a pet that runs under a different process than a 
          // pet we found before on the same SSI, thus it shall not spawn
          // but contribute its cores to the previously found pet
          spawnflag[i]=0;
          contribute[i]=issiid[vm.ssiid[vm.cid[i][0]]];  // contribute cores
          cspawnid[i]=spawnflag[issiid[vm.ssiid[vm.cid[i][0]]]]; // to  pet
          ++spawnflag[issiid[vm.ssiid[vm.cid[i][0]]]];   // increment spawnflag
        }
      }else{
        // maximum number of threads per process reached -> start over
        nssiid[vm.ssiid[vm.cid[i][0]]]=0; // reset
        if (vm.pid[i]==vm.pid[issiid[vm.ssiid[vm.cid[i][0]]]]){
          // a previous pet with the same pid has been found before, 
          // therefore I cannot do anything with this pet...
          spawnflag[i]=0;     // don't spawn from this pet
          contribute[i]=-1;   // don't contribute any cores
          cspawnid[i]=-1;     // invalidate
        }else{
          // pet on same SSI but under new process -> make this process master
          spawnflag[i]=1;     // spawn at least one thread in new VMK
          contribute[i]=i;    // contribute cores to itself
          cspawnid[i]=0;      // contribute cores to itself
          issiid[vm.ssiid[vm.cid[i][0]]]=i;   // need this pets id for later
          nssiid[vm.ssiid[vm.cid[i][0]]]=1;   // count threads for this ssi
        }
      }
    }
  }
  delete [] issiid;
  delete [] nssiid;
  // now deal with mypet specific members
  nspawn = spawnflag[vm.mypet];
  if (nspawn > 1) eachChildPetOwnPthread = true;
}


int VMKPlan::vmkplan_maxthreads(VMK &vm, int max, int *plist, int nplist,
  int pref_intra_process, int pref_intra_ssi, int pref_inter_ssi,
  bool forceEachChildPetOwnPthread){
  // set the communication preferences
  if (pref_intra_process >= 0)
    this->pref_intra_process = pref_intra_process;
  if (pref_intra_ssi >= 0)
    this->pref_intra_ssi = pref_intra_ssi;
  if (pref_inter_ssi >= 0)
    this->pref_inter_ssi = pref_inter_ssi;
  vmkplan_maxthreads(vm, max, plist, nplist);
  if ((vm.isPthreadsEnabled()==false) && supportContributors) return 1; // error
  if (forceEachChildPetOwnPthread) eachChildPetOwnPthread=true;
  return 0;
}


void VMKPlan::vmkplan_minthreads(VMK &vm){
  // set up a VMKPlan that will only have single threaded pet
  // instantiations and claim all cores of pets that don't make it through
  vmkplan_minthreads(vm, 0);
}


void VMKPlan::vmkplan_minthreads(VMK &vm, int max){
  // set up a VMKPlan that will only have single threaded pet
  // instantiations and claim all cores of pets that don't make it through, up
  // to max cores/pet
  vmkplan_minthreads(vm, max, NULL, 0);
}


void VMKPlan::vmkplan_minthreads(VMK &vm, int max, int *plist, int nplist){
  // set up a VMKPlan that will only have single threaded pet
  // instantiations and claim all cores of pets that don't make it through, up
  // to max cores/pet but only allow PETs listed in plist to participate
  // first do garbage collection on current object
  vmkplan_garbage();
  // now set stuff up...
  supportContributors = true; // this plan will support contributors
  openmphandling = 3; // default to pin OpenMP threads
  openmpnumthreads = -1; // default to local peCount
  npets = vm.npets;
  if (nplist != 0)
    this->nplist = nplist;
  else
    this->nplist = npets;
  petlist = new int[npets];
  spawnflag = new int[npets];
  contribute = new int[npets];
  cspawnid = new int[npets];
  // setup petlist
  for (int i=0; i<npets; i++)
    petlist[i] = i;   //default sequence
  if (nplist != 0){
    // an explicit petlist was provided
    for (int i=0; i<nplist; i++){
      int pet = plist[i];
      int j;
      for (j=0; j<npets; j++){
        // search for 'pet'
        if (petlist[j] == pet) break;
      }
      // j is position that holds 'pet' -> swap elements
      petlist[j] = petlist[i];
      petlist[i] = pet;
    }
  }
  // need temporary array to hold core count for each lpid
  int *core_count = new int[vm.npets];        // maximum lpid is <= npets
  int *first_pet_index = new int[vm.npets];   // maximum lpid is <= npets
  // loop over pets and set spawnflag
  for(int i=0; i<npets; i++){
    if (nplist != 0){
      int j;
      for (j=0; j<nplist; j++)
        if (plist[j]==i) break;
      if (j==nplist){
        spawnflag[i]=0;     // this PET is not spawn into new VMK
        contribute[i]=-1;   // don't contribute any cores
        cspawnid[i]=-1;     // invalidate
        continue;
      }
    }
    if (vm.tid[i]==0){
      // only pass pets with tid 0
      spawnflag[i]=1;     // spawn one thread in new virtual machine from pet
      contribute[i]=i;    // contribute cores to itself
      cspawnid[i]=0;      // contribute cores to itself
      // record the number of cores that this pet brings in for this lpid
      core_count[vm.lpid[i]]=vm.ncpet[i];
      first_pet_index[vm.lpid[i]]=i;
    }else{
      // all other pets don't spawn, ...
      spawnflag[i]=0;     // don't spawn from this pet
      // ... but might contribute
      if (core_count[vm.lpid[i]] < max){
        // maximum number of cores not yet reached -> contribute all cores
        contribute[i]=first_pet_index[vm.lpid[i]];
        cspawnid[i]=0;      // contribute to the single-threaded pet spawned
        core_count[vm.lpid[i]]+=vm.ncpet[i]; // add this pet's cores to count
      }else{
        // maximum number of cores per pet has been reached -> don't contribute
        contribute[i]=-1;   // don't contribute any cores
        cspawnid[i]=-1;     // invalidate
      }
    }
  }
  delete [] core_count;
  delete [] first_pet_index;
  // now deal with mypet specific members
  nspawn = spawnflag[vm.mypet];
  if (nspawn > 1) eachChildPetOwnPthread = true;
}


int VMKPlan::vmkplan_minthreads(VMK &vm, int max, int *plist, int nplist,
  int pref_intra_process, int pref_intra_ssi, int pref_inter_ssi,
  bool forceEachChildPetOwnPthread){
  // set the communication preferences
  if (pref_intra_process >= 0)
    this->pref_intra_process = pref_intra_process;
  if (pref_intra_ssi >= 0)
    this->pref_intra_ssi = pref_intra_ssi;
  if (pref_inter_ssi >= 0)
    this->pref_inter_ssi = pref_inter_ssi;
  vmkplan_minthreads(vm, max, plist, nplist);
  if ((vm.isPthreadsEnabled()==false) && supportContributors) return 1; // error
  if (forceEachChildPetOwnPthread) eachChildPetOwnPthread=true;
  return 0;
}


void VMKPlan::vmkplan_maxcores(VMK &vm){
  // set up a VMKPlan that will have pets with the maximum number of cores
  // available
  vmkplan_maxcores(vm, 0);
}


void VMKPlan::vmkplan_maxcores(VMK &vm, int max){
  // set up a VMKPlan that will have pets with the maximum number of cores
  // available, but not more than max
  vmkplan_maxcores(vm, max, NULL, 0);
}


void VMKPlan::vmkplan_maxcores(VMK &vm, int max, int *plist, int nplist){
  // set up a VMKPlan that will have pets with the maximum number of cores
  // available, but not more than max and only use PETs listed in plist
  // first do garbage collection on current object
  vmkplan_garbage();
  // now set stuff up...
  supportContributors = true; // this plan will support contributors
  openmphandling = 3; // default to pin OpenMP threads
  openmpnumthreads = -1; // default to local peCount
  npets = vm.npets;
  if (nplist != 0)
    this->nplist = nplist;
  else
    this->nplist = npets;
  petlist = new int[npets];
  spawnflag = new int[npets];
  contribute = new int[npets];
  cspawnid = new int[npets];
  // setup petlist
  for (int i=0; i<npets; i++)
    petlist[i] = i;   //default sequence
  if (nplist != 0){
    // an explicit petlist was provided
    for (int i=0; i<nplist; i++){
      int pet = plist[i];
      int j;
      for (j=0; j<npets; j++){
        // search for 'pet'
        if (petlist[j] == pet) break;
      }
      // j is position that holds 'pet' -> swap elements
      petlist[j] = petlist[i];
      petlist[i] = pet;
    }
  }
  // set up a table for ssiid-to-petid mapping and invalidate all entries
  int *issiid = new int[npets]; // can only have as many different ssi as pets
  int *nssiid = new int[npets]; // can only have as many different ssi as pets
  for (int i=0; i<npets; i++)
    issiid[i]=-1;
  // loop over pets and set spawnflag
  for(int ii=0; ii<npets; ii++){
    int i = petlist[ii];  // consider petlist mapping
    if (nplist != 0){
      int j;
      for (j=0; j<nplist; j++)
        if (plist[j]==i) break;
      if (j==nplist){
        spawnflag[i]=0;     // this PET is not spawn into new VMK
        contribute[i]=-1;   // don't contribute any cores
        cspawnid[i]=-1;     // invalidate
        continue;
      }
    }
    if (issiid[vm.ssiid[vm.cid[i][0]]]==-1){
      // new ssiid
      spawnflag[i]=1;     // spawn at least one thread in new virtual machine   
      contribute[i]=i;    // contribute cores to itself
      cspawnid[i]=0;      // contribute cores to itself
      issiid[vm.ssiid[vm.cid[i][0]]]=i;   // need this pets id for later
      nssiid[vm.ssiid[vm.cid[i][0]]]=vm.ncpet[i]; // count how many cores
    }else{
      // same ssiid
      if (max <= 0 || nssiid[vm.ssiid[vm.cid[i][0]]] < max){
        // maximum degree of cores per pet not yet reached
        nssiid[vm.ssiid[vm.cid[i][0]]]+=vm.ncpet[i]; // increment counter
        // contribute cores to the previously found pet
        spawnflag[i]=0;
        contribute[i]=issiid[vm.ssiid[vm.cid[i][0]]];  // contribute cores
        cspawnid[i]=0;                                 // to  pet
      }else{
        // maximum number of cores per pet reached -> start over
        nssiid[vm.ssiid[vm.cid[i][0]]]=0; // reset
        // start spawning from this pet
        spawnflag[i]=1;     // spawn
        contribute[i]=i;    // contribute core to itself
        cspawnid[i]=0 ;     // contribute core to itself
        issiid[vm.ssiid[vm.cid[i][0]]]=i;   // need this pets id for later
        nssiid[vm.ssiid[vm.cid[i][0]]]=vm.ncpet[i]; // count how many cores
      }
    }
  }
  delete [] issiid;
  delete [] nssiid;
  // now deal with mypet specific members
  nspawn = spawnflag[vm.mypet];
  if (nspawn > 1) eachChildPetOwnPthread = true;
}


int VMKPlan::vmkplan_maxcores(VMK &vm, int max, int *plist, int nplist,
  int pref_intra_process, int pref_intra_ssi, int pref_inter_ssi,
  bool forceEachChildPetOwnPthread){
  // set the communication preferences
  if (pref_intra_process >= 0)
    this->pref_intra_process = pref_intra_process;
  if (pref_intra_ssi >= 0)
    this->pref_intra_ssi = pref_intra_ssi;
  if (pref_inter_ssi >= 0)
    this->pref_inter_ssi = pref_inter_ssi;
  vmkplan_maxcores(vm, max, plist, nplist);
  if ((vm.isPthreadsEnabled()==false) && supportContributors) return 1; // error
  if (forceEachChildPetOwnPthread) eachChildPetOwnPthread=true;
  return 0;
}


void VMKPlan::vmkplan_print(){
  // print info about the VMKPlan object
  printf("--- vmkplan_print start ---\n");
  printf("supportContributors = %d\n", supportContributors);
  printf("eachChildPetOwnPthread = %d\n", eachChildPetOwnPthread);
  printf("parentVMflag = %d\n", parentVMflag);
  printf("npets = %d\n", npets);
  for (int i=0; i<npets; i++)
    printf("  spawnflag[%d]=%d, contribute[%d]=%d, cspawnid[%d]=%d\n", 
      i, spawnflag[i], i, contribute[i], i, cspawnid[i]);
  printf("pref_intra_process:\t%d\n", pref_intra_process);
  printf("pref_intra_ssi:\t%d\n", pref_intra_ssi);
  printf("pref_inter_ssi:\t%d\n", pref_inter_ssi);
  printf("openmphandling   = %d\n", openmphandling);
  printf("openmpnumthreads = %d\n", openmpnumthreads);
  printf("--- vmkplan_print end ---\n");
}


// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// ~~~ Communication Handle and Communication Handle Queue
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~


void VMK::commqueueitem_link(commhandle *ch){
  commhandle *handle;
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_lock(pth_mutex2);
#endif
  if (nhandles==0){
    firsthandle=ch;
    ch->prev_handle=NULL;
    ch->next_handle=NULL;
  }else{
    handle=firsthandle;
    while (handle->next_handle!=NULL)
      handle=handle->next_handle;
    handle->next_handle=ch;
    ch->prev_handle=handle;
    ch->next_handle=NULL;
  }
  ++nhandles;
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_unlock(pth_mutex2);
#endif
}

int VMK::commqueueitem_unlink(commhandle *ch){
  commhandle *handle;
  int found = 0; // reset
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_lock(pth_mutex2);
#endif
  if(nhandles >= 1){
    handle=firsthandle;
    while (handle->next_handle!=NULL){
      if (handle==ch) break;
      handle=handle->next_handle;
    }
    if (handle==ch){
      // found commhandle in queue
      found = 1;
      --nhandles;
      if (handle->prev_handle==NULL){
        // commhandle was firsthandle in queue -> special treatment
        firsthandle=handle->next_handle;
        if (firsthandle != NULL)
          firsthandle->prev_handle=NULL;
      }else
        // regular unlink if commhandle was any other element in queue
        handle->prev_handle->next_handle=handle->next_handle;
      if (handle->next_handle!=NULL)
        // finish unlink by back linking next element to previous element
        handle->next_handle->prev_handle=handle->prev_handle;
    }
  }
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_unlock(pth_mutex2);
#endif
  return found;
}


int VMK::commtest(commhandle **ch, int *completeFlag, status *status){
  // test all of the communications pointed to by *ch. For completed
  // ones delete all of the inside contents of *ch (even if it is a
  // tree)
  // finally unlink the *ch container from the commqueue and delete the
  // container (only) if the *ch was part of the commqueue!
//fprintf(stderr, "(%d)VMK::commtest: nhandles=%d\n", mypet, nhandles);
//fprintf(stderr, "(%d)VMK::commtest: *ch=%p\n", mypet, *ch);
  int localrc=0;
  if (status) {
    memset (status, 0, sizeof (*status));     // quiet valgrind
    status->comm_type = VM_COMM_TYPE_MPIUNI;  // safe initialization
  }
  if ((ch!=NULL) && ((*ch)!=NULL)){
    // wait for all non-blocking requests in commhandle to complete
    int localCompleteFlag = 0;
    if ((*ch)->type==0){
      // this is a commhandle container
      for (int i=0; i<(*ch)->nelements; i++){
        localrc = commwait(&((*ch)->handles[i]));  // recursive call
        delete (*ch)->handles[i];
      }
      delete [] (*ch)->handles;
    }else if ((*ch)->type==1){
      // this commhandle contains MPI_Requests
      if (status)
        status->comm_type = VM_COMM_TYPE_MPI1;
      MPI_Status *mpi_s;
      if (status)
        mpi_s = &(status->mpi_s);
      else
        mpi_s = MPI_STATUS_IGNORE;
      // TODO: status will only reflect the last communiction in the i-loop!
      for (int i=0; i<(*ch)->nelements; i++){
//fprintf(stderr, "(%d)VMK::commtest: right before MPI_Test(): ch=%p\n",
//    mypet, &((*ch)->mpireq[i]));
#ifndef ESMF_NO_PTHREADS
        if (mpi_mutex_flag) pthread_mutex_lock(pth_mutex);
#endif
        localrc = MPI_Test(&((*ch)->mpireq[i]), &localCompleteFlag,
          mpi_s);
#ifndef ESMF_NO_PTHREADS
        if (mpi_mutex_flag) pthread_mutex_unlock(pth_mutex);
#endif
//fprintf(stderr, "(%d)VMK::commtest: right after MPI_Test()\n", mypet);
        if (status && localCompleteFlag){
          if (!(*ch)->sendFlag){
            int cancelled;
            MPI_Test_cancelled(mpi_s, &cancelled);
            if (!cancelled){
              if (lpid[mpi_s->MPI_SOURCE] == mpi_s->MPI_SOURCE)
                status->srcPet = mpi_s->MPI_SOURCE;
              else{
                for (int k=0; k<npets; k++)
                  if (lpid[k] == mpi_s->MPI_SOURCE)
                    status->srcPet = mpi_s->MPI_SOURCE;
              }
              status->tag     = mpi_s->MPI_TAG;
              status->error   = mpi_s->MPI_ERROR;
            }
          }
        }
      }
      if (localCompleteFlag)
        delete [] (*ch)->mpireq;
    }else if ((*ch)->type==-1){
      // this is a dummy commhandle and there is nothing to wait for...
      // ... but set localCompleteFlag
      localCompleteFlag = 1;      
    }else{
      std::stringstream msg;
      msg << "VMK::commtest():" << __LINE__
        << " only MPI non-blocking implemented";
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
      localrc = VMK_ERROR;
    }
    // if this *ch is in the request queue x-> unlink and delete
    if (localCompleteFlag){
      if (commqueueitem_unlink(*ch)){ 
        delete *ch; // delete the container commhandle that was linked
        *ch = NULL; // ensure this container will not point to anything
      }
    }
    if (completeFlag != NULL)
      *completeFlag = localCompleteFlag;
  }
  return localrc;
}


int VMK::commwait(commhandle **ch, status *status, int nanopause){
  // wait for all of the communications pointed to by *ch to complete
  // and delete all of the inside contents of *ch (even if it is a tree)
  // finally unlink the *ch container from the commqueue and delete the
  // container (only) if the *ch was part of the commqueue!
//fprintf(stderr, "(%d)VMK::commwait: nhandles=%d\n", mypet, nhandles);
//fprintf(stderr, "(%d)VMK::commwait: *ch=%p\n", mypet, *ch);
#ifdef VM_COMMQUEUELOG_on
  {
    std::stringstream msg;
    msg << "VMK::commwait():" << __LINE__ << " ch=" << ch << " *ch=" << *ch;
    ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
  }
#endif
  int localrc=0;
  if (status)
    status->comm_type = VM_COMM_TYPE_MPIUNI;  // safe initialization
  if ((epoch!=epochBuffer) && (ch!=NULL) && ((*ch)!=NULL)){
#ifdef VM_COMMQUEUELOG_on
  {
    std::stringstream msg;
    msg << "VMK::commwait():" << __LINE__ << " processing...";
    ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
  }
#endif
    // wait for all non-blocking requests in commhandle to complete
    if ((*ch)->type==0){
      // this is a commhandle container
#ifdef VM_COMMQUEUELOG_on
  {
    std::stringstream msg;
    msg << "VMK::commwait():" << __LINE__ << " commhandle container -> recurse"
      << " nelements=" << (*ch)->nelements;
    ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
  }
#endif
      for (int i=0; i<(*ch)->nelements; i++){
        localrc = commwait(&((*ch)->handles[i]));  // recursive call
        delete (*ch)->handles[i];
      }
      delete [] (*ch)->handles;
    }else if ((*ch)->type==1){
      // this commhandle contains MPI_Requests
#ifdef VM_COMMQUEUELOG_on
  {
    std::stringstream msg;
    msg << "VMK::commwait():" << __LINE__ << " MPI_Requests"
      << " nelements=" << (*ch)->nelements;
    ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
  }
#endif
      if (status)
        status->comm_type = VM_COMM_TYPE_MPI1;
      MPI_Status *mpi_s;
      if (status)
        mpi_s = &(status->mpi_s);
      else
        mpi_s = MPI_STATUS_IGNORE;
      // TODO: status will only reflect the last communiction in the i-loop!
      for (int i=0; i<(*ch)->nelements; i++){
//fprintf(stderr, "MPI_Wait: ch=%p\n", &((*ch)->mpireq[i]));
        if (nanopause){
          // use nanosleep to pause between tests to lower impact on CPU load
#ifdef ESMF_NO_NANOSLEEP
#else
#if !defined (ESMF_OS_MinGW)
          struct timespec dt = {0, nanopause};
#endif
#endif
          int completeFlag = 0;
          for(;;){
#ifndef ESMF_NO_PTHREADS
            if (mpi_mutex_flag) pthread_mutex_lock(pth_mutex);
#endif
            localrc = MPI_Test(&((*ch)->mpireq[i]), &completeFlag,
              mpi_s);
#ifndef ESMF_NO_PTHREADS
            if (mpi_mutex_flag) pthread_mutex_unlock(pth_mutex);
#endif
            if (completeFlag) break;
#ifdef ESMF_NO_NANOSLEEP
#else
#if !defined (ESMF_OS_MinGW)
            nanosleep(&dt, NULL);
#else
            Sleep (1); // 1 millisec delay
#endif
#endif
          }
          if (status){
            if (!(*ch)->sendFlag){
              int cancelled;
              MPI_Test_cancelled(mpi_s, &cancelled);
              if (!cancelled){
                if (lpid[mpi_s->MPI_SOURCE] == mpi_s->MPI_SOURCE)
                  status->srcPet = mpi_s->MPI_SOURCE;
                else{
                  for (int k=0; k<npets; k++)
                    if (lpid[k] == mpi_s->MPI_SOURCE)
                      status->srcPet = mpi_s->MPI_SOURCE;
                }
                status->tag     = mpi_s->MPI_TAG;
                status->error   = mpi_s->MPI_ERROR;
              }
            }
          }
        }else{
#ifndef ESMF_NO_PTHREADS
          if (mpi_mutex_flag) pthread_mutex_lock(pth_mutex);
#endif
#ifdef VM_COMMQUEUELOG_on
          MPI_Status mpis;
          localrc = MPI_Wait(&((*ch)->mpireq[i]), &mpis);
          int canc;
          MPI_Test_cancelled(&mpis, &canc);
          int cnt;
          MPI_Get_count(&mpis, MPI_BYTE, &cnt);
          {
            std::stringstream msg;
            msg << "VMK::commwait() <DEBUG-WAIT>: " << __LINE__
              << " canc=" << canc
              << " src=" << mpis.MPI_SOURCE
              << " tag=" << mpis.MPI_TAG
              << " cnt=" << cnt;
            ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
          }
#else
          localrc = MPI_Wait(&((*ch)->mpireq[i]), mpi_s);
#endif

#ifndef ESMF_NO_PTHREADS
          if (mpi_mutex_flag) pthread_mutex_unlock(pth_mutex);
#endif
          if (status){
            if (!(*ch)->sendFlag){
              int cancelled;
              MPI_Test_cancelled(mpi_s, &cancelled);
              if (!cancelled){
                if (lpid[mpi_s->MPI_SOURCE] == mpi_s->MPI_SOURCE)
                  status->srcPet = mpi_s->MPI_SOURCE;
                else{
                  for (int k=0; k<npets; k++)
                    if (lpid[k] == mpi_s->MPI_SOURCE)
                      status->srcPet = mpi_s->MPI_SOURCE;
                }
                status->tag     = mpi_s->MPI_TAG;
                status->error   = mpi_s->MPI_ERROR;
              }
            }
          }
        }
      }
      delete [] (*ch)->mpireq;
#if 0
    //TODO: totally wrong code here!!!!
    }else if ((*ch)->type==5){
      // this commhandle is based on POSIXIPC share memory channels
      if ((*ch)->ptr){
        // This transfer was not complete when it was initiated -> wait now
        volatile void *ptr = *((void **)((*ch)->ptr));
        while (ptr);
      }
#endif
    }else if ((*ch)->type==-1){
      // this is a dummy commhandle and there is nothing to wait for...
    }else{
      std::stringstream msg;
      msg << "VMK::commwait():" << __LINE__
        << " only MPI non-blocking implemented";
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
      localrc = VMK_ERROR;
    }
  }
  // if this *ch is in the request queue x-> unlink and delete
  if (commqueueitem_unlink(*ch)){
#ifdef VM_COMMQUEUELOG_on
  {
    std::stringstream msg;
    msg << "VMK::commwait():" << __LINE__ << " unlinked *ch=" << *ch
      << " now delete!";
    ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
  }
#endif
    delete *ch; // delete the container commhandle that was linked
    *ch = NULL; // ensure this container will not point to anything
  }
  return localrc;
}


void VMK::commqueuewait(){
#ifdef VM_COMMQUEUELOG_on
  {
    std::stringstream msg;
    msg << "VMK::commqueuewait():" << __LINE__ << " nhandles=" <<
      nhandles;
    ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
  }
#endif
  int n=nhandles;
  commhandle *fh;
  for (int i=0; i<n; i++){
    fh = firsthandle;
    commwait(&fh);
  }
#ifdef VM_COMMQUEUELOG_on
  {
    std::stringstream msg;
    msg << "VMK::commqueuewait():" << __LINE__ << " nhandles=" <<
      nhandles;
    ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
  }
#endif
}


void VMK::commcancel(commhandle **commh){
//fprintf(stderr, "VMK::commcancel: nhandles=%d\n", nhandles);
//fprintf(stderr, "VMK::commcancel: commh=%p\n", (*commh));
  if ((*commh)!=NULL){
    // cancel all non-blocking requests in commhandle to complete
    if ((*commh)->type==0){
      // this is a commhandle container
      for (int i=0; i<(*commh)->nelements; i++){
        commcancel(&((*commh)->handles[i]));  // recursive call
      }
    }else if ((*commh)->type==1){
      // this commhandle contains MPI_Requests
      for (int i=0; i<(*commh)->nelements; i++){
//fprintf(stderr, "MPI_Cancel: commh=%p\n", &((*commh)->mpireq[i]));
#ifndef ESMF_NO_PTHREADS
        if (mpi_mutex_flag) pthread_mutex_lock(pth_mutex);
#endif
        MPI_Cancel(&((*commh)->mpireq[i]));
#ifndef ESMF_NO_PTHREADS
        if (mpi_mutex_flag) pthread_mutex_unlock(pth_mutex);
#endif
      }
    }else{
      std::stringstream msg;
      msg << "VMK::commwait():" << __LINE__
        << " only MPI non-blocking implemented";
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
    }
  }
}


bool VMK::cancelled(status *status){
  if (status->comm_type == VM_COMM_TYPE_MPI1){
    int flag;
    MPI_Test_cancelled(&(status->mpi_s), &flag);
    if (flag)
      return true;
    else
      return false;
  }else{
    return false;
  }
}


// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// ~~~ Epoch support
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

void VMK::epochInit(){
#undef  ESMC_METHOD
#define ESMC_METHOD "ESMCI::VMK::epochInit()"
#ifdef VM_EPOCHLOG_on
  ESMC_LogDefault.Write("epochInit", ESMC_LOGMSG_DEBUG);
#endif
  epoch=epochNone;
  epochSetFirst();
}

void VMK::epochFinal(){
#undef  ESMC_METHOD
#define ESMC_METHOD "ESMCI::VMK::epochFinal()"
  // loop over the sendMap and wait for outstanding comms
#ifdef VM_EPOCHLOG_on
  ESMC_LogDefault.Write("epochFinal", ESMC_LOGMSG_DEBUG);
#endif
  std::map<int, sendBuffer>:: iterator its;
  for (its=sendMap.begin(); its!=sendMap.end(); ++its){
    sendBuffer *sm = &(its->second);
#ifdef VM_EPOCHLOG_on
#if (EPOCH_BUFFER_OPTION == 0)
    // use strstream
    void *buffer = (void *)sm->stream.str();  // access the buffer -> freeze
    unsigned long long int size = sm->stream.pcount(); // bytes in stream buffer
#elif (EPOCH_BUFFER_OPTION == 1)
    // use stringstream
    void *buffer = (void *)sm->streamBuffer.data(); // access contig. buffer
    unsigned long long int size = sm->streamBuffer.size(); // bytes in stream buffer
#elif (EPOCH_BUFFER_OPTION == 2)
    // use vector<char>
    void *buffer = (void *)&(sm->charBuffer[0]);  // access the buffer
    unsigned long long int size = sm->charBuffer.size(); // bytes in buffer
#endif
    std::stringstream msg;
    msg << "epochBuffer:" << __LINE__ << " ready to clear outstanding comm:"
    << " dst=" << getVas(lpid[its->first]) << " size=" << size
    << " buffer=" << buffer;
    ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
#endif
    sm->clear();
    // drain acknowledge data structure
    while (sm->ackQueue.size() > 0){
#ifdef VM_EPOCHLOG_on
      std::stringstream msg;
      msg << "epochBuffer:" << __LINE__ << " drain unacknowledged sends to"
        << " dst=" << getVas(lpid[its->first]) << ": "
        << sm->ackQueue.size();
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
#endif
      ackElement *ackE = &(sm->ackQueue.front());
      MPI_Wait(&(ackE->ackReq), MPI_STATUS_IGNORE);
      sm->ackQueue.pop();
    }
  }
  sendMap.clear();  // completely clear the sendMap
  recvMap.clear();  // completely clear the recvMap
}

void VMK::epochEnter(vmEpoch epoch_, bool keepAlloc, int throttle){
#undef  ESMC_METHOD
#define ESMC_METHOD "ESMCI::VMK::epochEnter()"
  epoch=epoch_;
  if (throttle < 1){
    int localrc;
    ESMC_LogDefault.MsgFoundError(ESMC_RC_ARG_OUTOFRANGE,
      "The value of throttle must be greater than zero.",
      ESMC_CONTEXT, &localrc);
    throw localrc;  // bail out with exception
  }
  epochThrottle = throttle;
  epochSetFirst(!keepAlloc);  // potentially test and release inactive buffers
}

void VMK::epochExit(bool keepAlloc){
#undef  ESMC_METHOD
#define ESMC_METHOD "ESMCI::VMK::epochExit()"
  if (epoch==epochBuffer){
    // loop over the sendMap and post non-blocking sends
    std::map<int, sendBuffer>:: iterator its;
    for (its=sendMap.begin(); its!=sendMap.end(); ++its){
      sendBuffer *sm = &(its->second);
      int tag = getDefaultTag(mypet,its->first);
#ifdef VM_EPOCHLOG_on
      {
        std::stringstream msg;
        msg << "epochBuffer:" << __LINE__ << " unacknowledged sends to"
          << " dst=" << getVas(lpid[its->first]) << ": "
          << sm->ackQueue.size();
        ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
      }
#endif
      // throttle
      while ((int)sm->ackQueue.size() > epochThrottle){
#ifdef VM_EPOCHLOG_on
        std::stringstream msg;
        msg << "epochBuffer:" << __LINE__ << "  throttling sends to"
          << " dst=" << getVas(lpid[its->first]);
        ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
#endif
        ackElement *ackE = &(sm->ackQueue.front());
        MPI_Wait(&(ackE->ackReq), MPI_STATUS_IGNORE);
        sm->ackQueue.pop();
#ifdef VM_EPOCHLOG_on
        msg.str(""); // clear
        msg << "epochBuffer:" << __LINE__ << " unacknowledged sends to"
          << " dst=" << getVas(lpid[its->first]) << ": "
          << sm->ackQueue.size();
        ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
#endif
      }
      bool needAck = false;
#ifdef VM_EPOCHMEMLOG_on
      VM::logMemInfo(std::string("VMK::epochExit:1.0"));
#endif
#if (EPOCH_BUFFER_OPTION == 0)
      // use strstream
      void *buffer = (void *)sm->stream.str();  // access the buffer -> freeze
      unsigned long long size = sm->stream.pcount(); // bytes in stream buffer
      sm->stream.seekp(0);  // reset stream to the beginning (not affect buff)
#elif (EPOCH_BUFFER_OPTION == 1)
      // use stringstream
      sm->streamBuffer = sm->stream.str();  // copy data into contiguous buffer
      sm->stream.str("");                   // clear out stream
      void *buffer = (void *)sm->streamBuffer.data(); // access contig. buffer
      unsigned long long size = sm->streamBuffer.size();
#elif (EPOCH_BUFFER_OPTION == 2)
      // use vector<char>
      void *buffer = (void *)&(sm->charBuffer[0]);  // access the buffer
      unsigned long long int size = sm->charBuffer.size(); // bytes in buffer
      sm->charBuffer.resize(0); // reset buffer, without affecting allocation
#endif
#ifdef VM_EPOCHMEMLOG_on
      VM::logMemInfo(std::string("VMK::epochExit:2.0"));
#endif
      if (size > 0){
#if (defined VM_EPOCHLOG_on || defined VM_SIZELOG_on)
        std::stringstream msg;
        msg << "epochBuffer:" << __LINE__ << " ready to post non-blocking send:"
          << " dst=" << getVas(lpid[its->first])
          << " size=" << size
          << " buffer=" << buffer;
        ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
        double t0; wtime(&t0);
#endif
        if (size <= VM_MPI_SIZE_LIMIT)
          // small message
          MPI_Isend(buffer, size, MPI_BYTE, lpid[its->first], tag, mpi_c,
            &sm->mpireq);
        else{
#if (EPOCH_BUFFER_OPTION == 2)
          // large message
          // - determine smallest customType to keep count < VM_MPI_SIZE_LIMIT
          int byteCount = 1;
          int i;
          for (i=0; i<(signed)customType.size(); i++){
            byteCount *= 2;
            if (size / byteCount < VM_MPI_SIZE_LIMIT) break;
          }
          if (i>=(signed)customType.size()){
            int localrc;
            ESMC_LogDefault.MsgFoundError(ESMC_RC_INTNRL_BAD,
              "Message too large to be supported",
              ESMC_CONTEXT, &localrc);
            throw localrc;  // bail out with exception
          }
          // - might need to pad the buffer to be a multiple of the customType
          int extra = size%byteCount;
          if (extra){
            size += byteCount - extra;
            sm->charBuffer.resize(size);
          }
          // - use the determined customType to send the message
          int count = size/byteCount;
          MPI_Isend(buffer, count, customType[i], lpid[its->first], tag, mpi_c,
            &sm->mpireq);
#else
          // large messages not supported
          int localrc;
          ESMC_LogDefault.MsgFoundError(ESMC_RC_INTNRL_BAD,
            "Message larger than 2GiB only supported for EPOCH_BUFFER_OPTION 2.",
            ESMC_CONTEXT, &localrc);
          throw localrc;  // bail out with exception
#endif
        }
        needAck = true;
#ifdef VM_EPOCHLOG_on
        double t1; wtime(&t1);
        msg.str(""); // clear
        msg << "epochBuffer:" << __LINE__ << " time in non-blocking send: "
          << t1 - t0 << " seconds";
        ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
#endif
      }
#ifdef VM_EPOCHMEMLOG_on
      VM::logMemInfo(std::string("VMK::epochExit:3.0"));
#endif
      if (needAck){
        // post the receive of the acknowledge message from receiver for throttle
        sm->ackQueue.push(ackElement());
        ackElement *ackE = &(sm->ackQueue.back());
        MPI_Irecv(&(ackE->ackDummy), sizeof(int), MPI_BYTE, lpid[its->first], tag,
          mpi_c, &(ackE->ackReq));
#ifdef VM_EPOCHLOG_on
        std::stringstream msg;
        msg << "epochBuffer:" << __LINE__ << " unacknowledged sends to"
          << " dst=" << getVas(lpid[its->first]) << ": "
          << sm->ackQueue.size();
        ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
#endif
      }
#ifdef VM_EPOCHMEMLOG_on
      VM::logMemInfo(std::string("VMK::epochExit:4.0"));
#endif
    }
    if (!keepAlloc){
      // clear the recvMap, freeing all receive buffers held
      // use this option in case the receiving side is tight on memory
      recvMap.clear();
    }
  }
  // reset the epoch member
  epoch=epochNone;
}

void VMK::epochSetFirst(bool testAndClearBuffers){
#undef  ESMC_METHOD
#define ESMC_METHOD "ESMCI::VMK::epochSetFirst()"
  // send buffer
  std::map<int, sendBuffer>:: iterator its;
  std::map<int, sendBuffer>:: iterator itErase = sendMap.end();
  for (its=sendMap.begin(); its!=sendMap.end(); ++its){
    if (itErase!=sendMap.end()){
#ifdef VM_EPOCHLOG_on
      std::stringstream msg;
      msg << "epochBuffer:" << __LINE__ << " erase sendBuffer to"
        << " dst=" << getVas(lpid[itErase->first]);
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
#endif
      // hold on to ackQueue, erase itErase from sendMap, recreate and restore
      std::queue<ackElement> ackQueue = itErase->second.ackQueue;
      int key = itErase->first;
      sendMap.erase(itErase);
      itErase = sendMap.end();
      sendMap[key].ackQueue = ackQueue;
      sendMap[key].firstFlag = true;
    }
    its->second.firstFlag=true;
    if(testAndClearBuffers){
      if (its->second.clear(true)) itErase = its;
    }else
      itErase = sendMap.end();
  }
  // handle case where the very last element needs to be erased
  if (itErase!=sendMap.end()){
#ifdef VM_EPOCHLOG_on
    std::stringstream msg;
    msg << "epochBuffer:" << __LINE__ << " erase sendBuffer to"
      << " dst=" << getVas(lpid[itErase->first]);
    ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
#endif
    // hold on to ackQueue, erase itErase from sendMap, recreate and restore
    std::queue<ackElement> ackQueue = itErase->second.ackQueue;
    int key = itErase->first;
    sendMap.erase(itErase);
    itErase = sendMap.end();
    sendMap[key].ackQueue = ackQueue;
    sendMap[key].firstFlag = true;
  }
  // receive buffer
  std::map<int, recvBuffer>:: iterator itr;
  for (itr=recvMap.begin(); itr!=recvMap.end(); ++itr){
    itr->second.firstFlag=true;
  }
}

bool VMK::sendBuffer::clear(bool justTest){
#undef  ESMC_METHOD
#define ESMC_METHOD "ESMCI::VMK::sendBuffer::clear"
#ifdef VM_EPOCHLOG_on
  std::stringstream msg;
  msg << "epochBuffer:" << __LINE__ << " ready to clear outstanding comm";
  ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
#endif
#ifdef VM_EPOCHMEMLOG_on
  VM::logMemInfo(std::string("VMK::sendBuffer::clear():1.0"));
#endif
  bool done = false;
  if (mpireq != MPI_REQUEST_NULL){
    if (justTest){
#ifdef VM_EPOCHLOG_on
      std::stringstream msg;
      msg << "epochBuffer:" << __LINE__ << " posting MPI_Test()";
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
      double t0; wtime(&t0);
#endif
      int flag;
      MPI_Test(&mpireq, &flag, MPI_STATUS_IGNORE); // sets mpireq to MPI_REQUEST_NULL if done
      done = (bool)flag;
#ifdef VM_EPOCHLOG_on
      double t1; wtime(&t1);
      msg.str(""); // clear
      msg << "epochBuffer:" << __LINE__ << " returned from MPI_Test(), flag: "
        << (bool)flag;
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
#endif
    }else{
#ifdef VM_EPOCHLOG_on
      std::stringstream msg;
      msg << "epochBuffer:" << __LINE__ << " posting MPI_Wait()";
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
      double t0; wtime(&t0);
#endif
      MPI_Wait(&mpireq, MPI_STATUS_IGNORE); // sets mpireq to MPI_REQUEST_NULL
      done = true;
#ifdef VM_EPOCHLOG_on
      double t1; wtime(&t1);
      msg.str(""); // clear
      msg << "epochBuffer:" << __LINE__ << " time in MPI_Wait(): "
        << t1 - t0 << " seconds";
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
#endif
      
    }
  }
#ifdef VM_EPOCHMEMLOG_on
  VM::logMemInfo(std::string("VMK::sendBuffer::clear():2.0"));
#endif
#if (EPOCH_BUFFER_OPTION == 0)
  // use strstream
  stream.freeze(false); // unfreeze the persistent buffer for deallocation
#elif (EPOCH_BUFFER_OPTION == 1)
  // use stringstream
  streamBuffer.clear(); // done with buffer
#elif (EPOCH_BUFFER_OPTION == 2)
  // use vector<char>
  std::vector<char>().swap(charBuffer); // done with buffer swap out of scope
#endif
#ifdef VM_EPOCHMEMLOG_on
  VM::logMemInfo(std::string("VMK::sendBuffer::clear():3.0"));
#endif
  return done;
}


// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// ~~~ Communication Calls
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

int VMK::getDefaultTag(int src, int dst){
  int tag = 1000*src+dst;   // default tag to simplify debugging
  // make sure to stay below max tag
  int maxTag = getMaxTag();
  if (maxTag > 0)
    tag = tag%maxTag;
  else
    tag = 0;
  return tag;
}


int VMK::send(const void *message, unsigned long long int size, int dest,
  int tag){
#undef  ESMC_METHOD
#define ESMC_METHOD "ESMCI::VMK::send()"
  // p2p send
#if (VERBOSITY > 9)
  printf("sending to: %d, %d\n", dest, lpid[dest]);
#endif
  int localrc=0;
  shared_mp *shmp;
  pipc_mp *pipcmp;
  int scpsize;
  char *pdest;
  char *psrc;
  int i;
  char *mess;
  // switch into the appropriate implementation
  switch(sendChannel[dest].comm_type){
  case VM_COMM_TYPE_MPI1:
    // MPI-1 implementation
    void *messageC; // for MPI C interface convert (const void *) -> (void *)
    memcpy(&messageC, &message, sizeof(void *));
    // use mutex to serialize mpi comm calls if mpi thread support requires it
#ifndef ESMF_NO_PTHREADS
    if (mpi_mutex_flag) pthread_mutex_lock(pth_mutex);
#endif
    if (tag == -1) tag = getDefaultTag(mypet,dest);
#ifdef VM_SIZELOG_on
    {
      std::stringstream msg;
      msg << "VMK::send():" << __LINE__ << ", size=" << size;
      if (size <= VM_MPI_SIZE_LIMIT)
        msg << " in one chunk";
      else
        msg << " in multiple chunks";
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
    }
#endif
    if (size <= VM_MPI_SIZE_LIMIT)
      localrc = MPI_Send(messageC, size, MPI_BYTE, lpid[dest], tag, mpi_c);
    else{
      // must send in multiple stages
      unsigned long long _size = size;
      char *messageCC = (char *)messageC;
      i=0;
      while(_size > VM_MPI_SIZE_LIMIT){
        localrc = MPI_Send(messageCC, VM_MPI_SIZE_LIMIT, MPI_BYTE, lpid[dest],
          tag+i, mpi_c);
        i++;
        _size -= VM_MPI_SIZE_LIMIT;
        messageCC += VM_MPI_SIZE_LIMIT;
      }
      if (_size > 0)
        localrc = MPI_Send(messageCC, _size, MPI_BYTE, lpid[dest], tag+i,
          mpi_c);
    }
#ifndef ESMF_NO_PTHREADS
    if (mpi_mutex_flag) pthread_mutex_unlock(pth_mutex);
#endif
    break;
  case VM_COMM_TYPE_PTHREAD:
    // Pthread implementation
    shmp = sendChannel[dest].shmp;  // shared memory mp channel
    shmp->ptr_src = message;                        // set the source pointer
    // synchronize with recv()
#ifndef ESMF_NO_PTHREADS
    pthread_mutex_lock(&(shmp->mutex1));
    shmp->tcounter++;
    if (shmp->tcounter < 2){
      // wait for recv()
      pthread_cond_wait(&(shmp->cond1), &(shmp->mutex1));
    }else{
      // reset counter and wake up recv()
      shmp->tcounter = 0;
      pthread_cond_broadcast(&(shmp->cond1));
    }
    pthread_mutex_unlock(&(shmp->mutex1));
#endif
    // now ptr_src and ptr_dst are valid for this message
    scpsize = size/2;   // send takes the lower half
    pdest = (char *)shmp->ptr_dst;
    psrc = (char *)shmp->ptr_src;
    // do the actual memcpy
    memcpy(pdest, psrc, scpsize);
    // synchronize with recv()
#ifndef ESMF_NO_PTHREADS
    pthread_mutex_lock(&(shmp->mutex2));
    shmp->tcounter++;
    if (shmp->tcounter < 2){
      // wait for recv()
      pthread_cond_wait(&(shmp->cond2), &(shmp->mutex2));
    }else{
      // reset counter and wake up recv()
      shmp->tcounter = 0;
      pthread_cond_broadcast(&(shmp->cond2));
    }
    pthread_mutex_unlock(&(shmp->mutex2));
#endif
    break;
  case VM_COMM_TYPE_SHMHACK:
    // Shared memory hack sync with spin-lock
    shmp = sendChannel[dest].shmp;  // shared memory mp channel
    if (size<=SHARED_BUFFER){
      // use buffer
      pdest = shmp->buffer;
      // wait until buffer is ready to be used
      sync_buffer_wait_empty(&shmp->shms, 0);
      // do the actual memcpy
      memcpy(pdest, message, size);
      // set flag indicating that send's memcpy() is done and buffer is valid
      sync_buffer_flag_fill(&shmp->shms, 0);
    }else{
      // don't use buffer
      shmp->ptr_src = message;                        // set the source pointer
      // synchronize with recv()
      sync_a_flip(&shmp->shms);
      // now ptr_src and ptr_dst are valid for this message
      scpsize = size/2;   // send takes the lower half
      pdest = (char *)shmp->ptr_dst;
      psrc = (char *)shmp->ptr_src;
      // do the actual memcpy
      memcpy(pdest, psrc, scpsize);
      // synchronize with recv()
      sync_a_flop(&shmp->shms);
    }
    break;
  case VM_COMM_TYPE_POSIXIPC:
    // Shared memory hack sync with spin-lock
    pipcmp = sendChannel[dest].pipcmp;  // shared memory mp channel
    i=0;
    mess = (char *)message;
    while (size>PIPC_BUFFER){
      pdest = pipcmp->buffer[i];
      // wait until buffer is ready to be used
      sync_buffer_wait_empty(&pipcmp->shms, i);
      // do the actual memcpy
      memcpy(pdest, mess, PIPC_BUFFER);
      // set flag indicating that send's memcpy() is done and buffer is valid
      sync_buffer_flag_fill(&pipcmp->shms, i);
      size -= PIPC_BUFFER;
      mess += PIPC_BUFFER;
      i = i^1; // 0->1  or  1->0
    }
    // do the remaining parts of the message
    pdest = pipcmp->buffer[i];
    // wait until buffer is ready to be used
    sync_buffer_wait_empty(&pipcmp->shms, i);
    // do the actual memcpy
    memcpy(pdest, mess, size);
    // set flag indicating that send's memcpy() is done and buffer is valid
    sync_buffer_flag_fill(&pipcmp->shms, i);
    break;
  case VM_COMM_TYPE_MPIUNI:
    // Shared memory hack for mpiuni
    // TODO: this assumes that send will arrive first, otherwise this will hang
    shmp = sendChannel[dest].shmp;  // shared memory mp channel
    if (size<=SHARED_BUFFER){
      // buffer is sufficient
      pdest = shmp->buffer;
      // wait until buffer is ready to be used
      sync_buffer_wait_empty(&shmp->shms, 0);
      // do the actual memcpy
      memcpy(pdest, message, size);
      // set flag indicating that send's memcpy() is done and buffer is valid
      sync_buffer_flag_fill(&shmp->shms, 0);
    }else{
      // buffer is insufficient
      ESMC_LogDefault.MsgFoundError(ESMC_RC_INTNRL_BAD,
        "buffer insufficient!", ESMC_CONTEXT, &localrc);
        return localrc;
    }
    break;
  default:
    printf("unknown comm_type.\n");
    break;
  }
  return localrc;
}


int VMK::send(const void *message, unsigned long long int size, int dest,
  commhandle **ch, int tag){
#undef  ESMC_METHOD
#define ESMC_METHOD "ESMCI::VMK::send()"
  // p2p send non-blocking
//fprintf(stderr, "VMK::send: ch=%p\n", *ch);
#if (VERBOSITY > 9)
  printf("sending to: %d, %d\n", dest, lpid[dest]);
#endif
  int localrc=0;
  shared_mp *shmp;
  pipc_mp *pipcmp;
  int scpsize;
  char *pdest;
  int sendCount;
  // check if this needs a new entry in the request queue
  if (*ch==NULL){
    *ch = new commhandle;
    commqueueitem_link(*ch);
  }
  // switch into the appropriate implementation
  switch(sendChannel[dest].comm_type){
  case VM_COMM_TYPE_MPI1:
    if (tag == -1) tag = getDefaultTag(mypet,dest);
    if (epoch==epochBuffer){
      sendBuffer *sm = &(sendMap[dest]);
#if (defined VM_EPOCHLOG_on || defined VM_SIZELOG_on)
      std::stringstream msg;
      msg << "epochBuffer:" << __LINE__ << " non-blocking send to" <<
      " dst=" << dest <<
      " firstFlag=" << sm->firstFlag <<
      " msg size=" << size;
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
#endif
      if (sm->firstFlag){
        // deal with the first actions here
        sm->firstFlag = false;  // reset the flag
        sm->clear();  // wait for outstanding comm and clear out.
      }
      // append the message into the epoch buffer stream
#if (EPOCH_BUFFER_OPTION == 0 || EPOCH_BUFFER_OPTION == 1)
      append(sm->stream, size);
      append(sm->stream, tag);
      sm->stream.write((const char*)message, size);
#elif (EPOCH_BUFFER_OPTION == 2)
      append(sm->charBuffer, size);
      append(sm->charBuffer, tag);
      append(sm->charBuffer, (const char*)message, size);
#endif
#if (defined VM_EPOCHLOG_on || defined VM_SIZELOG_on)
      msg.str(""); // clear
      msg << "epochBuffer:" << __LINE__ << " non-blocking send write complete"<<
      ", current stream size=" <<
#if (EPOCH_BUFFER_OPTION == 0)
      sm->stream.pcount();
#elif (EPOCH_BUFFER_OPTION == 1)
      sm->stream.tellp();
#elif (EPOCH_BUFFER_OPTION == 2)
      sm->charBuffer.size();
#endif
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
#endif
    }else{
      int nelements = size/VM_MPI_SIZE_LIMIT;
      if (size%VM_MPI_SIZE_LIMIT) nelements++;
      (*ch)->nelements=nelements;
      (*ch)->type=1;          // MPI
      (*ch)->sendFlag=true;   // send request
      (*ch)->mpireq = new MPI_Request[nelements];
      // MPI-1 implementation
      void *messageC; // for MPI C interface convert (const void *) -> (void *)
      memcpy(&messageC, &message, sizeof(void *));
      // use mutex to serialize mpi comm calls if mpi thread support requires it
#ifndef ESMF_NO_PTHREADS
      if (mpi_mutex_flag) pthread_mutex_lock(pth_mutex);
#endif
#ifdef VM_MEMLOG_on
      VM::logMemInfo(std::string("VM::send():1.0"));
#endif
#ifdef VM_SIZELOG_on
      {
        std::stringstream msg;
        msg << "VMK::send():" << __LINE__ << ", size=" << size
          << " in " << nelements << " chunks";
        ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
      }
#endif
      if (size <= VM_MPI_SIZE_LIMIT)
        localrc = MPI_Isend(messageC, size, MPI_BYTE, lpid[dest], tag, mpi_c,
          (*ch)->mpireq);
      else{
        unsigned long long _size = size;
        char *messageCC = (char *)messageC;
        int i=0;
        while(_size > VM_MPI_SIZE_LIMIT){
          localrc = MPI_Isend(messageCC, VM_MPI_SIZE_LIMIT, MPI_BYTE,
            lpid[dest], tag+i, mpi_c, &((*ch)->mpireq[i]));
          i++;
          _size -= VM_MPI_SIZE_LIMIT;
          messageCC += VM_MPI_SIZE_LIMIT;
        }
        if (_size > 0)
          localrc = MPI_Isend(messageCC, _size, MPI_BYTE,
            lpid[dest], tag+i, mpi_c, &((*ch)->mpireq[i]));
      }
#ifdef VM_MEMLOG_on
      VM::logMemInfo(std::string("VM::send():2.0"));
#endif
#ifndef ESMF_NO_PTHREADS
      if (mpi_mutex_flag) pthread_mutex_unlock(pth_mutex);
#endif
    }
    break;
  case VM_COMM_TYPE_PTHREAD:
    // Pthread implementation
    // TODO: implement this using the SHARED_NONBLOCK_CHANNELS mechanism
    printf("non-blocking send not implemented for VM_COMM_TYPE_PTHREAD.\n");
    break;
  case VM_COMM_TYPE_SHMHACK:
    // Shared memory hack sync with spin-lock
    // TODO: implement this using the SHARED_NONBLOCK_CHANNELS mechanism
    printf("non-blocking send not implemented for VM_COMM_TYPE_SHMHACK.\n");
    break;
  case VM_COMM_TYPE_POSIXIPC:
    // Shared memory hack sync with spin-lock
    printf("non-blocking not implemented for VM_COMM_TYPE_POSIXIPC.\n");
#if 0
    //TODO: the following is totally incorrect code!!!!!!!!!
    (*ch)->type=5;          // POSIXIPC share memory channels
    pipcmp = sendChannel[dest].pipcmp;  // shared memory mp channel
    sendCount = pipcmp->sendCount;
    //TODO: enter mutex with "dest"
    pdest = (char *)pipcmp->ptr_dst_nb[sendCount];
    if (pdest != NULL){
      // recv() already set the pointer, send() came second -> copy data
      memcpy(pdest, message, size);      
      // reset ptr_dst_nb entry
      pipcmp->ptr_dst_nb[sendCount] = NULL;
      (*ch)->ptr=NULL;        // indicate that this transfer is complete
    }else{
      // send() came first, set pointer for recv() side
      pipcmp->ptr_src_nb[sendCount] = message;  // set the destination pointer
      (*ch)->ptr=&(pipcmp->ptr_src_nb[sendCount]);  // check by reference
    }
    //TODO: exit mutex with "dest"
    // increment sendCount
    ++sendCount;
    pipcmp->sendCount = sendCount%SHARED_NONBLOCK_CHANNELS;
#endif
    break;
  case VM_COMM_TYPE_MPIUNI:
    // Shared memory hack for mpiuni
    // This shared memory implementation is naturally non-blocking.
    // Limited to SHARED_NONBLOCK_CHANNELS per src/dst pair
#ifdef VM_SIZELOG_on
    {
      std::stringstream msg;
      msg << "VMK::send():" << __LINE__ << ", size=" << size
        << " using MPIUNI channel via memcpy() in one chunk";
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
    }
#endif
    (*ch)->type=-1; // indicate that this is a dummy commhandle
    shmp = sendChannel[dest].shmp;  // shared memory mp channel
    sendCount = shmp->sendCount;
    pdest = (char *)shmp->ptr_dst_nb[sendCount];
    if (pdest != NULL){
      // recv() already set the pointer, send() came second -> copy data
      memcpy(pdest, message, size);      
      // reset ptr_dst_nb entry
      shmp->ptr_dst_nb[sendCount] = NULL;
    }else{
      // send() came first, set pointer for recv() side
      shmp->ptr_src_nb[sendCount] = message;  // set the destination pointer
    }
    // increment sendCount
    ++sendCount;
    shmp->sendCount = sendCount%SHARED_NONBLOCK_CHANNELS;
    break;
  default:
    printf("unknown comm_type.\n");
    break;
  }
  return localrc;
}


int VMK::recv(void *message, unsigned long long int size, int source, int tag,
  status *status){
#undef  ESMC_METHOD
#define ESMC_METHOD "ESMCI::VMK::recv()"
  // p2p recv
#if (VERBOSITY > 9)
  printf("receiving from: %d, %d\n", source, lpid[source]);
#endif
  int localrc=0;
  pipc_mp *pipcmp;
  shared_mp *shmp;
  int scpsize, rcpsize;
  char *pdest;
  char *psrc;
  int i;
  char *mess;
  int comm_type;
  if (source == VM_ANY_SRC){
    if (!mpionly) return VMK_ERROR; // bail out
    comm_type = VM_COMM_TYPE_MPI1;
  }else{
    // use the predefined comm_type between source and destination (mypet)
    comm_type = recvChannel[source].comm_type;
  }
  // set comm_type in status
  if (status)
    status->comm_type = comm_type;
  // switch into the appropriate implementation
  switch(comm_type){
  case VM_COMM_TYPE_MPI1:
    // MPI-1 implementation
    // use mutex to serialize mpi comm calls if mpi thread support requires it
#ifndef ESMF_NO_PTHREADS
    if (mpi_mutex_flag) pthread_mutex_lock(pth_mutex);
#endif
    if (tag == -1) tag = getDefaultTag(source,mypet);
    else if (tag == VM_ANY_TAG) tag = MPI_ANY_TAG;
    int mpiSource;
    if (source == VM_ANY_SRC) mpiSource = MPI_ANY_SOURCE;
    else mpiSource = lpid[source];
    MPI_Status *mpi_s;
    if (status)
      mpi_s = &(status->mpi_s);
    else
      mpi_s = MPI_STATUS_IGNORE;
#ifdef VM_SIZELOG_on
      {
        std::stringstream msg;
        msg << "VMK::recv():" << __LINE__ << ", size=" << size;
        if (size <= VM_MPI_SIZE_LIMIT)
          msg << " in one chunk";
        else
          msg << " in multiple chunks";
        ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
      }
#endif
    if (size <= VM_MPI_SIZE_LIMIT)
      localrc = MPI_Recv(message, size, MPI_BYTE, mpiSource, tag, mpi_c, mpi_s);
    else{
      // must receive in multiple stages
      unsigned long long _size = size;
      char *messageCC = (char *)message;
      i=0;
      while(_size > VM_MPI_SIZE_LIMIT){
        localrc = MPI_Recv(messageCC, VM_MPI_SIZE_LIMIT, MPI_BYTE, mpiSource,
          tag+i, mpi_c, mpi_s);
        i++;
        _size -= VM_MPI_SIZE_LIMIT;
        messageCC += VM_MPI_SIZE_LIMIT;
      }
      if (_size > 0)
        localrc = MPI_Recv(messageCC, _size, MPI_BYTE, mpiSource, tag+i, mpi_c,
          mpi_s);
    }

#ifndef ESMF_NO_PTHREADS
    if (mpi_mutex_flag) pthread_mutex_unlock(pth_mutex);
#endif
    if (status){
      int cancelled;
      MPI_Test_cancelled(mpi_s, &cancelled);
      if (!cancelled){
        if (lpid[mpi_s->MPI_SOURCE] == mpi_s->MPI_SOURCE)
          status->srcPet = mpi_s->MPI_SOURCE;
        else{
          for (int k=0; k<npets; k++)
            if (lpid[k] == mpi_s->MPI_SOURCE)
              status->srcPet = mpi_s->MPI_SOURCE;
        }
        status->tag     = mpi_s->MPI_TAG;
        status->error   = mpi_s->MPI_ERROR;
      }
    }
    break;
  case VM_COMM_TYPE_PTHREAD:
    // Pthread implementation
    shmp = recvChannel[source].shmp;   // shared memory mp channel
    shmp->ptr_dst = message;               // set the destination pointer
    // synchronize with send()
#ifndef ESMF_NO_PTHREADS
    pthread_mutex_lock(&(shmp->mutex1));
    shmp->tcounter++;
    if (shmp->tcounter < 2){
      // wait for send()
      pthread_cond_wait(&(shmp->cond1), &(shmp->mutex1));
    }else{
      // reset counter and wake up send()
      shmp->tcounter = 0;
      pthread_cond_broadcast(&(shmp->cond1));
    }
    pthread_mutex_unlock(&(shmp->mutex1));
#endif
    // now ptr_src and ptr_dst are valid for this message
    scpsize = size/2;           // send takes the lower half
    rcpsize = size - scpsize;   // recv takes the upper half
    pdest = (char *)shmp->ptr_dst;
    psrc = (char *)shmp->ptr_src;
    // do actual memcpy
    memcpy(pdest + scpsize, psrc + scpsize, rcpsize);
    // synchronize with send()
#ifndef ESMF_NO_PTHREADS
    pthread_mutex_lock(&(shmp->mutex2));
    shmp->tcounter++;
    if (shmp->tcounter < 2){
      // wait for send()
      pthread_cond_wait(&(shmp->cond2), &(shmp->mutex2));
    }else{
      // reset counter and wake up send()
      shmp->tcounter = 0;
      pthread_cond_broadcast(&(shmp->cond2));
    }
    pthread_mutex_unlock(&(shmp->mutex2));
#endif
    break;
  case VM_COMM_TYPE_SHMHACK:
    // Shared memory hack sync with spin-lock
    shmp = recvChannel[source].shmp;   // shared memory mp channel
    if (size<=SHARED_BUFFER){
      // use buffer
      psrc = shmp->buffer;
      // wait until buffer is ready to be used
      sync_buffer_wait_fill(&shmp->shms, 0);
      // do actual memcpy
      memcpy(message, psrc, size);
      // set flag indicating that recv's memcpy() is done and buffer is empty
      sync_buffer_flag_empty(&shmp->shms, 0);
    }else{
      // don't use buffer
      shmp->ptr_dst = message;               // set the destination pointer
      // synchronize with send()
      sync_b_flip(&shmp->shms);
      // now ptr_src and ptr_dst are valid for this message
      scpsize = size/2;           // send takes the lower half
      rcpsize = size - scpsize;   // recv takes the upper half
      pdest = (char *)shmp->ptr_dst;
      psrc = (char *)shmp->ptr_src;
      // do actual memcpy
      memcpy(pdest + scpsize, psrc + scpsize, rcpsize);
      // synchronize with send()
      sync_b_flop(&shmp->shms);
    }
    break;
  case VM_COMM_TYPE_POSIXIPC:
    // Shared memory hack sync with spin-lock
    pipcmp = recvChannel[source].pipcmp;   // shared memory mp channel
    i=0;
    mess = (char *)message;
    while (size>PIPC_BUFFER){
      psrc = pipcmp->buffer[i];
      // wait until buffer is ready to be used
      sync_buffer_wait_fill(&pipcmp->shms, i);
      // do the actual memcpy
      memcpy(mess, psrc, PIPC_BUFFER);
      // set flag indicating that send's memcpy() is done and buffer is valid
      sync_buffer_flag_empty(&pipcmp->shms, i);
      size -= PIPC_BUFFER;
      mess += PIPC_BUFFER;
      i = i^1; // 0->1  or  1->0
    }
    // do the remaining parts of the message
    psrc = pipcmp->buffer[i];
    // wait until buffer is ready to be used
    sync_buffer_wait_fill(&pipcmp->shms, i);
    // do the actual memcpy
    memcpy(mess, psrc, size);
    // set flag indicating that send's memcpy() is done and buffer is valid
    sync_buffer_flag_empty(&pipcmp->shms, i);
    break;
  case VM_COMM_TYPE_MPIUNI:
    // Shared memory hack for mpiuni
    // TODO: this assumes that send will arrive first, otherwise this will hang
    shmp = recvChannel[source].shmp;   // shared memory mp channel
    if (size<=SHARED_BUFFER){
      // buffer is sufficient
      psrc = shmp->buffer;
      // wait until buffer is ready to be used
      sync_buffer_wait_fill(&shmp->shms, 0);
      // do actual memcpy
      memcpy(message, psrc, size);
      // set flag indicating that recv's memcpy() is done and buffer is empty
      sync_buffer_flag_empty(&shmp->shms, 0);
    }else{
      // buffer is insufficient
      ESMC_LogDefault.MsgFoundError(ESMC_RC_INTNRL_BAD,
        "buffer insufficient!", ESMC_CONTEXT, &localrc);
        return localrc;
    }
    break;
  default:
    printf("unknown comm_type.\n");
    break;
  }
  return localrc;
}


int VMK::recv(void *message, unsigned long long int size, int source,
  commhandle **ch, int tag){
#undef  ESMC_METHOD
#define ESMC_METHOD "ESMCI::VMK::recv()"
  // p2p recv non-blocking
//fprintf(stderr, "VMK::recv: ch=%p\n", *ch);
#if (VERBOSITY > 9)
  printf("receiving from: %d, %d\n", source, lpid[source]);
#endif
  int localrc=0;
  pipc_mp *pipcmp;
  shared_mp *shmp;
  int scpsize, rcpsize;
  char *psrc;
  int recvCount;
  // check if this needs a new entry in the request queue
  if (*ch==NULL){
    *ch = new commhandle;
    commqueueitem_link(*ch);
  }
  int comm_type;
  if (source == VM_ANY_SRC){
    if (!mpionly) return VMK_ERROR; // bail out
    comm_type = VM_COMM_TYPE_MPI1;
  }else{
    // use the predefined comm_type between source and destination (mypet)
    comm_type = recvChannel[source].comm_type;
  }
  // switch into the appropriate implementation
  switch(comm_type){
  case VM_COMM_TYPE_MPI1:
    if (tag == -1) tag = getDefaultTag(source,mypet);
    else if (tag == VM_ANY_TAG) tag = MPI_ANY_TAG;
    if (epoch==epochBuffer){
      recvBuffer *rm = &(recvMap[source]);
#if (defined VM_EPOCHLOG_on || defined VM_SIZELOG_on)
      std::stringstream msg;
      msg << "epochBuffer:" << __LINE__ << " non-blocking recv" << 
      " firstFlag=" << rm->firstFlag <<
      " size=" << size;
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
#endif
      if (rm->firstFlag){
        // deal with the first actions here
        rm->firstFlag = false;  // reset the flag
        // query the message size with probe
#if (defined VM_EPOCHLOG_on || defined VM_SIZELOG_on)
        std::stringstream msg;
#endif
#ifdef VM_EPOCHLOG_on
        msg << "epochBuffer:" << __LINE__ << " ready to probe:"
        << " src=" << getVas(lpid[source]);
        ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
        double t0; wtime(&t0);
#endif
        int defaultTag = getDefaultTag(source,mypet);
        MPI_Status mpistat;
        MPI_Probe(lpid[source], defaultTag, mpi_c, &mpistat);

        // - determine smallest customType to keep count < VM_MPI_SIZE_LIMIT
        int count;
        int i;
        for (i=(signed)customType.size()-1; i>=0; i--){
          MPI_Get_count(&mpistat, customType[i], &count);
          if (count > VM_MPI_SIZE_LIMIT/2) break;
        }
        unsigned long long int byteCount;
        if (i<0){
          // small message
          MPI_Get_count(&mpistat, MPI_BYTE, &count);
          byteCount = count;
        }else{
          // large message
          byteCount = 2;
          for (int j=0; j<i; j++)
            byteCount *= 2;
          byteCount *= count;
        }
        // prepare the receive buffer
        rm->streamBuffer.resize(byteCount);
        rm->buffer = (void *)rm->streamBuffer.data();

#ifdef VM_EPOCHLOG_on
        double t1; wtime(&t1);
        msg.str(""); // clear
        msg << "epochBuffer:" << __LINE__ << " time in blocking probe: "
          << t1 - t0 << " seconds";
        ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
        msg.str(""); // clear
        msg << "epochBuffer:" << __LINE__ << " incoming message of size="
          << rm->streamBuffer.size();
        ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
#endif

        // send the acknowledge message to sender for throttle
        int ackDummy=1;
        MPI_Send(&ackDummy, sizeof(int), MPI_BYTE, lpid[source], defaultTag,
          mpi_c);

        // post blocking recv of the entire epoch buffer
#if (defined VM_EPOCHLOG_on || defined VM_SIZELOG_on)
        msg.str(""); // clear
        msg << "epochBuffer:" << __LINE__ << " ready to post blocking recv:"
        << " src=" << getVas(lpid[source])
        << " size=" << rm->streamBuffer.size()
        << " streamBuffer=" << (void *)rm->streamBuffer.data();
        ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
        double t2; wtime(&t2);
#endif
        if (i<0){
          // small message
          MPI_Recv(rm->buffer, count, MPI_BYTE,
            lpid[source], defaultTag, mpi_c, MPI_STATUS_IGNORE);
        }else{
          // large message
          MPI_Recv(rm->buffer, count, customType[i],
            lpid[source], defaultTag, mpi_c, MPI_STATUS_IGNORE);
        }
#ifdef VM_EPOCHLOG_on
        double t3; wtime(&t3);
        msg.str(""); // clear
        msg << "epochBuffer:" << __LINE__ << " time in blocking recv: "
          << t3 - t2 << " seconds";
        ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
#endif
      }
      // now service the specific receive call with chunk of data from buffer
      unsigned long long int *ipl = (unsigned long long int *)rm->buffer;
      unsigned long long int chunkSize = *ipl++;
      int *ip = (int *)ipl;
      int chunkTag = *ip++;
      char *cp = (char *)ip;

#ifdef VM_EPOCHLOG_on
      msg.str(""); // clear
      msg << "epochBuffer:" << __LINE__ << " processing recv chunk:"
      << " buffer=" << rm->buffer
      << " chunkSize=" << chunkSize << " chunkTag=" << chunkTag;
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
#endif      
      if (chunkSize!=size){
        ESMC_LogDefault.MsgFoundError(ESMC_RC_INTNRL_BAD,
          "chunk size incorrect", ESMC_CONTEXT, &localrc);
        return localrc;
      }
      if (chunkTag!=tag){
        ESMC_LogDefault.MsgFoundError(ESMC_RC_INTNRL_BAD,
          "chunk tag incorrect", ESMC_CONTEXT, &localrc);
        return localrc;
      }
      memcpy(message, cp, size);
      rm->buffer = (void *)(cp+chunkSize);  // next chunk in buffer
    }else{
      int nelements = size/VM_MPI_SIZE_LIMIT;
      if (size%VM_MPI_SIZE_LIMIT) nelements++;
      (*ch)->nelements=nelements;
      (*ch)->type=1;          // MPI
      (*ch)->sendFlag=false;  // not a send request
      (*ch)->mpireq = new MPI_Request[nelements];
      // MPI-1 implementation
      // use mutex to serialize mpi comm calls if mpi thread support requires it
#ifndef ESMF_NO_PTHREADS
      if (mpi_mutex_flag) pthread_mutex_lock(pth_mutex);
#endif
      int mpiSource;
      if (source == VM_ANY_SRC) mpiSource = MPI_ANY_SOURCE;
      else mpiSource = lpid[source];
#ifdef VM_SIZELOG_on
      {
        std::stringstream msg;
        msg << "VMK::recv():" << __LINE__ << ", size=" << size
          << " in " << nelements << " chunks";
        ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
      }
#endif
      if (size <= VM_MPI_SIZE_LIMIT)
        localrc = MPI_Irecv(message, size, MPI_BYTE, mpiSource, tag, mpi_c,
          (*ch)->mpireq);
      else{
        unsigned long long _size = size;
        char *messageCC = (char *)message;
        int i=0;
        while(_size > VM_MPI_SIZE_LIMIT){
          localrc = MPI_Irecv(messageCC, VM_MPI_SIZE_LIMIT, MPI_BYTE, mpiSource,
            tag+i, mpi_c, &((*ch)->mpireq[i]));
          i++;
          _size -= VM_MPI_SIZE_LIMIT;
          messageCC += VM_MPI_SIZE_LIMIT;
        }
        if (_size > 0)
          localrc = MPI_Irecv(messageCC, _size, MPI_BYTE, mpiSource,
            tag+i, mpi_c, &((*ch)->mpireq[i]));
      }
#ifndef ESMF_NO_PTHREADS
      if (mpi_mutex_flag) pthread_mutex_unlock(pth_mutex);
#endif
    }
    break;
  case VM_COMM_TYPE_PTHREAD:
    // Pthread implementation
    // TODO: implement this using the SHARED_NONBLOCK_CHANNELS mechanism
    printf("non-blocking recv not implemented for VM_COMM_TYPE_PTHREAD.\n");
    break;
  case VM_COMM_TYPE_SHMHACK:
    // Shared memory hack sync with spin-lock
    // TODO: implement this using the SHARED_NONBLOCK_CHANNELS mechanism
    printf("non-blocking recv not implemented for VM_COMM_TYPE_SHMHACK.\n");
    break;
  case VM_COMM_TYPE_POSIXIPC:
    // Shared memory hack sync with spin-lock
    printf("non-blocking recv not implemented for VM_COMM_TYPE_POSIXIPC.\n");
#if 0    
    //TODO: the following is totally incorrect code!!!!!!!!!
    (*ch)->type=5;          // POSIXIPC share memory channels
    pipcmp = recvChannel[source].pipcmp;  // shared memory mp channel
    recvCount = pipcmp->recvCount;
    //TODO: enter mutex with "source"
    psrc = (char *)pipcmp->ptr_src_nb[recvCount];
    if (psrc != NULL){
      // send() already set the pointer, recv() came second -> copy data
      memcpy(message, psrc, size);
      // reset ptr_src_nb entry
      pipcmp->ptr_src_nb[recvCount] = NULL;
      (*ch)->ptr=NULL;         // indicate that this transfer is complete
    }else{
      // recv() came first, set pointer for send() side
      pipcmp->ptr_dst_nb[recvCount] = message;  // set the destination pointer
      (*ch)->ptr=&(pipcmp->ptr_src_nb[recvCount]);  // check by reference
    }
    //TODO: exit mutex with "source"
    // increment recvCount
    ++recvCount;
    pipcmp->recvCount = recvCount%SHARED_NONBLOCK_CHANNELS;
#endif
    break;
  case VM_COMM_TYPE_MPIUNI:
    // Shared memory hack for mpiuni
    // This shared memory implementation is naturally non-blocking.
    // Limited to SHARED_NONBLOCK_CHANNELS per src/dst pair
#ifdef VM_SIZELOG_on
    {
      std::stringstream msg;
      msg << "VMK::recv():" << __LINE__ << ", size=" << size
        << " using MPIUNI channel via memcpy() in one chunk";
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
    }
#endif
    (*ch)->type=-1; // indicate that this is a dummy commhandle
    shmp = recvChannel[source].shmp;  // shared memory mp channel
    recvCount = shmp->recvCount;
    psrc = (char *)shmp->ptr_src_nb[recvCount];
    if (psrc != NULL){
      // send() already set the pointer, recv() came second -> copy data
      memcpy(message, psrc, size);
      // reset ptr_src_nb entry
      shmp->ptr_src_nb[recvCount] = NULL;
    }else{
      // recv() came first, set pointer for send() side
      shmp->ptr_dst_nb[recvCount] = message;  // set the destination pointer
    }
    // increment recvCount
    ++recvCount;
    shmp->recvCount = recvCount%SHARED_NONBLOCK_CHANNELS;
    break;
  default:
    printf("unknown comm_type.\n");
    break;
  }
  return localrc;
}


int VMK::vassend(void *message, int size, int destVAS, commhandle **ch,
  int tag){
  // non-blocking send where the destination is a VAS, _not_ a PET
  // todo: currently this is just a stub that uses the PET-based 
  //       non-blocking send. Hence this will not work for the ESMF-threading
  //       case for which it is actually thought for!
  // for this stub implementation figure out first PET to run in the destVAS
  int localrc=0;
  int dest;
  for (dest=0; dest<npets; dest++)
    if (pid[dest] == destVAS) break;
  localrc = send(message, size, dest, ch, tag);
  return localrc;
}


int VMK::vasrecv(void *message, int size, int srcVAS, commhandle **ch,
  int tag){
  // non-blocking recv where the source is a VAS, _not_ a PET
  // todo: currently this is just a stub that uses the PET-based 
  //       non-blocking recv. Hence this will not work for the ESMF-threading
  //       case for which it is actually thought for!
  // for this stub implementation figure out first PET to run in the sourceVAS
  int localrc=0;
  int src;
  for (src=0; src<npets; src++)
    if (pid[src] == srcVAS) break;
  localrc = recv(message, size, src, ch, tag);
  return localrc;
}

  
int VMK::barrier(){
  // collective barrier over all PETs
  int localrc=0;
  if (mpionly){
    localrc = MPI_Barrier(mpi_c);
    return localrc;
  }
  int myp = pid[mypet];
  int myt = tid[mypet];
  if (myt==0){
    // mypet is the zero thread for this PID
    for (int i=0; i<npets; i++)
      if (i!=mypet && pid[i]==myp){
        // pet "i" is another thread under same PID
        shared_mp *shmp = sendChannel[i].shmp;
        sync_a_flip(&shmp->shms);
      }
    // now all threads are "flip"-synced under their master thread
    // master thread will use MPI-1 to sync with all the other masters
    MPI_Barrier(mpi_c);
    // now master thread is synced against all other masters
    for (int i=0; i<npets; i++)
      if (i!=mypet && pid[i]==myp){
        // pet "i" is another thread under same PID
        shared_mp *shmp = sendChannel[i].shmp;
        sync_a_flop(&shmp->shms);
      }
  }else{
    // mypet is not the master thread for this PID -> find master
    int i;
    for (i=0; i<npets; i++)
      if (pid[i]==myp && tid[i]==0) break;
    // now PET "i" is the master thread for this PID
    shared_mp *shmp = recvChannel[i].shmp;
    sync_b_flip(&shmp->shms);
    // now all threads are "flip"-synced under their master thread
    // master will sync against all other masters using MPI-1 and then do flop
    sync_b_flop(&shmp->shms);
  }
  return localrc;
}


int VMK::sendrecv(void *sendData, int sendSize, int dst, void *recvData,
  int recvSize, int src, int dstTag, int srcTag){
  // p2p sendrecv
  int localrc=0;
  if (mpionly){
    MPI_Status mpi_s;
    if (dstTag == -1){
      dstTag = 1000*mypet+dst;  // default tag to simplify debugging
      // make sure to stay below max tag
      int maxTag = getMaxTag();
      if (maxTag > 0)
        dstTag = dstTag%maxTag;
      else
        dstTag = 0;
    }else if (dstTag == VM_ANY_TAG)
      dstTag = MPI_ANY_TAG;
    if (srcTag == -1){
      srcTag = 1000*src+mypet;  // default tag to simplify debugging
      // make sure to stay below max tag
      int maxTag = getMaxTag();
      if (maxTag > 0)
        srcTag = srcTag%maxTag;
      else
        srcTag = 0;
    }else if (srcTag == VM_ANY_TAG)
      srcTag = MPI_ANY_TAG;
    localrc = MPI_Sendrecv(sendData, sendSize, MPI_BYTE, dst, dstTag, 
      recvData, recvSize, MPI_BYTE, src, srcTag, mpi_c, &mpi_s);
  }else{
    // A unique order of the send and receive is given by the PET index.
    // This very simplistic implementation establishes a unique order by
    // first transferring data to the smallest receiver PET and then to the
    // other one. A sendrecv has two receiver PETs, one is the local PET and
    // the other is rcv.
    if (mypet<dst){
      // mypet is the first receiver
      localrc = recv(recvData, recvSize, src);
      if (localrc) return localrc;
      localrc = send(sendData, sendSize, dst);
      if (localrc) return localrc;
    }else{
      // dst is first receiver
      localrc = send(sendData, sendSize, dst);
      if (localrc) return localrc;
      localrc = recv(recvData, recvSize, src);
      if (localrc) return localrc;
    }
  }
  return localrc;
}
  
int VMK::sendrecv(void *sendData, int sendSize, int dst, void *recvData,
  int recvSize, int src, commhandle **ch){
  // check if this needs a new entry in the request queue
//fprintf(stderr, "VMK::sendrecv: ch=%p\n", *ch);
  int localrc=0;
  if (*ch==NULL){
    *ch = new commhandle;
    commqueueitem_link(*ch);
  }
  // p2p sendrecv non-blocking
  (*ch)->nelements = 2; // 2 requests for send/recv
  (*ch)->type=0; // subhandles
  (*ch)->handles = new commhandle*[(*ch)->nelements];
  for (int i=0; i<(*ch)->nelements; i++)
    (*ch)->handles[i] = new commhandle;
  // MPI does not offer a non-blocking sendrecv operation, hence there is no
  // point in checking if the mpionly flag is set in this VM, in either case
  // an explicit implementation based on send and recv must be used:
  // A unique order of the send and receive is given by the PET index.
  // This very simplistic implementation establishes a unique order by
  // first transferring data to the smallest receiver PET and then to the
  // other one. A sendrecv has two receiver PETs, one is the local PET and
  // the other is rcv.
  if (mypet<dst){
    // mypet is the first receiver
    localrc = recv(recvData, recvSize, src, &((*ch)->handles[0]));
    if (localrc) return localrc;
    localrc = send(sendData, sendSize, dst, &((*ch)->handles[1]));
    if (localrc) return localrc;
  }else{
    // dst is first receiver
    localrc = send(sendData, sendSize, dst, &((*ch)->handles[0]));
    if (localrc) return localrc;
    localrc = recv(recvData, recvSize, src, &((*ch)->handles[1]));
    if (localrc) return localrc;
  }
  return localrc;
}
  
int VMK::threadbarrier(){
  int localrc=0;
  if (!mpionly && threadsflag){
    // collective barrier over all PETs in thread group with mypet
    int myp = pid[mypet];
    int myt = tid[mypet];
    if (myt==0){
      // mypet is the zero thread for this PID
      // todo: optimize by storing PETs of thread group within VM as list
      for (int i=0; i<npets; i++)
        if (i!=mypet && pid[i]==myp){
          // pet "i" is another thread under same PID
          shared_mp *shmp = sendChannel[i].shmp;
          sync_a_flip(&shmp->shms);
        }
      for (int i=0; i<npets; i++)
        if (i!=mypet && pid[i]==myp){
          // pet "i" is another thread under same PID
          shared_mp *shmp = sendChannel[i].shmp;
          sync_a_flop(&shmp->shms);
        }
    }else{
      // mypet is not the master thread for this PID -> find master
      int i;
      // todo: optimize by storing root PET of thread group within VM
      for (i=0; i<npets; i++)
        if (pid[i]==myp && tid[i]==0) break;
      // now PET "i" is the master thread for this PID
      shared_mp *shmp = recvChannel[i].shmp;
      sync_b_flip(&shmp->shms);
      // now all threads are "flip"-synced under their master thread
      sync_b_flop(&shmp->shms);
    }
  }
  return localrc;
}


int VMK::reduce(void *in, void *out, int len, vmType type, vmOp op, int root){
  int localrc=0;
  // sanity check root
  if (root<0 || root>=npets){
    ESMC_LogDefault.MsgFoundError(ESMC_RC_ARG_OUTOFRANGE,
      "'root' out of range", ESMC_CONTEXT, &localrc);
    return localrc;
  }
  if (mpionly){
    // Find corresponding MPI operation
    MPI_Op mpiop;
    switch (op){
    case vmSUM:
      mpiop = MPI_SUM;
      break;
    case vmMIN:
      mpiop = MPI_MIN;
      break;
    case vmMAX:
      mpiop = MPI_MAX;
      break;
    }
    // Find corresponding MPI data type
    MPI_Datatype mpitype;
    switch (type){
    case vmI4:
      mpitype = MPI_INT;
      break;
    case vmI8:
      mpitype = MPI_LONG_LONG_INT;
      break;
    case vmR4:
      mpitype = MPI_FLOAT;
      break;
    case vmR8:
      mpitype = MPI_DOUBLE;
      break;
    case vmBYTE:
    case vmL4:
      localrc = -1;   // error
      return localrc; // bail out
    }
    localrc = MPI_Reduce(in, out, len, mpitype, mpiop, root, mpi_c);
  }else{
    // This is a very simplistic, probably very bad peformance implementation.
    int templen = len;
    switch (type){
    case vmI4:
      templen *= 4;   // 4 bytes
      break;
    case vmI8:
      templen *= 8;   // 8 bytes
      break;
    case vmR4:
      templen *= 4;   // 4 bytes
      break;
    case vmR8:
      templen *= 8;   // 8 bytes
      break;
    case vmBYTE:
    case vmL4:
      localrc = -1;   // error
      return localrc; // bail out
    }
    char *temparray;
    if (mypet==root)
      temparray = new char[templen*npets]; // allocate temp data array
    // gather all data onto root PET
    localrc = gather(in, temparray, templen, root);
    // root does the entire reduction on its local temparray data
    if (mypet==root){
      switch (op){
      case vmSUM:
        switch (type){
        case vmI4:
          {
            int *tempdata = (int *)temparray;
            int *outdata = (int *)out;
            for (int i=0; i<len; i++){
              *outdata = 0;
              for (int j=0; j<npets; j++){
                *outdata += tempdata[j*len];
              }
              ++tempdata;
              ++outdata;
            }
          }
          break;
        case vmI8:
          {
            long long int *tempdata = (long long int *)temparray;
            long long int *outdata = (long long int *)out;
            for (int i=0; i<len; i++){
              *outdata = 0;
              for (int j=0; j<npets; j++){
                *outdata += tempdata[j*len];
              }
              ++tempdata;
              ++outdata;
            }
          }
          break;
        case vmR4:
          {
            float *tempdata = (float *)temparray;
            float *outdata = (float *)out;
            for (int i=0; i<len; i++){
              *outdata = 0;
              for (int j=0; j<npets; j++){
                *outdata += tempdata[j*len];
              }
              ++tempdata;
              ++outdata;
            }
          }
          break;
        case vmR8:
          {
            double *tempdata = (double *)temparray;
            double *outdata = (double *)out;
            for (int i=0; i<len; i++){
              *outdata = 0;
              for (int j=0; j<npets; j++){
                *outdata += tempdata[j*len];
                }
              ++tempdata;
              ++outdata;
            }
          }
          break;
        case vmBYTE:
        case vmL4:
          localrc = -1;   // error
          return localrc; // bail out
        }
        break;
      case vmMIN:
        switch (type){
        case vmI4:
          {
            int *tempdata = (int *)temparray;
            int *outdata = (int *)out;
            for (int i=0; i<len; i++){
              *outdata = tempdata[0];
              for (int j=1; j<npets; j++){
                if (tempdata[j*len] < *outdata)
                  *outdata = tempdata[j*len];
              }
              ++tempdata;
              ++outdata;
            }
          }
          break;
        case vmI8:
          {
            long long int *tempdata = (long long int *)temparray;
            long long int *outdata = (long long int *)out;
            for (int i=0; i<len; i++){
              *outdata = tempdata[0];
              for (int j=1; j<npets; j++){
                if (tempdata[j*len] < *outdata)
                  *outdata = tempdata[j*len];
              }
              ++tempdata;
              ++outdata;
            }
          }
          break;
        case vmR4:
          {
            float *tempdata = (float *)temparray;
            float *outdata = (float *)out;
            for (int i=0; i<len; i++){
              *outdata = tempdata[0];
              for (int j=1; j<npets; j++){
                if (tempdata[j*len] < *outdata)
                  *outdata = tempdata[j*len];
              }
              ++tempdata;
              ++outdata;
            }
          }
          break;
        case vmR8:
          {
            double *tempdata = (double *)temparray;
            double *outdata = (double *)out;
            for (int i=0; i<len; i++){
              *outdata = tempdata[0];
              for (int j=1; j<npets; j++){
                if (tempdata[j*len] < *outdata)
                  *outdata = tempdata[j*len];
              }
              ++tempdata;
              ++outdata;
            }
          }
          break;
        case vmBYTE:
        case vmL4:
          localrc = -1;   // error
          return localrc; // bail out
        }
        break;
      case vmMAX:
        switch (type){
        case vmI4:
          {
            int *tempdata = (int *)temparray;
            int *outdata = (int *)out;
            for (int i=0; i<len; i++){
              *outdata = tempdata[0];
              for (int j=1; j<npets; j++){
                if (tempdata[j*len] > *outdata)
                  *outdata = tempdata[j*len];
              }
              ++tempdata;
              ++outdata;
            }
          }
          break;
        case vmI8:
          {
            long long int *tempdata = (long long int *)temparray;
            long long int *outdata = (long long int *)out;
            for (int i=0; i<len; i++){
              *outdata = tempdata[0];
              for (int j=1; j<npets; j++){
                if (tempdata[j*len] > *outdata)
                  *outdata = tempdata[j*len];
              }
              ++tempdata;
              ++outdata;
            }
          }
          break;
        case vmR4:
          {
            float *tempdata = (float *)temparray;
            float *outdata = (float *)out;
            for (int i=0; i<len; i++){
              *outdata = tempdata[0];
              for (int j=1; j<npets; j++){
                if (tempdata[j*len] > *outdata)
                  *outdata = tempdata[j*len];
              }
              ++tempdata;
              ++outdata;
            }
          }
          break;
        case vmR8:
          {
            double *tempdata = (double *)temparray;
            double *outdata = (double *)out;
            for (int i=0; i<len; i++){
              *outdata = tempdata[0];
              for (int j=1; j<npets; j++){
                if (tempdata[j*len] > *outdata)
                  *outdata = tempdata[j*len];
              }
              ++tempdata;
              ++outdata;
            }
          }
          break;
        case vmBYTE:
        case vmL4:
          localrc = -1;   // error
          return localrc; // bail out
        }
        break;
      }
      delete [] temparray;
    }
  }
  return localrc;
}


int VMK::allreduce(void *in, void *out, int len, vmType type, vmOp op){
  int localrc=0;
  if (mpionly){
    // Find corresponding MPI operation
    MPI_Op mpiop;
    switch (op){
    case vmSUM:
      mpiop = MPI_SUM;
      break;
    case vmMIN:
      mpiop = MPI_MIN;
      break;
    case vmMAX:
      mpiop = MPI_MAX;
      break;
    }
    // Find corresponding MPI data type
    MPI_Datatype mpitype;
    switch (type){
    case vmI4:
      mpitype = MPI_INT;
      break;
    case vmI8:
      mpitype = MPI_LONG_LONG_INT;
      break;
    case vmR4:
      mpitype = MPI_FLOAT;
      break;
    case vmR8:
      mpitype = MPI_DOUBLE;
      break;
    case vmBYTE:
    case vmL4:
      localrc = -1;   // error
      return localrc; // bail out
    }
    localrc = MPI_Allreduce(in, out, len, mpitype, mpiop, mpi_c);
  }else{
    // This is a very simplistic, probably very bad peformance implementation.
    int templen = len;
    switch (type){
    case vmI4:
      templen *= 4;   // 4 bytes
      break;
    case vmI8:
      templen *= 8;   // 8 bytes
      break;
    case vmR4:
      templen *= 4;   // 4 bytes
      break;
    case vmR8:
      templen *= 8;   // 8 bytes
      break;
    case vmBYTE:
    case vmL4:
      localrc = -1;   // error
      return localrc; // bail out
    }
    char *temparray = new char[templen*npets]; // allocate temp data array
    // gather all data onto each PET
    for (int i=0; i<npets; i++){
      localrc = gather(in, temparray, templen, i);
      if (localrc) return localrc;
    }
    // each PET does its own reduction on its local temparray data
    switch (op){
    case vmSUM:
      switch (type){
      case vmI4:
        {
          int *tempdata = (int *)temparray;
          int *outdata = (int *)out;
          for (int i=0; i<len; i++){
            *outdata = 0;
            for (int j=0; j<npets; j++){
              *outdata += tempdata[j*len];
            }
            ++tempdata;
            ++outdata;
          }
        }
        break;
      case vmI8:
        {
          long long int *tempdata = (long long int *)temparray;
          long long int *outdata = (long long int *)out;
          for (int i=0; i<len; i++){
            *outdata = 0;
            for (int j=0; j<npets; j++){
              *outdata += tempdata[j*len];
            }
            ++tempdata;
            ++outdata;
          }
        }
        break;
      case vmR4:
        {
          float *tempdata = (float *)temparray;
          float *outdata = (float *)out;
          for (int i=0; i<len; i++){
            *outdata = 0;
            for (int j=0; j<npets; j++){
              *outdata += tempdata[j*len];
            }
            ++tempdata;
            ++outdata;
          }
        }
        break;
      case vmR8:
        {
          double *tempdata = (double *)temparray;
          double *outdata = (double *)out;
          for (int i=0; i<len; i++){
            *outdata = 0;
            for (int j=0; j<npets; j++){
              *outdata += tempdata[j*len];
            }
            ++tempdata;
            ++outdata;
          }
        }
        break;
      case vmBYTE:
      case vmL4:
        localrc = -1;   // error
        return localrc; // bail out
      }
      break;
    case vmMIN:
      switch (type){
      case vmI4:
        {
          int *tempdata = (int *)temparray;
          int *outdata = (int *)out;
          for (int i=0; i<len; i++){
            *outdata = tempdata[0];
            for (int j=1; j<npets; j++){
              if (tempdata[j*len] < *outdata)
                *outdata = tempdata[j*len];
            }
            ++tempdata;
            ++outdata;
          }
        }
        break;
      case vmI8:
        {
          long long int *tempdata = (long long int *)temparray;
          long long int *outdata = (long long int *)out;
          for (int i=0; i<len; i++){
            *outdata = tempdata[0];
            for (int j=1; j<npets; j++){
              if (tempdata[j*len] < *outdata)
                *outdata = tempdata[j*len];
            }
            ++tempdata;
            ++outdata;
          }
        }
        break;
      case vmR4:
        {
          float *tempdata = (float *)temparray;
          float *outdata = (float *)out;
          for (int i=0; i<len; i++){
            *outdata = tempdata[0];
            for (int j=1; j<npets; j++){
              if (tempdata[j*len] < *outdata)
                *outdata = tempdata[j*len];
            }
            ++tempdata;
            ++outdata;
          }
        }
        break;
      case vmR8:
        {
          double *tempdata = (double *)temparray;
          double *outdata = (double *)out;
          for (int i=0; i<len; i++){
            *outdata = tempdata[0];
            for (int j=1; j<npets; j++){
              if (tempdata[j*len] < *outdata)
                *outdata = tempdata[j*len];
            }
            ++tempdata;
            ++outdata;
          }
        }
        break;
      case vmBYTE:
      case vmL4:
        localrc = -1;   // error
        return localrc; // bail out
      }
      break;
    case vmMAX:
      switch (type){
      case vmI4:
        {
          int *tempdata = (int *)temparray;
          int *outdata = (int *)out;
          for (int i=0; i<len; i++){
            *outdata = tempdata[0];
            for (int j=1; j<npets; j++){
              if (tempdata[j*len] > *outdata)
                *outdata = tempdata[j*len];
            }
            ++tempdata;
            ++outdata;
          }
        }
        break;
      case vmI8:
        {
          long long int *tempdata = (long long int *)temparray;
          long long int *outdata = (long long int *)out;
          for (int i=0; i<len; i++){
            *outdata = tempdata[0];
            for (int j=1; j<npets; j++){
              if (tempdata[j*len] > *outdata)
                *outdata = tempdata[j*len];
            }
            ++tempdata;
            ++outdata;
          }
        }
        break;
      case vmR4:
        {
          float *tempdata = (float *)temparray;
          float *outdata = (float *)out;
          for (int i=0; i<len; i++){
            *outdata = tempdata[0];
            for (int j=1; j<npets; j++){
              if (tempdata[j*len] > *outdata)
                *outdata = tempdata[j*len];
            }
            ++tempdata;
            ++outdata;
          }
        }
        break;
      case vmR8:
        {
          double *tempdata = (double *)temparray;
          double *outdata = (double *)out;
          for (int i=0; i<len; i++){
            *outdata = tempdata[0];
            for (int j=1; j<npets; j++){
              if (tempdata[j*len] > *outdata)
                *outdata = tempdata[j*len];
            }
            ++tempdata;
            ++outdata;
          }
        }
        break;
      case vmBYTE:
      case vmL4:
        localrc = -1;   // error
        return localrc; // bail out
      }
      break;
    }
    delete [] temparray;
  }
  return localrc;
}


int VMK::allfullreduce(void *in, void *out, int len, vmType type, vmOp op){
  int localrc=0;
  void *localresult;
  int local_i4;
  long long int local_i8;
  float local_r4;
  double local_r8;
  // first reduce the vector on each PET
  switch (op){
  case vmSUM:
    switch (type){
    case vmI4:
      {
        localresult = (void *)&local_i4;
        local_i4 = 0;
        int *tempdata = (int *)in;        // type cast for pointer arithmetic
        for (int j=0; j<len; j++)
          local_i4 += tempdata[j];
      }
      break;
    case vmI8:
      {
        localresult = (void *)&local_i8;
        local_i8 = 0;
        // type cast for pointer arithmetic
        long long int *tempdata = (long long int *)in;
        for (int j=0; j<len; j++)
          local_i8 += tempdata[j];
      }
      break;
    case vmR4:
      {
        localresult = (void *)&local_r4;  // type cast for pointer arithmetic
        local_r4 = 0.;
        float *tempdata = (float *)in;
        for (int j=0; j<len; j++)
          local_r4 += tempdata[j];
      }
      break;
    case vmR8:
      {
        localresult = (void *)&local_r8;  // type cast for pointer arithmetic
        local_r8 = 0.;
        double *tempdata = (double *)in;
        for (int j=0; j<len; j++)
          local_r8 += tempdata[j];
      }
      break;
    case vmBYTE:
    case vmL4:
      localrc = -1;   // error
      return localrc; // bail out
    }
    break;
  case vmMIN:
    switch (type){
    case vmI4:
      {
        localresult = (void *)&local_i4;
        int *tempdata = (int *)in;        // type cast for pointer arithmetic
        local_i4 = tempdata[0];
        for (int j=1; j<len; j++)
          if (tempdata[j] < local_i4) local_i4 = tempdata[j];
      }
      break;
    case vmI8:
      {
        localresult = (void *)&local_i8;
        // type cast for pointer arithmetic
        long long int *tempdata = (long long int *)in;
        local_i8 = tempdata[0];
        for (int j=1; j<len; j++)
          if (tempdata[j] < local_i8) local_i8 = tempdata[j];
      }
      break;
    case vmR4:
      {
        localresult = (void *)&local_r4;  // type cast for pointer arithmetic
        float *tempdata = (float *)in;
        local_r4 = tempdata[0];
        for (int j=1; j<len; j++)
          if (tempdata[j] < local_r4) local_r4 = tempdata[j];
      }
      break;
    case vmR8:
      {
        localresult = (void *)&local_r8;  // type cast for pointer arithmetic
        double *tempdata = (double *)in;
        local_r8 = tempdata[0];
        for (int j=1; j<len; j++)
          if (tempdata[j] < local_r8) local_r8 = tempdata[j];
      }
      break;
    case vmBYTE:
    case vmL4:
      localrc = -1;   // error
      return localrc; // bail out
    }
    break;
  case vmMAX:
    switch (type){
    case vmI4:
      {
        localresult = (void *)&local_i4;
        int *tempdata = (int *)in;        // type cast for pointer arithmetic
        local_i4 = tempdata[0];
        for (int j=1; j<len; j++)
          if (tempdata[j] > local_i4) local_i4 = tempdata[j];
      }
      break;
    case vmI8:
      {
        localresult = (void *)&local_i8;
        // type cast for pointer arithmetic
        long long int *tempdata = (long long int *)in;
        local_i8 = tempdata[0];
        for (int j=1; j<len; j++)
          if (tempdata[j] > local_i8) local_i8 = tempdata[j];
      }
      break;
    case vmR4:
      {
        localresult = (void *)&local_r4;  // type cast for pointer arithmetic
        float *tempdata = (float *)in;
        local_r4 = tempdata[0];
        for (int j=1; j<len; j++)
          if (tempdata[j] > local_r4) local_r4 = tempdata[j];
      }
      break;
    case vmR8:
      {
        localresult = (void *)&local_r8;  // type cast for pointer arithmetic
        double *tempdata = (double *)in;
        local_r8 = tempdata[0];
        for (int j=1; j<len; j++)
          if (tempdata[j] > local_r8) local_r8 = tempdata[j];
      }
      break;
    case vmBYTE:
    case vmL4:
      localrc = -1;   // error
      return localrc; // bail out
    }
    break;
  }
  localrc = allreduce(localresult, out, 1, type, op);
  return localrc;
}


int VMK::reduce_scatter(void *in, void *out, int *outCounts,
  vmType type, vmOp op){
  int localrc=0;
  if (mpionly){
    // Find corresponding MPI operation
    MPI_Op mpiop;
    switch (op){
    case vmSUM:
      mpiop = MPI_SUM;
      break;
    case vmMIN:
      mpiop = MPI_MIN;
      break;
    case vmMAX:
      mpiop = MPI_MAX;
      break;
    }
    // Find corresponding MPI data type
    MPI_Datatype mpitype;
    switch (type){
    case vmI4:
      mpitype = MPI_INT;
      break;
    case vmI8:
      mpitype = MPI_LONG_LONG_INT;
      break;
    case vmR4:
      mpitype = MPI_FLOAT;
      break;
    case vmR8:
      mpitype = MPI_DOUBLE;
      break;
    case vmBYTE:
    case vmL4:
      localrc = -1;   // error
      return localrc; // bail out
    }
    localrc = MPI_Reduce_scatter(in, out, outCounts, mpitype, mpiop, mpi_c);
  }else{
    // TODO: not yet implemented
    localrc = VMK_ERROR;
  }
  return localrc;
}

    
int VMK::scatter(void *in, void *out, int len, int root){
  int localrc=0;
  // sanity check root
  if (root<0 || root>=npets){
    ESMC_LogDefault.MsgFoundError(ESMC_RC_ARG_OUTOFRANGE,
      "'root' out of range", ESMC_CONTEXT, &localrc);
    return localrc;
  }
  if (mpionly){
    localrc = MPI_Scatter(in, len, MPI_BYTE, out, len, MPI_BYTE, root, mpi_c);
  }else{
    // This is a very simplistic, probably very bad peformance implementation.
    if (mypet==root){
      // I am root -> send chunks to all other PETs
      char *rootin = (char *)in;
      for (int i=0; i<root; i++){
        localrc = send(rootin, len, i);
        if (localrc) return localrc;
        rootin += len;
      }
      // memcpy root's chunk
      memcpy(out, rootin, len);
      rootin += len;
      // keep sending chunks
      for (int i=root+1; i<npets; i++){
        localrc = send(rootin, len, i);
        if (localrc) return localrc;
        rootin += len;
      }
    }else{
      // all other PETs receive their chunk
      localrc = recv(out, len, root);
    }
  }
  return localrc;
}


int VMK::scatter(void *in, void *out, int len, int root, commhandle **ch){
  int localrc=0;
  // sanity check root
  if (root<0 || root>=npets){
    ESMC_LogDefault.MsgFoundError(ESMC_RC_ARG_OUTOFRANGE,
      "'root' out of range", ESMC_CONTEXT, &localrc);
    return localrc;
  }
  // check if this needs a new entry in the request queue
  if (*ch==NULL){
    *ch = new commhandle;
    commqueueitem_link(*ch);
  }
  // MPI does not offer a non-blocking scatter operation, hence there is no
  // point in checking if the mpionly flag is set in this VM, in either case
  // an explicit implementation based on send and recv must be used.
  // The number of elements in the commhandle depends on whether the localPET
  // is root or not.
  // This is a very simplistic, probably very bad peformance implementation.
  if (mypet==root){
    // I am root -> send chunks to all other PETs
    // rootPET will need to issue (npets-1) sends.
    (*ch)->nelements = npets-1;   // number of non-blocking sends
    (*ch)->type=0;                // these are subhandles
    (*ch)->handles = new commhandle*[(*ch)->nelements];
    for (int i=0; i<(*ch)->nelements; i++)
      (*ch)->handles[i] = new commhandle; // allocate handles
    // get ready to send chunks
    char *rootin = (char *)in;
    for (int i=0; i<root; i++){
      localrc = send(rootin, len, i, &((*ch)->handles[i]));
      if (localrc) return localrc;
      rootin += len;
    }
    // memcpy root's chunk
    memcpy(out, rootin, len);
    rootin += len;
    // keep sending chunks
    for (int i=root+1; i<npets; i++){
      localrc = send(rootin, len, i, &((*ch)->handles[i-1]));
      if (localrc) return localrc;
      rootin += len;
    }
  }else{
    // all other PETs receive their chunk
    // there will be a single receive that needs to be issued
    (*ch)->nelements = 1;
    (*ch)->type=0;                // these are subhandles
    (*ch)->handles = new commhandle*[1];
    (*ch)->handles[0] = new commhandle; // allocate handle
    localrc = recv(out, len, root, &((*ch)->handles[0]));
  }
  return localrc;
}


int VMK::scatterv(void *in, int *inCounts, int *inOffsets, void *out,
  int outCount, vmType type, int root){
  int localrc=0;
  // sanity check root
  if (root<0 || root>=npets){
    ESMC_LogDefault.MsgFoundError(ESMC_RC_ARG_OUTOFRANGE,
      "'root' out of range", ESMC_CONTEXT, &localrc);
    return localrc;
  }
  if (mpionly){
    // Find corresponding MPI data type
    MPI_Datatype mpitype;
    switch (type){
    case vmBYTE:
      mpitype = MPI_BYTE;
      break;
    case vmI4:
      mpitype = MPI_INT;
      break;
    case vmI8:
      mpitype = MPI_LONG_LONG_INT;
      break;
    case vmR4:
      mpitype = MPI_FLOAT;
      break;
    case vmR8:
      mpitype = MPI_DOUBLE;
      break;
    case vmL4:
      localrc = -1;   // error
      return localrc; // bail out
    }
    localrc = MPI_Scatterv(in, inCounts, inOffsets, mpitype, out, outCount,
      mpitype, root, mpi_c);
  }else{
    // This is a very simplistic, probably very bad peformance implementation.
    int size=0;
    switch (type){
    case vmI4:
      size=4;
      break;
    case vmI8:
      size=8;
      break;
    case vmR4:
      size=4;
      break;
    case vmR8:
      size=8;
      break;
    case vmBYTE:
    case vmL4:
      localrc = -1;   // error
      return localrc; // bail out
    }
    int root = 0; // arbitrary root, 0 always exists!
    if (mypet==root){
      // I am root -> send chunks to all other PETs
      int len;
      char *rootin;
      for (int i=0; i<root; i++){
        len = inCounts[i] * size;
        rootin = (char *)in + inOffsets[i] * size;
        localrc = send(rootin, len, i);
        if (localrc) return localrc;
      }
      // memcpy root's chunk
      len = inCounts[root] * size;
      rootin = (char *)in + inOffsets[root] * size;
      memcpy(out, rootin, len);
      // keep sending chunks
      for (int i=root+1; i<npets; i++){
        len = inCounts[i] * size;
        rootin = (char *)in + inOffsets[i] * size;
        localrc = send(rootin, len, i);
        if (localrc) return localrc;
      }
    }else{
      // all other PETs receive their chunk
      int len = outCount * size;
      localrc = recv(out, len, root);
    }
  }
  return localrc;
}


int VMK::gather(void *in, void *out, int len, int root){
  int localrc=0;
  // sanity check root
  if (root<0 || root>=npets){
    ESMC_LogDefault.MsgFoundError(ESMC_RC_ARG_OUTOFRANGE,
      "'root' out of range", ESMC_CONTEXT, &localrc);
    return localrc;
  }
  if (mpionly){
    localrc = MPI_Gather(in, len, MPI_BYTE, out, len, MPI_BYTE, root, mpi_c);
  }else{
    // This is a very simplistic, probably very bad peformance implementation.
    if (mypet==root){
      // I am root -> receive chunks from all other PETs
      char *rootout = (char *)out;
      for (int i=0; i<root; i++){
        localrc = recv(rootout, len, i);
        if (localrc) return localrc;
        rootout += len;
      }
      // memcpy root's chunk
      memcpy(rootout, in, len);
      rootout += len;
      // keep receiving chunks
      for (int i=root+1; i<npets; i++){
        localrc = recv(rootout, len, i);
        if (localrc) return localrc;
        rootout += len;
      }
    }else{
      // all other PETs send their chunk
      localrc = send(in, len, root);
    }
  }
  return localrc;
}


int VMK::gather(void *in, void *out, int len, int root, commhandle **ch){
  int localrc = 0;
  // sanity check root
  if (root<0 || root>=npets){
    ESMC_LogDefault.MsgFoundError(ESMC_RC_ARG_OUTOFRANGE,
      "'root' out of range", ESMC_CONTEXT, &localrc);
    return localrc;
  }
  // check if this needs a new entry in the request queue
  if (*ch==NULL){
    *ch = new commhandle;
    commqueueitem_link(*ch);
  }
  // MPI does not offer a non-blocking gather operation, hence there is no
  // point in checking if the mpionly flag is set in this VM, in either case
  // an explicit implementation based on send and recv must be used.
  // The number of elements in the commhandle depends on whether the localPET
  // is root or not.
  // This is a very simplistic, probably very bad peformance implementation.
  if (mypet==root){
    // I am root -> receive chunks from all other PETs
    // rootPET will need to issue (npets-1) recvs.
    (*ch)->nelements = npets-1;   // number of non-blocking recvs
    (*ch)->type=0;                // these are subhandles
    (*ch)->handles = new commhandle*[(*ch)->nelements];
    for (int i=0; i<(*ch)->nelements; i++)
      (*ch)->handles[i] = new commhandle; // allocate handles
    // get ready to receive chunks
    char *rootout = (char *)out;
    for (int i=0; i<root; i++){
      localrc = recv(rootout, len, i, &((*ch)->handles[i]));
      if (localrc) return localrc;
      rootout += len;
    }
    // memcpy root's chunk
    memcpy(rootout, in, len);
    rootout += len;
    // keep receiving chunks
    for (int i=root+1; i<npets; i++){
      localrc = recv(rootout, len, i, &((*ch)->handles[i-1]));
      if (localrc) return localrc;
      rootout += len;
    }
  }else{
    // all other PETs send their chunk
    // there will be a single send that needs to be issued
    (*ch)->nelements = 1;
    (*ch)->type=0;                // these are subhandles
    (*ch)->handles = new commhandle*[1];
    (*ch)->handles[0] = new commhandle; // allocate handle
    localrc = send(in, len, root, &((*ch)->handles[0]));
  }
  return localrc;
}


int VMK::gatherv(void *in, int inCount, void *out, int *outCounts,
  int *outOffsets, vmType type, int root){
  int localrc=0;
  // sanity check root
  if (root<0 || root>=npets){
    ESMC_LogDefault.MsgFoundError(ESMC_RC_ARG_OUTOFRANGE,
      "'root' out of range", ESMC_CONTEXT, &localrc);
    return localrc;
  }
  if (mpionly){
    // Find corresponding MPI data type
    MPI_Datatype mpitype;
    switch (type){
    case vmBYTE:
      mpitype = MPI_BYTE;
      break;
    case vmI4:
      mpitype = MPI_INT;
      break;
    case vmI8:
      mpitype = MPI_LONG_LONG_INT;
      break;
    case vmR4:
      mpitype = MPI_FLOAT;
      break;
    case vmR8:
      mpitype = MPI_DOUBLE;
      break;
    case vmL4:
      localrc = -1;   // error
      return localrc; // bail out
    }
    localrc = MPI_Gatherv(in, inCount, mpitype, out, outCounts, outOffsets,
      mpitype, root, mpi_c);
  }else{
    // This is a very simplistic, probably very bad peformance implementation.
    int size=0;
    switch (type){
    case vmBYTE:
      size=1;
      break;
    case vmI4:
      size=4;
      break;
    case vmI8:
      size=8;
      break;
    case vmR4:
      size=4;
      break;
    case vmR8:
      size=8;
      break;
    case vmL4:
      localrc = -1;   // error
      return localrc; // bail out
    }
    int root = 0; // arbitrary root, 0 always exists!
    if (mypet==root){
      // I am root -> receive chunks from all other PETs
      int len;
      char *rootout;
      for (int i=0; i<root; i++){
        len = outCounts[i] * size;
        rootout = (char *)out + outOffsets[i] * size;
        localrc = recv(rootout, len, i);
        if (localrc) return localrc;
      }
      // memcpy root's chunk
      len = outCounts[root] * size;
      rootout = (char *)out + outOffsets[root] * size;
      memcpy(rootout, in, len);
      // keep receiving chunks
      for (int i=root+1; i<npets; i++){
        len = outCounts[i] * size;
        rootout = (char *)out + outOffsets[i] * size;
        localrc = recv(rootout, len, i);
        if (localrc) return localrc;
      }
    }else{
      // all other PETs send their chunk
      int len = inCount * size;
      localrc = send(in, len, root);
    }
  }
  return localrc;
}


int VMK::allgather(void *in, void *out, int len){
  int localrc=0;
  if (mpionly){
    localrc = MPI_Allgather(in, len, MPI_BYTE, out, len, MPI_BYTE, mpi_c);
  }else{
    // This is a very simplistic, probably very bad peformance implementation.
    int root = 0; // arbitrary root, 0 always exists!
    if (mypet==root){
      // I am root -> receive chunks from all other PETs
      char *rootout = (char *)out;
      for (int i=0; i<root; i++){
        localrc = recv(rootout, len, i);
        if (localrc) return localrc;
        rootout += len;
      }
      // memcpy root's chunk
      memcpy(rootout, in, len);
      rootout += len;
      // keep receiving chunks
      for (int i=root+1; i<npets; i++){
        localrc = recv(rootout, len, i);
        if (localrc) return localrc;
        rootout += len;
      }
    }else{
      // all other PETs send their chunk
      localrc = send(in, len, root);
      if (localrc) return localrc;
    }
    // now broadcast root's out to all other PETs
    localrc = broadcast(out, len, root);
  }
  return localrc;
}


int VMK::allgather(void *in, void *out, int len, commhandle **ch){
  int localrc=0;
  // check if this needs a new entry in the request queue
  if (*ch==NULL){
    *ch = new commhandle;
    commqueueitem_link(*ch);
  }
  // MPI does not offer a non-blocking allgather operation, hence there is no
  // point in checking if the mpionly flag is set in this VM, in either case
  // an explicit implementation must be used.
  // There will be as many commhandles as there are PETs
  (*ch)->nelements = npets;     // number of non-blocking gathers
  (*ch)->type=0;                // these are subhandles
  (*ch)->handles = new commhandle*[(*ch)->nelements];
  for (int i=0; i<(*ch)->nelements; i++)
    (*ch)->handles[i] = new commhandle; // allocate handles
  // This is a very simplistic, probably very bad peformance implementation.
  for (int root=0; root<npets; root++){
    // Each PET is considered the root PET once for a non-blocking gather
    localrc = gather(in, out, len, root, &((*ch)->handles[root]));
    if (localrc) return localrc;
  }
  return localrc;
}


int VMK::allgatherv(void *in, int inCount, void *out, int *outCounts,
  int *outOffsets, vmType type){
  int localrc=0;
  if (mpionly){
    // Find corresponding MPI data type
    MPI_Datatype mpitype;
    switch (type){
    case vmBYTE:
      mpitype = MPI_BYTE;
      break;
    case vmI4:
      mpitype = MPI_INT;
      break;
    case vmI8:
      mpitype = MPI_LONG_LONG_INT;
      break;
    case vmR4:
      mpitype = MPI_FLOAT;
      break;
    case vmR8:
      mpitype = MPI_DOUBLE;
      break;
    case vmL4:
      localrc = -1;   // error
      return localrc; // bail out
    }
    localrc = MPI_Allgatherv(in, inCount, mpitype, out, outCounts, outOffsets,
      mpitype, mpi_c);
  }else{
    // This is a very simplistic, probably very bad peformance implementation.
    int size=0;
    switch (type){
    case vmBYTE:
      size=1;
      break;
    case vmI4:
      size=4;
      break;
    case vmI8:
      size=8;
      break;
    case vmR4:
      size=4;
      break;
    case vmR8:
      size=8;
      break;
    case vmL4:
      localrc = -1;   // error
      return localrc; // bail out
    }
    int root = 0; // arbitrary root, 0 always exists!
    if (mypet==root){
      // I am root -> receive chunks from all other PETs
      int len;
      char *rootout;
      for (int i=0; i<root; i++){
        len = outCounts[i] * size;
        rootout = (char *)out + outOffsets[i] * size;
        localrc = recv(rootout, len, i);
        if (localrc) return localrc;
      }
      // memcpy root's chunk
      len = outCounts[root] * size;
      rootout = (char *)out + outOffsets[root] * size;
      memcpy(rootout, in, len);
      // keep receiving chunks
      for (int i=root+1; i<npets; i++){
        len = outCounts[i] * size;
        rootout = (char *)out + outOffsets[i] * size;
        localrc = recv(rootout, len, i);
        if (localrc) return localrc;
      }
    }else{
      // all other PETs send their chunk
      int len = inCount * size;
      localrc = send(in, len, root);
      if (localrc) return localrc;
    }
    // now broadcast root's out to all other PETs
    int len=0;
    for (int i=0; i<npets; i++)
      len += outCounts[i] * size;
    localrc = broadcast(out, len, root);
  }
  return localrc;
}


int VMK::alltoall(void *in, int inCount, void *out, int outCount,
  vmType type){
  int localrc=0;
  if (mpionly){
    // Find corresponding MPI data type
    MPI_Datatype mpitype;
    switch (type){
    case vmBYTE:
      mpitype = MPI_BYTE;
      break;
    case vmI4:
      mpitype = MPI_INT;
      break;
    case vmI8:
      mpitype = MPI_LONG_LONG_INT;
      break;
    case vmR4:
      mpitype = MPI_FLOAT;
      break;
    case vmR8:
      mpitype = MPI_DOUBLE;
      break;
    case vmL4:
      mpitype = MPI_LOGICAL;
      break;
    }
    localrc = MPI_Alltoall(in, inCount, mpitype, out, outCount, mpitype, mpi_c);
  }else{
    // This is a very simplistic, probably very bad peformance implementation.
    int size=0;
    switch (type){
    case vmBYTE:
      size=1;
      break;
    case vmI4:
      size=4;
      break;
    case vmI8:
      size=8;
      break;
    case vmR4:
      size=4;
      break;
    case vmR8:
      size=8;
      break;
    case vmL4:
      size=4;
      break;
    }
    char *inC = (char *)in;
    char *outC = (char *)out;
    // send to all PETs with id smaller than mypet
    for (int i=0; i<mypet; i++){
      localrc = send(inC+inCount*i*size, inCount*size, i);
      if (localrc) return localrc;
    }
    // memcpy the local chunk
    memcpy(outC+outCount*mypet*size, inC+inCount*mypet*size, inCount*size);
    // receive the data from all Pets with id larger than mypet
    for (int i=mypet+1; i<npets; i++){
      localrc = recv(outC+outCount*i*size, outCount*size, i);
      if (localrc) return localrc;
    }
    // send to all PETs with larger than mypet
    for (int i=mypet+1; i<npets; i++){
      localrc = send(inC+inCount*i*size, inCount*size, i);
      if (localrc) return localrc;
    }
    // receive the data from all Pets with id smaller than mypet
    for (int i=0; i<mypet; i++){
      localrc = recv(outC+outCount*i*size, outCount*size, i);
      if (localrc) return localrc;
    }
  }
  return localrc;
}


int VMK::alltoallv(void *in, int *inCounts, int *inOffsets, void *out,
  int *outCounts, int *outOffsets, vmType type){
  int localrc=0;
  if (mpionly){
    // Find corresponding MPI data type
    MPI_Datatype mpitype;
    switch (type){
    case vmBYTE:
      mpitype = MPI_BYTE;
      break;
    case vmI4:
      mpitype = MPI_INT;
      break;
    case vmI8:
      mpitype = MPI_LONG_LONG_INT;
      break;
    case vmR4:
      mpitype = MPI_FLOAT;
      break;
    case vmR8:
      mpitype = MPI_DOUBLE;
      break;
    case vmL4:
      mpitype = MPI_LOGICAL;
      break;
    }
    localrc = MPI_Alltoallv(in, inCounts, inOffsets, mpitype, out, outCounts,
      outOffsets, mpitype, mpi_c);
  }else{
    // This is a very simplistic, probably very bad peformance implementation.
    int size=0;
    switch (type){
    case vmBYTE:
      size=1;
      break;
    case vmI4:
      size=4;
      break;
    case vmI8:
      size=8;
      break;
    case vmR4:
      size=4;
      break;
    case vmR8:
      size=8;
      break;
    case vmL4:
      size=4;
      break;
    }
    char *inC = (char *)in;
    char *outC = (char *)out;
    // send to all PETs with id smaller than mypet
    for (int i=0; i<mypet; i++){
      localrc = send(inC+inOffsets[i]*size, inCounts[i]*size, i);
      if (localrc) return localrc;
    }
    // memcpy the local chunk
    memcpy(outC+outOffsets[mypet]*size, inC+inOffsets[mypet]*size,
      inCounts[mypet]*size);
    // receive the data from all Pets with id larger than mypet
    for (int i=mypet+1; i<npets; i++){
      localrc = recv(outC+outOffsets[i]*size, outCounts[i]*size, i);
      if (localrc) return localrc;
    }
    // send to all PETs with larger than mypet
    for (int i=mypet+1; i<npets; i++){
      localrc = send(inC+inOffsets[i]*size, inCounts[i]*size, i);
      if (localrc) return localrc;
    }
    // receive the data from all Pets with id smaller than mypet
    for (int i=0; i<mypet; i++){
      localrc = recv(outC+outOffsets[i]*size, outCounts[i]*size, i);
      if (localrc) return localrc;
    }
  }
  return localrc;
}


int VMK::broadcast(void *data, int len, int root){
  int localrc=0;
  // sanity check root
  if (root<0 || root>=npets){
    ESMC_LogDefault.MsgFoundError(ESMC_RC_ARG_OUTOFRANGE,
      "'root' out of range", ESMC_CONTEXT, &localrc);
    return localrc;
  }
  if (mpionly){
    localrc = MPI_Bcast(data, len, MPI_BYTE, root, mpi_c);
  }else{
    // This is a very simplistic, probably very bad peformance implementation.
    if (mypet==root){
      // I am root -> send my data to all other PETs
      for (int i=0; i<npets; i++) {
        if (i==mypet) continue; // skip root PET
        localrc = send(data, len, i);
        if (localrc) return localrc;
      }
    }else{
      // all other PETs receive the broadcasted data
      localrc = recv(data, len, root);
    }
  }
  return localrc;
}


int VMK::broadcast(void *data, int len, int root, commhandle **ch){
  int localrc=0;
  // sanity check root
  if (root<0 || root>=npets){
    ESMC_LogDefault.MsgFoundError(ESMC_RC_ARG_OUTOFRANGE,
      "'root' out of range", ESMC_CONTEXT, &localrc);
    return localrc;
  }
  // check if this needs a new entry in the request queue
  if (*ch==NULL){
    *ch = new commhandle;
    commqueueitem_link(*ch);
  }
  // MPI does not offer a non-blocking broadcast operation, hence there is no
  // point in checking if the mpionly flag is set in this VM, in either case
  // an explicit implementation based on send and recv must be used.
  // The number of elements in the commhandle depends on whether the localPET
  // is root or not.
  // This is a very simplistic, probably very bad peformance implementation.
  if (mypet==root){
    // I am root -> send my data to all other PETs
    // rootPET will need to issue (npets-1) sends
    (*ch)->nelements = npets-1;   // number of non-blocking recvs
    (*ch)->type=0;                // these are subhandles
    (*ch)->handles = new commhandle*[(*ch)->nelements];
    for (int i=0; i<(*ch)->nelements; i++)
      (*ch)->handles[i] = new commhandle; // allocate handles
    // get ready to send chunks
    for (int i=0; i<root; i++){
      localrc = send(data, len, i, &((*ch)->handles[i]));
      if (localrc) return localrc;
    }
    // skip root
    for (int i=root+1; i<npets; i++){
      localrc = send(data, len, i, &((*ch)->handles[i-1]));
      if (localrc) return localrc;
    }
  }else{
    // all other PETs receive the broadcasted data
    // there will be a single receive that needs to be issued
    (*ch)->nelements = 1;
    (*ch)->type=0;                // these are subhandles
    (*ch)->handles = new commhandle*[1];
    (*ch)->handles[0] = new commhandle; // allocate handle
    localrc = recv(data, len, root, &((*ch)->handles[0]));
  }
  return localrc;
}


// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// ~~~ Timing Calls
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~


void VMK::wtime(double *time){
  *time = MPI_Wtime() - wtime0;
}


void VMK::wtimeprec(double *prec){
  double temp_prec = 0.;
  double t1, t2, dt;
  for(int i=0; i<10; i++){
    wtime(&t1);
    t2 = t1;
    while(fabs(t2-t1)<DBL_MIN)
      wtime(&t2);
    dt = t2 - t1;
    if (dt > temp_prec) temp_prec = dt;
  }
  *prec = temp_prec;
}


void VMK::wtimedelay(double delay){
  double t1, t2;
  wtime(&t1);
  t2 = t1;
  while(t2-t1<delay)
    wtime(&t2);
}


// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// ~~~ SSI shared memory methods
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

int VMK::ssishmAllocate(vector<unsigned long>&bytes, memhandle *memh, 
  bool contigFlag){
#if (MPI_VERSION >= 3) || (defined ESMF_MPIUNI)
#ifndef ESMF_MPIUNI
  MPI_Comm_rank(mpi_c_ssi, &(memh->localPet));
  MPI_Comm_size(mpi_c_ssi, &(memh->localPetCount));
  memh->counts.resize(memh->localPetCount);
  int count = (int)bytes.size();
  MPI_Allgather(&count, 1, MPI_INT, &(memh->counts[0]), 1, MPI_INT, mpi_c_ssi);
  int maxCount = *(max_element(memh->counts.begin(), memh->counts.end()));
#else
  memh->localPet = 0;
  memh->localPetCount = 1;
  memh->counts.resize(memh->localPetCount);
  int count = (int)bytes.size();
  memh->counts[0] = count;
  int maxCount = count;
#endif
#ifdef VM_SSISHMLOG_on
  {
    std::stringstream msg;
    msg << "ssishmAllocate#" << __LINE__
      << " localPetCount=" << memh->localPetCount
      << " bytes request maxCount=" << maxCount;
    ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
  }
#endif
  void *dummyPtr;
#ifndef ESMF_MPIUNI
  memh->wins.resize(maxCount);
#else
  memh->mems.resize(maxCount);
  memh->sizes.resize(maxCount);
#endif
  for (int i=0; i<maxCount; i++){
    unsigned long size=0;
    if (i<count) size = bytes[i];
#ifndef ESMF_MPIUNI
    MPI_Info info;
    MPI_Info_create(&info);
    if (!contigFlag){
      // allow system to allocate non-contiguous over SSI
      MPI_Info_set(info, "alloc_shared_noncontig", "true");
    }
    MPI_Win_allocate_shared(size, 1, info, mpi_c_ssi, &dummyPtr,
      &(memh->wins[i]));
    MPI_Info_free(&info);
#else
    memh->mems[i] = (void *)malloc(size);
    memh->sizes[i] = size;
#endif
  }
  return ESMF_SUCCESS;
#else
  std::stringstream msg;
  msg << "VMKernel in line #" << __LINE__ << " Method requires MPI3 support.";
  ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_ERROR);
  return ESMC_RC_INTNRL_BAD; // bail with error
#endif
}

int VMK::ssishmFree(memhandle *memh){
#if (MPI_VERSION >= 3) || (defined ESMF_MPIUNI)
#ifdef VM_SSISHMLOG_on
  {
    std::stringstream msg;
    msg << "ssishmFree#" << __LINE__ << " number of shared memory windows=" 
#ifndef ESMF_MPIUNI
      << memh->wins.size();
#else
      << memh->mems.size();
#endif
    ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
  }
#endif
  ssishmSync(*memh);
#ifndef ESMF_MPIUNI
  for (unsigned i=0; i<memh->wins.size(); i++)
    MPI_Win_free(&(memh->wins[i]));
  memh->wins.resize(0);
#else
  for (unsigned i=0; i<memh->mems.size(); i++)
    free(memh->mems[i]);
  memh->mems.resize(0);
  memh->sizes.resize(0);
#endif
  memh->counts.resize(0);
  memh->localPetCount=-1; // invalidate
#ifdef VM_SSISHMLOG_on
  {
    std::stringstream msg;
    msg << "ssishmFree#" << __LINE__ << " done.";
    ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
  }
#endif
  return ESMF_SUCCESS;
#else
  std::stringstream msg;
  msg << "VMKernel in line #" << __LINE__ << " Method requires MPI3 support.";
  ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_ERROR);
  return ESMC_RC_INTNRL_BAD; // bail with error
#endif
}

int VMK::ssishmGetMems(memhandle memh, int pet, vector<void *>*mems,
  vector<unsigned long> *bytes){
#if (MPI_VERSION >= 3) || (defined ESMF_MPIUNI)
  // error check the incoming information
  if (pet < 0 || pet > memh.localPetCount){
    std::stringstream msg;
    msg << "VMKernel in line #" << __LINE__ << " pet argument out of range.";
    ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_ERROR);
    return 1; // bail with error
  }
  // prepare the return vectors
  if (mems) mems->resize(memh.counts[pet]);
  if (bytes) bytes->resize(memh.counts[pet]);
  // construct the return information
  for (int i=0; i<memh.counts[pet]; i++){
#ifndef ESMF_MPIUNI
    MPI_Aint size;
    int disp_unit;
    void *baseptr;
    MPI_Win_shared_query(memh.wins[i], pet, &size, &disp_unit, &baseptr);
#else
    unsigned long size = memh.sizes[i];
    void *baseptr = memh.mems[i];
#endif
    if (mems) (*mems)[i]=baseptr;
    if (bytes) (*bytes)[i]=size;
#ifdef VM_SSISHMLOG_on
    {
      std::stringstream msg;
      msg << "ssishmGetMems#" << __LINE__
        << " pet=" << pet << " baseptr=" << baseptr << " size=" << size;
      ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
    }
#endif
  }
  return ESMF_SUCCESS;
#else
  std::stringstream msg;
  msg << "VMKernel in line #" << __LINE__ << " Method requires MPI3 support.";
  ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_ERROR);
  return ESMC_RC_INTNRL_BAD; // bail with error
#endif
}

int VMK::ssishmSync(memhandle memh){
#if (MPI_VERSION >= 3) || (defined ESMF_MPIUNI)
#ifdef VM_SSISHMLOG_on
  {
    std::stringstream msg;
    msg << "ssishmSync#" << __LINE__ << " entering.";
    ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
  }
#endif
#ifndef ESMF_MPIUNI
  // call barrier for the ssi-local communicator
  MPI_Barrier(mpi_c_ssi);
  return ESMF_SUCCESS;
#else
  // nothing to be done for MPIUNI
  return ESMF_SUCCESS;
#endif
#else
  std::stringstream msg;
  msg << "VMKernel in line #" << __LINE__ << " Method requires MPI3 support.";
  ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_ERROR);
  return ESMC_RC_INTNRL_BAD; // bail with error
#endif
}

// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// ~~~ IntraProcessSharedMemoryAllocation List Methods
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~


void *VMK::ipshmallocate(int bytes, int *firstFlag){
  if (firstFlag != NULL) *firstFlag = 0; // reset
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_lock(ipshmMutex);
#endif
  if (ipshmLocalTop == *ipshmTop){
    // this is the first thread for this new request: add element + allocate
    *ipshmTop = new ipshmAlloc;  // new top element in shared allocation list
    (*ipshmTop)->allocation = (void *)malloc(bytes); // actual memory alloc
    (*ipshmTop)->auxCounter = getNthreads(getMypet()); // reset
    (*ipshmTop)->prev = NULL;          // indicate end of list
    (*ipshmTop)->next = ipshmLocalTop; // link to previous top element in list
    if (ipshmLocalTop != NULL)
      ipshmLocalTop->prev = *ipshmTop;  // back link
    ipshmLocalTop = *ipshmTop;          // update local top pointer
    if (firstFlag != NULL) *firstFlag = 1; // set caller provided flag
  }else{
    // this is a secondary thread for this request: find allocation element
    if (ipshmLocalTop != NULL)
      ipshmLocalTop = ipshmLocalTop->prev;
    else{
      // need to search for the previous element
      ipshmAlloc *ipshmTemp = *ipshmTop; // start at the top
      while (ipshmTemp != NULL){
        if (ipshmTemp->next == ipshmLocalTop) break;
        ipshmTemp = ipshmTemp->next;
      }
      if (ipshmTemp != NULL)
        ipshmLocalTop = ipshmTemp;  // new local top element
    }
  }
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_unlock(ipshmMutex);
#endif
  return ipshmLocalTop->allocation;  // return allocation of local top
}


void VMK::ipshmdeallocate(void *pointer){
  // this call has undefined behavior if called multiple times from the same
  // thread with identical pointer argument
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_lock(ipshmMutex);
#endif

  if (getNthreads(getMypet()) == 1){
    // PET is in single-thread group -> much simpler and faster
    // don't do anything here, and have the VMK::finalize() take care of it
  }else{
    // PET is part of a multi-thread group -> need to search for entry
    // maybe using a std::map would help for more efficient search as number
    // of objects increases
    ipshmAlloc *ipshmTemp = *ipshmTop; // start at the current top of the list
    while (ipshmTemp != NULL){
      if (ipshmTemp->allocation == pointer) break;
      ipshmTemp = ipshmTemp->next;
    }
    if (ipshmTemp!=NULL){
      // found the allocation
      --(ipshmTemp->auxCounter); // count this thread's deallocate call
      if (ipshmTemp->auxCounter == 0){
        // this was the last thread to call deallocate for this allocation
        //printf("freeing %p\n", pointer);
        free(pointer);
        // Cannot deallocate the allocation element in list here without
        // disturbing list structure which is shared between threads.
        // It is anyway safer to do a centralized deallocation of this structure
        // during VM shutdown as it gives a chance to free any remaining
        // pointers that are still allocated in order to prevent memory leaks
        // because of improper user code.
      }
    }
  }
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_unlock(ipshmMutex);
#endif
}


// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// ~~~ IntraProcessMutex Methods
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

VMK::ipmutex *VMK::ipmutexallocate(){
  int firstFlag;
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_lock(ipSetupMutex);
#endif
  ipmutex *ipm = (ipmutex *)
    ipshmallocate(sizeof(ipmutex), &firstFlag);
#ifndef ESMF_NO_PTHREADS
  if (firstFlag) pthread_mutex_init(&(ipm->pth_mutex), NULL);
#endif
  ipm->lastFlag = getNthreads(getMypet()); //reset
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_unlock(ipSetupMutex);
#endif
  return ipm;
}

void VMK::ipmutexdeallocate(ipmutex *ipm){
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_lock(ipSetupMutex);
#endif
  --(ipm->lastFlag);  // register this thread
#ifndef ESMF_NO_PTHREADS
  if (ipm->lastFlag == 0) pthread_mutex_destroy(&(ipm->pth_mutex));
#endif
  ipshmdeallocate(ipm);
#ifndef ESMF_NO_PTHREADS
  pthread_mutex_unlock(ipSetupMutex);
#endif
}

int VMK::ipmutexlock(ipmutex *ipm){
#ifndef ESMF_NO_PTHREADS
  return pthread_mutex_lock(&(ipm->pth_mutex));
#else
  return 0;
#endif
}

int VMK::ipmutexunlock(ipmutex *ipm){
#ifndef ESMF_NO_PTHREADS
  return pthread_mutex_unlock(&(ipm->pth_mutex));
#else
  return 0;
#endif
}


// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// ~~~ Simple thread-safety lock/unlock using internal pth_mutex
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

int VMK::lock(){
#ifndef ESMF_NO_PTHREADS
  return pthread_mutex_lock(pth_mutex);
#else
  return 0;
#endif
}

int VMK::unlock(){
#ifndef ESMF_NO_PTHREADS
  return pthread_mutex_unlock(pth_mutex);
#else
  return 0;
#endif
}

} // namespace ESMCI


// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// ~~~ Sync Calls
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// I noticed that you get slightly better performance if the sync functions are
// compiled without optimization! For this feature they need to be placed into
// a separate file again...


void sync_a_flip(shmsync *shms){
  volatile int *b_done = shms->b_done;
  *(shms->a_done) = 1;      // signal that send has arrived
  while (*b_done == 0);     // check whether recv has arrived
}

void sync_a_flop(shmsync *shms){
  volatile int *a_done = shms->a_done;
  *(shms->b_done) = 0;      // reset for next time
  while (*a_done == 1);     // check whether recv has arrived
}


void sync_b_flip(shmsync *shms){
  volatile int *a_done = shms->a_done;
  *(shms->b_done) = 1;      // signal that recv has arrived
  while (*a_done == 0);     // check whether send has arrived
}

void sync_b_flop(shmsync *shms){
  volatile int *b_done = shms->b_done;
  *(shms->a_done) = 0;      // reset for next time
  while (*b_done == 1);     // check whether send has arrived
}


void sync_buffer_flag_fill(shmsync *shms, int select){
  *(shms->buffer_done[select]) = 1;      // flag that buffer has been filled
}

void sync_buffer_flag_empty(shmsync *shms, int select){
  *(shms->buffer_done[select]) = 0;      // flag that buffer has been cleared
}

void sync_buffer_wait_fill(shmsync *shms, int select){
  volatile int *buffer_done = shms->buffer_done[select];
  while (*buffer_done == 0);     // check whether buffer has been filled
}

void sync_buffer_wait_empty(shmsync *shms, int select){
  volatile int *buffer_done = shms->buffer_done[select];
  while (*buffer_done == 1);     // check whether buffer has been cleared
}


void sync_reset(shmsync *shms){
  *(shms->a_done) = 0;
  *(shms->b_done) = 0;
  for (int i=0; i<SYNC_NBUFFERS; i++)
    *(shms->buffer_done[i]) = 0;
}


//==============================================================================
//==============================================================================
//==============================================================================
// ComPat: abstract class providing basic communication patters
//==============================================================================
//==============================================================================
//==============================================================================

#define DEBUG_COMPAT_off

namespace ESMCI{
  void ComPat::totalExchange(VMK *vmk){
    int petCount = vmk->getNpets();
    int localPet = vmk->getMypet();
    // prepare commhandles and message buffers
    vector<VMK::commhandle *> sendCommhList(petCount);
    vector<VMK::commhandle *> recvCommhList(petCount);
    vector<char *> sendBuffer(petCount);
    vector<char *> recvBuffer(petCount);
    const int boostSize = 512;  // max number of posted non-blocking calls:
                                // stay below typical system limits
    int iiStart = localPet+1; // initialize
    int sendIndexOffset = 2*localPet+petCount;
    do{
      int iiEnd = iiStart + boostSize;
      if (iiEnd > localPet+petCount)
        iiEnd = localPet+petCount;
      // localPet acts as receiver, posting non-blocking recvs for all senders
      for (int ii=iiStart; ii<iiEnd; ii++){
        // localPet-dependent shifted loop reduces communication contention
        int i = ii%petCount;  // fold back into [0,..,petCount-1] range
        // receive message from Pet "i"
        int size = messageSize(i, localPet);
        if (size>0){
          recvBuffer[i] = new char[size];
          recvCommhList[i] = NULL;
          vmk->recv(recvBuffer[i], size, i, &(recvCommhList[i]));
#ifdef DEBUG_COMPAT_on
          {
            std::stringstream msg;
            msg << "ComPat#" << __LINE__
              << " posting receive from i=" << i << " size=" << size
              << " recvBuffer=" << (void *)recvBuffer[i];
            ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
          }
#endif
        }
      }

#ifndef USE_OLD_MESSAGEPREPARE

      // Allocate buffers
      for (int ii=sendIndexOffset-iiStart; ii>sendIndexOffset-iiEnd; ii--){
        // localPet-dependent shifted loop reduces communication contention
        int dstPet = ii%petCount;  // fold back into [0,..,petCount-1] range
        // send message to dstPet
        int size = messageSize(localPet, dstPet);
        if (size>0)
          sendBuffer[dstPet] = new char[size];
        else
          sendBuffer[dstPet] = NULL;
      }
      // Fill buffers
      messagePrepareSearch(vmk, sendIndexOffset, iiStart, iiEnd, sendBuffer);
      // Communicate buffers
      for (int ii=sendIndexOffset-iiStart; ii>sendIndexOffset-iiEnd; ii--){
        // localPet-dependent shifted loop reduces communication contention
        int dstPet = ii%petCount;  // fold back into [0,..,petCount-1] range
        // send message to Pet "i"
        int size = messageSize(localPet, dstPet);
        if (size>0){
#ifdef MUST_USE_BLOCKING_SEND
          vmk->send(sendBuffer[dstPet], size, dstPet);
#else
          sendCommhList[dstPet] = NULL;
          vmk->send(sendBuffer[dstPet], size, dstPet, &(sendCommhList[dstPet]));
#endif
#ifdef DEBUG_COMPAT_on
          {
            std::stringstream msg;
            msg << "ComPat#" << __LINE__
              << " posting send to i=" << dstPet << " size=" << size
              << " sendBuffer=" << (void *)sendBuffer[dstPet];
            ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
          }
#endif
        }
      }
#else
      // localPet acts as a sender, constructs message and sends to receiver
      for (int ii=sendIndexOffset-iiStart; ii>sendIndexOffset-iiEnd; ii--){
        // localPet-dependent shifted loop reduces communication contention
        int i = ii%petCount;  // fold back into [0,..,petCount-1] range
        // send message to Pet "i"
        int size = messageSize(localPet, i);
        if (size>0){
          sendBuffer[i] = new char[size];
          messagePrepare(localPet, i, sendBuffer[i]);
#ifdef MUST_USE_BLOCKING_SEND
          vmk->send(sendBuffer[i], size, i);
#else
          sendCommhList[i] = NULL;
          vmk->send(sendBuffer[i], size, i, &(sendCommhList[i]));
#endif
#ifdef DEBUG_COMPAT_on
          {
            std::stringstream msg;
            msg << "ComPat#" << __LINE__
              << " posting send to i=" << i << " size=" << size
              << " sendBuffer=" << (void *)sendBuffer[i];
            ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
          }
#endif
        }
      }
#endif

      if (iiStart==localPet+1){
        // localPet does local prepare and process
#ifdef DEBUG_COMPAT_on
        {
          std::stringstream msg;
          msg << "ComPat#" << __LINE__
            << " doing localPrepareAndProcess for PET=" << localPet;
          ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
        }
#endif
        localPrepareAndProcess(localPet);
      }
      // localPet acts receiver, processing message
      for (int ii=iiStart; ii<iiEnd; ii++){
        // localPet-dependent shifted loop reduces communication contention
        int i = ii%petCount;  // fold back into [0,..,petCount-1] range
        // receive message from Pet "i"
        int size = messageSize(i, localPet);
        if (size>0){
          vmk->commwait(&(recvCommhList[i]));   // wait for receive to finish
#ifdef DEBUG_COMPAT_on
          {
            std::stringstream msg;
            msg << "ComPat#" << __LINE__
              << " finished receive from i=" << i << " size=" << size
              << " recvBuffer=" << (void *)recvBuffer[i];
            ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
          }
#endif
          messageProcess(i, localPet, recvBuffer[i]);
          delete [] recvBuffer[i];              // garbage collection
        }
      }
      // localPet finishes up as sender
      for (int ii=sendIndexOffset-iiStart; ii>sendIndexOffset-iiEnd; ii--){
        // localPet-dependent shifted loop reduces communication contention
        int i = ii%petCount;  // fold back into [0,..,petCount-1] range
        // was sending message to Pet "i"
        int size = messageSize(localPet, i);
        if (size>0){
#ifndef MUST_USE_BLOCKING_SEND
          vmk->commwait(&(sendCommhList[i]));   // wait for send to finish
#endif
#ifdef DEBUG_COMPAT_on
          {
            std::stringstream msg;
            msg << "ComPat#" << __LINE__
              << " finished send to i=" << i << " size=" << size
              << " sendBuffer=" << (void *)sendBuffer[i];
            ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
          }
#endif
          delete [] sendBuffer[i];              // garbage collection
        }
      }
      iiStart = iiEnd;
    }while (iiStart < localPet+petCount);
  }
} // namespace ESMCI


//==============================================================================
//==============================================================================
//==============================================================================
// ComPat2: abstract class providing basic communication patters
//==============================================================================
//==============================================================================
//==============================================================================

#define DEBUG_COMPAT2_off

namespace ESMCI{
  void ComPat2::totalExchange(VMK *vmk){
    int petCount = vmk->getNpets();
    int localPet = vmk->getMypet();
    // prepare commhandles and message buffers
    VMK::commhandle *sendCommh1 = NULL;
    VMK::commhandle *sendCommh2 = NULL;
    VMK::commhandle *sendCommh3 = NULL;
    VMK::commhandle *sendCommh4 = NULL;
    VMK::commhandle *recvCommh1 = NULL;
    VMK::commhandle *recvCommh2 = NULL;
    char *sendRequestBuffer;
    char *sendResponseBuffer;
    char *recvBuffer1;
    char *recvBuffer2;
    for (int i=0; i<petCount; i++){
      int requestPet = (petCount + localPet-i) % petCount;
      int responsePet = (localPet+i) % petCount;
#ifdef DEBUG_COMPAT2_on
      {
        std::stringstream msg;
        msg << "ComPat2#" << __LINE__
          << " requestPet=" << requestPet 
          << " responsePet=" << responsePet;
        ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
      }
#endif
      if (i==0){
        // the localPet handles its own local operations
        handleLocal();
      }else{
        // localPet interacts with requestPet and responsePet as their
        // responder and requester, respectively.
        int recvResponseSize=0; // reset
        int sendResponseSize=0; // reset
        // localPet acts as responder
        int recvRequestSize;
        vmk->recv(&recvRequestSize, sizeof(int), requestPet, &recvCommh1);
        // localPet acts as requester
        int sendRequestSize;
        generateRequest(responsePet, sendRequestBuffer, sendRequestSize);
#ifdef MUST_USE_BLOCKING_SEND
        vmk->send(&sendRequestSize, sizeof(int), responsePet);
#else
        vmk->send(&sendRequestSize, sizeof(int), responsePet, &sendCommh1);
#endif
        // localPet acts as responder
        vmk->commwait(&recvCommh1); // wait for valid recvRequestSize
#ifdef DEBUG_COMPAT2_on
        {
          std::stringstream msg;
          msg << "ComPat2#" << __LINE__
            << " recvRequestSize=" << recvRequestSize
            << " sendRequestSize=" << sendRequestSize;
          ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
        }
#endif
        if (recvRequestSize>0){
          recvBuffer1 = new char[recvRequestSize];
          vmk->recv(recvBuffer1, recvRequestSize, requestPet, &recvCommh1);
#ifdef DEBUG_COMPAT2_on
          {
            std::stringstream msg;
            msg << "ComPat2#" << __LINE__
              << " receiving request from requestPet=" << requestPet
              << " in recvBuffer1=" << (void*)recvBuffer1;
            ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
          }
#endif
        }
        // localPet acts as requester
        if (sendRequestSize>0){
#ifdef MUST_USE_BLOCKING_SEND
          vmk->send(sendRequestBuffer, sendRequestSize, responsePet);
#else
          vmk->send(sendRequestBuffer, sendRequestSize, responsePet,
            &sendCommh2);
#endif
#ifdef DEBUG_COMPAT2_on
          {
            std::stringstream msg;
            msg << "ComPat2#" << __LINE__
              << " sending request to responsePet=" << responsePet
              << " in sendRequestBuffer=" << (void*)sendRequestBuffer;
            ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
          }
#endif
          vmk->recv(&recvResponseSize, sizeof(int), responsePet, &recvCommh2);
        }
        // localPet acts as responder
        if (recvRequestSize>0){
          vmk->commwait(&recvCommh1); // wait for valid recvBuffer1
#ifdef DEBUG_COMPAT2_on
          {
            std::stringstream msg;
            msg << "ComPat2#" << __LINE__
              << " finished receiving request from requestPet=" << requestPet
              << " in recvBuffer1=" << (void*)recvBuffer1;
            ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
          }
#endif
          sendResponseBuffer = NULL; // detectable reset
          handleRequest(requestPet, recvBuffer1, recvRequestSize,
            sendResponseBuffer, sendResponseSize);
#ifdef MUST_USE_BLOCKING_SEND
          vmk->send(&sendResponseSize, sizeof(int), requestPet);
#else
          vmk->send(&sendResponseSize, sizeof(int), requestPet, &sendCommh3);
#endif
        }
        // localPet acts as requester
        if (sendRequestSize>0){
          vmk->commwait(&recvCommh2); // wait for valid recvResponseSize
#ifndef MUST_USE_BLOCKING_SEND
          vmk->commwait(&sendCommh2); // wait to be done with sendRequestBuffer
#endif
        }
#ifdef DEBUG_COMPAT2_on
        {
          std::stringstream msg;
          msg << "ComPat2#" << __LINE__
            << " sendResponseSize=" << sendResponseSize
            << " recvResponseSize=" << recvResponseSize;
          ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
        }
#endif
        recvBuffer2 = NULL; // detectable reset
        if (recvResponseSize>0){
          recvBuffer2 = new char[recvResponseSize];
          vmk->recv(recvBuffer2, recvResponseSize, responsePet, &recvCommh2);
#ifdef DEBUG_COMPAT2_on
          {
            std::stringstream msg;
            msg << "ComPat2#" << __LINE__
              << " receiving response from responsePet=" << responsePet
              << " in recvBuffer2=" << (void*)recvBuffer2;
            ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
          }
#endif
        }
        // localPet acts as responder
        if (sendResponseSize>0){
#ifdef MUST_USE_BLOCKING_SEND
          vmk->send(sendResponseBuffer, sendResponseSize, requestPet);
#else          
          vmk->send(sendResponseBuffer, sendResponseSize, requestPet,
            &sendCommh4);
#endif
#ifdef DEBUG_COMPAT2_on
          {
            std::stringstream msg;
            msg << "ComPat2#" << __LINE__
              << " sending response to requestPet=" << requestPet
              << " in sendResponseBuffer=" << (void*)sendResponseBuffer;
            ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
          }
#endif
        }
        // localPet acts as requester
        if (recvResponseSize>0){        
          vmk->commwait(&recvCommh2); // wait for valid recvBuffer2
#ifdef DEBUG_COMPAT2_on
          {
            std::stringstream msg;
            msg << "ComPat2#" << __LINE__
              << " finished receiving response from responsePet=" << responsePet
              << " in recvBuffer2=" << (void*)recvBuffer2;
            ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
          }
#endif
          handleResponse(responsePet, recvBuffer2, recvResponseSize);
        }
        // localPet acts as requester
#ifndef MUST_USE_BLOCKING_SEND
        vmk->commwait(&sendCommh1);
#endif
        if (recvResponseSize>0){
          delete [] recvBuffer2;
        }
        // localPet acts as responder
        if (sendResponseSize>0){
#ifndef MUST_USE_BLOCKING_SEND
          vmk->commwait(&sendCommh4);
#endif
        }
        if (recvRequestSize>0){
#ifndef MUST_USE_BLOCKING_SEND
          vmk->commwait(&sendCommh3);
#endif
          if ((sendResponseBuffer != NULL) && (sendResponseBuffer!=recvBuffer1))
            delete [] sendResponseBuffer;
          delete [] recvBuffer1;
        }
      }
      
    }
  }
  
  //===========================================================================
  
  void ComPat2::selectiveExchange(VMK *vmk, std::vector<int>&responderPet,
    std::vector<int>&requesterPet){
    int petCount = vmk->getNpets();
    int localPet = vmk->getMypet();
    // prepare commhandles and message buffers
    VMK::commhandle *sendCommh1 = NULL;
    VMK::commhandle *sendCommh2 = NULL;
    VMK::commhandle *sendCommh3 = NULL;
    VMK::commhandle *sendCommh4 = NULL;
    VMK::commhandle *recvCommh1 = NULL;
    VMK::commhandle *recvCommh2 = NULL;
    char *sendRequestBuffer;
    char *sendResponseBuffer;
    char *recvBuffer1;
    char *recvBuffer2;
    for (int i=0; i<petCount; i++){
      int requestPet = (petCount + localPet-i) % petCount;
      int responsePet = (localPet+i) % petCount;
#ifdef DEBUG_COMPAT2_on
      {
        std::stringstream msg;
        msg << "ComPat2#" << __LINE__
          << " requestPet=" << requestPet 
          << " responsePet=" << responsePet;
        ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
      }
#endif
      if (i==0){
        // the localPet handles its own local operations
        handleLocal();
      }else{
        // localPet interacts with requestPet and responsePet as their
        // responder and requester, respectively.
        int recvResponseSize=0; // reset
        int sendResponseSize=0; // reset
        // localPet acts as responder
        int recvRequestSize;
        if (requesterPet[requestPet])
          vmk->recv(&recvRequestSize, sizeof(int), requestPet, &recvCommh1);
        else
          recvRequestSize=0;
        // localPet acts as requester
        int sendRequestSize;
        if (responderPet[responsePet]){
          generateRequest(responsePet, sendRequestBuffer, sendRequestSize);
          vmk->send(&sendRequestSize, sizeof(int), responsePet, &sendCommh1);
        }else
          sendRequestSize=0;
        // localPet acts as responder
        if (requesterPet[requestPet])
          vmk->commwait(&recvCommh1); // wait for valid recvRequestSize
#ifdef DEBUG_COMPAT2_on
          {
            std::stringstream msg;
            msg << "ComPat2#" << __LINE__
              << " recvRequestSize=" << recvRequestSize
              << " sendRequestSize=" << sendRequestSize;
            ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
          }
#endif
        if (recvRequestSize>0){
          recvBuffer1 = new char[recvRequestSize];
          vmk->recv(recvBuffer1, recvRequestSize, requestPet, &recvCommh1);
#ifdef DEBUG_COMPAT2_on
          {
            std::stringstream msg;
            msg << "ComPat2#" << __LINE__
              << " receiving request from requestPet=" << requestPet
              << " in recvBuffer1=" << (void*)recvBuffer1;
            ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
          }
#endif
        }
        // localPet acts as requester
        if (sendRequestSize>0){
          vmk->send(sendRequestBuffer, sendRequestSize, responsePet,
            &sendCommh2);
#ifdef DEBUG_COMPAT2_on
          {
            std::stringstream msg;
            msg << "ComPat2#" << __LINE__
              << " sending request to responsePet=" << responsePet
              << " in sendRequestBuffer=" << (void*)sendRequestBuffer;
            ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
          }
#endif
          vmk->recv(&recvResponseSize, sizeof(int), responsePet, &recvCommh2);
        }
        // localPet acts as responder
        if (recvRequestSize>0){
          vmk->commwait(&recvCommh1); // wait for valid recvBuffer1
#ifdef DEBUG_COMPAT2_on
          {
            std::stringstream msg;
            msg << "ComPat2#" << __LINE__
              << " finished receiving request from requestPet=" << requestPet
              << " in recvBuffer1=" << (void*)recvBuffer1;
            ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
          }
#endif
          sendResponseBuffer = NULL; // detectable reset
          handleRequest(requestPet, recvBuffer1, recvRequestSize,
            sendResponseBuffer, sendResponseSize);
          vmk->send(&sendResponseSize, sizeof(int), requestPet, &sendCommh3);
        }
        // localPet acts as requester
        if (sendRequestSize>0){
          vmk->commwait(&recvCommh2); // wait for valid recvResponseSize
          vmk->commwait(&sendCommh2); // wait to be done with sendRequestBuffer
        }
#ifdef DEBUG_COMPAT2_on
        {
          std::stringstream msg;
          msg << "ComPat2#" << __LINE__
            << " sendResponseSize=" << sendResponseSize
            << " recvResponseSize=" << recvResponseSize;
          ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
        }
#endif
        recvBuffer2 = NULL; // detectable reset
        if (recvResponseSize>0){
          recvBuffer2 = new char[recvResponseSize];
          vmk->recv(recvBuffer2, recvResponseSize, responsePet, &recvCommh2);
#ifdef DEBUG_COMPAT2_on
          {
            std::stringstream msg;
            msg << "ComPat2#" << __LINE__
              << " receiving response from responsePet=" << responsePet
              << " in recvBuffer2=" << (void*)recvBuffer2;
            ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
          }
#endif
        }
        // localPet acts as responder
        if (sendResponseSize>0){
          vmk->send(sendResponseBuffer, sendResponseSize, requestPet,
            &sendCommh4);
#ifdef DEBUG_COMPAT2_on
          {
            std::stringstream msg;
            msg << "ComPat2#" << __LINE__
              << " sending response to requestPet=" << requestPet
              << " in sendResponseBuffer=" << (void*)sendResponseBuffer;
            ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
          }
#endif
        }
        // localPet acts as requester
        if (recvResponseSize>0){        
          vmk->commwait(&recvCommh2); // wait for valid recvBuffer2
#ifdef DEBUG_COMPAT2_on
          {
            std::stringstream msg;
            msg << "ComPat2#" << __LINE__
              << " finished receiving response from responsePet=" << responsePet
              << " in recvBuffer2=" << (void*)recvBuffer2;
            ESMC_LogDefault.Write(msg.str(), ESMC_LOGMSG_DEBUG);
          }
#endif
          handleResponse(responsePet, recvBuffer2, recvResponseSize);
        }
        // localPet acts as requester
        if (responderPet[responsePet])
          vmk->commwait(&sendCommh1);
        if (recvResponseSize>0){
          delete [] recvBuffer2;
        }
        // localPet acts as responder
        if (sendResponseSize>0){
          vmk->commwait(&sendCommh4);
        }
        if (recvRequestSize>0){
          vmk->commwait(&sendCommh3);
          if ((sendResponseBuffer != NULL) && (sendResponseBuffer!=recvBuffer1))
            delete [] sendResponseBuffer;
          delete [] recvBuffer1;
        }
      }
      
    }
  }
  
} // namespace ESMCI

//==============================================================================
//==============================================================================
//==============================================================================
// Socket based VMKernel prototyping
//==============================================================================
//==============================================================================
//==============================================================================

#ifndef ESMF_NO_SOCKETS

#ifdef ESMF_OS_MinGW

#include <Windows.h>
#include <Winsock.h>
typedef int socklen_t;
typedef char* value_ptr_t;
#define ECONNABORTED WSAECONNABORTED
#define EALREADY WSAEALREADY
#define ECONNREFUSED WSAECONNREFUSED
#define EINPROGRESS WSAEINPROGRESS
#define EWOULDBLOCK WSAEWOULDBLOCK
// #define errno WSAGetLastError()

#else

#include <sys/socket.h>
#include <sys/select.h>
#include <netinet/in.h>
#include <netdb.h>
#include <unistd.h>
#include <fcntl.h>
typedef void* value_ptr_t;

#endif

#include <errno.h>

#define SERVERFILE "server.txt"   // file containing server name
#define PORT 54320                // a random port for prototype testing

#endif

namespace ESMCI {

  int socketServerInit(
    int port,               // port number
    double timeout          // timeout in seconds
    //--------------------------------------------------------------------------
    // Attempt to open an INET socket as server and wait for a client to connect
    // The return value are:
    // >SOCKERR_UNSPEC  -- successfully connected socket
    // SOCKERR_UNSPEC   -- unspecified error, may be fatal, prints perror()
    // SOCKERR_TIMEOUT  -- timeout condition was reached
    //--------------------------------------------------------------------------
  ){

    fprintf(stderr, "Hi there from socketServerInit()\n");
    
#ifdef ESMF_NO_SOCKETS
    fprintf(stderr, "ESMF was built with ESMF_NO_SOCKETS\n");
    return SOCKERR_UNSPEC;
#else
    
    // create an inet/stream socket
    int sock = socket(PF_INET, SOCK_STREAM, 0);
    if (sock < 0){
      perror("socketServerInit: socket()");
      return SOCKERR_UNSPEC;  // bail out
    }
    // allow immediate address + port reuse in bind
    int value = 1;
    if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (value_ptr_t)&value,
      sizeof(value)) < 0){
      perror("socketServerInit: setsockopt()");
      return SOCKERR_UNSPEC;  // bail out
    }
    // bind the hosts address + a port to it
    struct sockaddr_in name;
    name.sin_family = AF_INET;
    name.sin_port = htons(port);
    name.sin_addr.s_addr = INADDR_ANY;  // system to fill in automatically
    if (::bind(sock, (struct sockaddr *) &name, sizeof(name)) < 0){
      perror("socketServerInit: bind()");
      return SOCKERR_UNSPEC;  // bail out
    }
    // turn it into a server socket that is listening for connections
    if (listen(sock, 1) < 0){
      perror("socketServerInit: listen()");
      return SOCKERR_UNSPEC;  // bail out
    }
    // make socket non-blocking
#if !defined (ESMF_OS_MinGW)
    int sockFlags = fcntl(sock, F_GETFL);
    fcntl(sock, F_SETFL, sockFlags | O_NONBLOCK);   // Add non-blocking flag
#else
    unsigned long nbio_on = 1;
    ioctlsocket (sock, FIONBIO, &nbio_on);  // Set non-blocking flag
#endif
    // accept incoming connection from client -> but limit to time out
    double t0, t1;
    VMK::wtime(&t0);
    VMK::wtime(&t1);
    int newSock;
    
//TODO: I think using select would be a better use here    
    
    while (((newSock = accept(sock, NULL, NULL)) < 0) && (t1 - t0 <= timeout)){
      VMK::wtime(&t1);
    }
    fprintf(stderr, "socketServerInit waited: %g\n", t1-t0);
    
    // close the original socket
#if !defined (ESMF_OS_MinGW)
    if (close(sock) < 0){
#else
    if (closesocket(sock) < 0) {
#endif
      perror("socketServerInit: close()");
      return SOCKERR_UNSPEC;  // bail out
    }

    if (newSock < 0){
      if ((t1-t0) > timeout)
        return SOCKERR_TIMEOUT; // bail out
      // error condition on accept()
      perror("socketServerInit: accept()");
      return SOCKERR_UNSPEC;  // bail out
    }
    
    // newSock socket does _not_ inherit the non-blocking setting
    // return successfully
    fprintf(stderr, "socketServerInit: CONNECTED!\n");
    return newSock; // return the connected socket
#endif
  }

  // ------------------------------------------------------------------------
  
  int socketClientInit(
    char const *serverName, // server by name
    int port,               // port number
    double timeout          // timeout in seconds
    //--------------------------------------------------------------------------
    // Attempt to open an INET socket and connect to the specified server.
    // The return value are:
    // >SOCKERR_UNSPEC  -- successfully connected socket
    // SOCKERR_UNSPEC   -- unspecified error, may be fatal, prints perror()
    // SOCKERR_TIMEOUT  -- timeout condition was reached
    //--------------------------------------------------------------------------
  ){
    
    fprintf(stderr, "Hi there from socketClientInit() with timeout %g\n", timeout);

#ifdef ESMF_NO_SOCKETS
    fprintf(stderr, "ESMF was built with ESMF_NO_SOCKETS\n");
    return SOCKERR_UNSPEC;
#else

    // construct the server address
    struct hostent *server = gethostbyname(serverName);
    if (server == NULL){
      perror("socketClientInit: gethostbyname()");
      return SOCKERR_UNSPEC;  // bail out
    }
    struct sockaddr_in name;
    name.sin_family = AF_INET;
    name.sin_port = htons(port);
    name.sin_addr = *(struct in_addr *)(server->h_addr_list[0]);

    // create an inet/stream socket
    int sock = socket(PF_INET, SOCK_STREAM, 0);
    if (sock < 0){
      perror("socketClientInit: socket()");
      return SOCKERR_UNSPEC;  // bail out
    }
    
    // make socket non-blocking
#if !defined (ESMF_OS_MinGW)
    int sockFlags = fcntl(sock, F_GETFL, 0);
    fcntl(sock, F_SETFL, sockFlags | O_NONBLOCK); // Set non-blocking flag
#else
    unsigned long nbio_on = 1;
    ioctlsocket (sock, FIONBIO, &nbio_on); // Set non-blocking flag
#endif

    // start timing
    double t0, t1;
    VMK::wtime(&t0);
    VMK::wtime(&t1);
    
    // timeout loop
    bool connected = false;
    while ((t1-t0) < timeout){
      // try to open connection 
      if (connect(sock, (struct sockaddr *) &name, sizeof(name)) < 0){
        // connection has not (yet) been established
        perror("socketClientInit connect(), not yet established... continue");
        if (errno==ECONNABORTED){
          // on some systems, e.g. linux, repeated call to connect may give this
          perror("socketClientInit connect(), but continue");
          VMK::wtime(&t1);  // update the endtime
          continue;         // next attempt
        }
        // check for unexpected error conditions and bail
        if (errno!=EINPROGRESS && errno!=EALREADY && errno!=EWOULDBLOCK
          && errno!=ECONNREFUSED){
          perror("socketClientInit: connect(), bailing");
          return SOCKERR_UNSPEC;  // bail out
        }
        bool refusedFlag = false; // initialize
        if (errno==ECONNREFUSED)
          refusedFlag = true;
        // wait in select for socket to become writable
        fd_set sendfds;
        FD_ZERO(&sendfds);
        FD_SET(sock, &sendfds);
        struct timeval timev = {1, 0};  // 1s wait time in select
        if (select(FD_SETSIZE, NULL, &sendfds, NULL, &timev) < 0){
          perror("socketClientInit: select(), bailing");
          return SOCKERR_UNSPEC;  // bail out
        }
        if (FD_ISSET(sock, &sendfds)){
          // socket is now indicated as writable, still could be success or not
          // look at SO_ERROR to determine success or failure to connect
          int error;
          socklen_t len = sizeof(error);
          if (getsockopt(sock, SOL_SOCKET, SO_ERROR, (value_ptr_t)&error, &len)
            < 0){
            perror("socketClientInit: getsockopt(), bailing");
            return SOCKERR_UNSPEC;  // bail out
          }
          VMK::wtime(&t1);
          fprintf(stderr, "socketClientInit: getsockopt() at %g SO_ERROR: "
            "%s\n", t1-t0, strerror(error));
          if (error==0 && !refusedFlag){
            // successful connection was made
            connected = true;
            break;
          }else if (error==0 && refusedFlag){
            // this happens on IBM, where getsockopt() doesn't return error
            // but the sock variable has become invalid due to failed connect()
            sock = socket(PF_INET, SOCK_STREAM, 0);
          }else if (error==ECONNREFUSED){
            // this happens on Darwin, and it requires that the sock variable
            // is re-initialized (just as in the IBM case above) - not doing 
            // this will lead to EINVAL in the next connect() attempt
            sock = socket(PF_INET, SOCK_STREAM, 0);
          }else if (error && error!=ECONNREFUSED){
            // bail if this wasn't just a straight refusal due to absent server
            fprintf(stderr, "socketClientInit: getsockopt() error and bail: "
              "%s\n", strerror(error));
            return SOCKERR_UNSPEC;  // bail out
          }
        }else{
          fprintf(stderr, "socketClientInit: select: TIMEOUT!\n");
        }
      }else{
        // connection was established right away
        connected = true;
        break;
      }
      // update the endtime
      VMK::wtimedelay(1.);  // 1s delay to lower load on network
      VMK::wtime(&t1);
    }
    // reset socket to be blocking
#if !defined (ESMF_OS_MinGW)
    sockFlags = fcntl(sock, F_GETFL, 0);
    fcntl(sock, F_SETFL, sockFlags & (~O_NONBLOCK));
#else
    unsigned long nbio_off = 0;
    ioctlsocket (sock, FIONBIO, &nbio_off);
#endif
    
    fprintf(stderr, "socketClientInit waited: %g\n", t1-t0);    

    if (!connected){
      fprintf(stderr, "socketClientInit: TIMEOUT!\n");
#if !defined (ESMF_OS_MinGW)
      close(sock);
#else
      closesocket (sock);
#endif
      return SOCKERR_TIMEOUT;
    }
    
    // return successfully
    fprintf(stderr, "socketClientInit: CONNECTED!\n");
    return sock;  // return the connected socket
#endif
  }

  // ------------------------------------------------------------------------

  int socketFinal(
    int sock,         // connected socket to be finalized
    double timeout    // timeout in seconds
    //--------------------------------------------------------------------------
    // Attempt to cleanly take down a socket connection.
    // The return value are:
    // 0                -- successfully disconnect hand-shake
    // SOCKERR_UNSPEC   -- unspecified error, may be fatal, prints perror()
    // SOCKERR_TIMEOUT  -- timeout condition was reached
    //--------------------------------------------------------------------------
  ){

    fprintf(stderr, "Hi there from socketFinal()\n");

#ifdef ESMF_NO_SOCKETS
    fprintf(stderr, "ESMF was built with ESMF_NO_SOCKETS\n");
    return SOCKERR_UNSPEC;
#else

    int const bufferSize = 1024;
    char buffer[bufferSize];
    if (shutdown(sock, 1) < 0){ // send EOF to other side
      perror("socketFinal: shutdown()");
      return SOCKERR_UNSPEC;  // bail out
    }
    fd_set recvfds;
    struct timeval timev = {1, 0};  // 1s wait time in select
    int len;
    
    // start timing
    double t0, t1;
    VMK::wtime(&t0);
    VMK::wtime(&t1);
    
    // timeout loop
    while ((t1-t0) < timeout){
    
      FD_ZERO(&recvfds);
      FD_SET(sock, &recvfds);
      if (select(FD_SETSIZE, &recvfds, NULL, NULL, &timev) < 0){
        perror("socketFinal: select()");
        return SOCKERR_UNSPEC;  // bail out
      }
      if (FD_ISSET(sock, &recvfds)){
        if ((len=recv(sock, buffer, bufferSize, 0)) < 0){
          perror("socketFinal: recv()");
          return SOCKERR_UNSPEC;  // bail out
        }
        if (len==0) break;  // received EOF from other side
        fprintf(stderr, "after shutdown received: len=%d\n", len);
      }
      // update the endtime
      VMK::wtime(&t1);
    }
   
    fprintf(stderr, "socketFinal waited: %g\n", t1-t0);    

    // return timeout condition
    if ((t1-t0) >= timeout){
      // complete shutdown
      if (shutdown(sock, 2) < 0){ // send EOF to other side
        perror("socketFinal: shutdown()");
        return SOCKERR_UNSPEC;  // bail out
      }
      return SOCKERR_TIMEOUT; // bail out
    }
    

    // close the socket
#if !defined (ESMF_OS_MinGW)
    if (close(sock) < 0){
#else
    if (closesocket (sock) < 0) {
#endif
      perror("socketFinal: close()");
      return SOCKERR_UNSPEC;  // bail out
    }

    // return successfully
    return 0;
#endif
  }
    
  // ------------------------------------------------------------------------

  int socketSend(
    int sock,             // socket holding the connection
    void const *buffer,   // data buffer
    size_t size,          // number of bytes to send out of buffer
    double timeout        // timeout in seconds
    //--------------------------------------------------------------------------
    // Attempt to send data through a socket connection.
    // The return value are:
    // >SOCKERR_UNSPEC  -- number of bytes sent
    // SOCKERR_UNSPEC   -- unspecified error, may be fatal, prints perror()
    // SOCKERR_TIMEOUT  -- timeout condition was reached
    //--------------------------------------------------------------------------
  ){

    fprintf(stderr, "Hi there from socketSend()\n");
    
#ifdef ESMF_NO_SOCKETS
    fprintf(stderr, "ESMF was built with ESMF_NO_SOCKETS\n");
    return SOCKERR_UNSPEC;
#else

    int waitSeconds = timeout;
    int waitMicro   = (timeout - waitSeconds)*1000000;
    
    fd_set sendfds;
    FD_ZERO(&sendfds);
    FD_SET(sock, &sendfds);
    struct timeval timev = {waitSeconds, waitMicro};
    
    int len;
    
    if (select(FD_SETSIZE, NULL, &sendfds, NULL, &timev) < 0){
      perror("socketSend: select()");
      return SOCKERR_UNSPEC;  // bail out
    }
    if (FD_ISSET(sock, &sendfds)){
      if ((len=send(sock, (value_ptr_t)buffer, size, 0)) < 0){
        perror("socketSend: send()");
        return SOCKERR_UNSPEC;  // bail out
      }
      if ((unsigned)len!=size){
        fprintf(stderr, "socketSend: incorrect number of bytes sent!\n");
        return SOCKERR_UNSPEC;  // bail out
      }
    }else{
      fprintf(stderr, "socketSend: select() TIMEOUT!\n");
      return SOCKERR_TIMEOUT;
    }
    
    fprintf(stderr, "socketSend: buffer size=%lu, bytes sent=%d\n", size, len);
    
    // return successfully
    return len;
#endif
  }
    
  // ------------------------------------------------------------------------

  int socketRecv(
    int sock,             // socket holding the connection
    void *buffer,         // data buffer
    size_t size,          // size of buffer in bytes
    double timeout        // timeout in seconds
    //--------------------------------------------------------------------------
    // Attempt to receive data through a socket connection.
    // The return value are:
    // >0                 -- number of bytes received
    // SOCKERR_UNSPEC     -- unspecified error, may be fatal, prints perror()
    // SOCKERR_TIMEOUT    -- timeout condition was reached
    // SOCKERR_DISCONNECT -- the other side has disconnected
    //--------------------------------------------------------------------------
  ){

    fprintf(stderr, "Hi there from socketRecv()\n");
    
#ifdef ESMF_NO_SOCKETS
    fprintf(stderr, "ESMF was built with ESMF_NO_SOCKETS\n");
    return SOCKERR_UNSPEC;
#else

    int waitSeconds = timeout;
    int waitMicro   = (timeout - waitSeconds)*1000000;
    
    // attempted recv with a timeout based on select
    fd_set recvfds;
    FD_ZERO(&recvfds);
    FD_SET(sock, &recvfds);
    struct timeval timev = {waitSeconds, waitMicro};

    int len;
    
    if (select(FD_SETSIZE, &recvfds, NULL, NULL, &timev) < 0){
      perror("socketRecv: select()");
      return SOCKERR_UNSPEC;  // bail out
    }
    if (FD_ISSET(sock, &recvfds)){
      if ((len=recv(sock, (value_ptr_t)buffer, size, 0)) < 0){
        perror("socketRecv: recv()");
        return SOCKERR_UNSPEC;  // bail out
      }
      if (len==0){
        perror("socketRecv: recv()");
        return SOCKERR_DISCONNECT;  // bail out
      }
    }else{
      fprintf(stderr, "socketRecv: select() TIMEOUT!\n");
      return SOCKERR_TIMEOUT;
    }
    
    fprintf(stderr, "socketRecv: buffer size=%lu, bytes recvd=%d\n", size, len);

    // return successfully
    return len;
#endif
  }
    
  // ------------------------------------------------------------------------
  // ------------------------------------------------------------------------
  // The following two calls are just code to help prototype the above
  // socket based methods. They are called from two independent MPI apps.
  // ------------------------------------------------------------------------
  // ------------------------------------------------------------------------

#define MORE_EXHAUSTIVE_SOCK_TESTING___disable
  
  int socketServer(void){
    
    fprintf(stderr, "Hi there from socketServer()\n");
    
#ifdef ESMF_NO_SOCKETS
    fprintf(stderr, "ESMF was built with ESMF_NO_SOCKETS\n");
    return SOCKERR_UNSPEC;
#else

    // attempt to open and connect a server socket
    int sock = socketServerInit(PORT, 60.);
    if (sock <= SOCKERR_UNSPEC){
      fprintf(stderr, "socketServer: socketServerInit() no client connected\n");
      return 0;   // bail out, but don't indicate abort
    }
    
    // prepare buffer for following tests    
    int const bufferSize = 256;
    char buffer[bufferSize];
    int len;

    // simple ping-pong test using blocking send/recv
    sprintf(buffer, "Hi, this is the PING message!");
    if (send(sock, buffer, strlen(buffer)+1, 0) < 0){
      perror("server: send()");
      return 0;   // bail out, but don't indicate abort
    }
    VMK::wtimedelay(3);  // delay the receive for 3s
    if ((len=recv(sock, buffer, bufferSize, 0)) < 0){
      perror("server: recv()");
      return 0;   // bail out, but don't indicate abort
    }
    fprintf(stderr, "server received: len=%d :: %s\n", len, buffer);
    
    // attempt a clean disconnect
    if (socketFinal(sock, 20.) <= SOCKERR_UNSPEC){
      fprintf(stderr, "socketServer: socketFinal() handshake failed\n");
      return 0;   // bail out, but don't indicate abort
    }
    fprintf(stderr, "server back from clean disconnect\n");
    
#ifdef MORE_EXHAUSTIVE_SOCK_TESTING
    VMK::wtimedelay(20.);
    
    // attempt to re-open and connect a server socket (while client is gone)
    sock = socketServerInit(PORT, 5.);
    if (sock <= SOCKERR_UNSPEC){
      fprintf(stderr, "socketServer: socketServerInit() failed"
        " - expected\n");
    }else{
      fprintf(stderr, "socketServer: socketServerInit() unexpected connect\n");
      return 0;   // bail out, but don't indicate abort
    }
    
    // attempt to re-open and connect a server socket (with client now there)
    sock = socketServerInit(PORT, 10.);
    if (sock <= SOCKERR_UNSPEC){
      fprintf(stderr, "socketServer: socketServerInit() failed\n");
      return 0;  // bail out, but don't indicate abort
    }
    
    // attempted send with timeout based on select
    sprintf(buffer, "Hi, this is the SELECT message!");
    if (socketSend(sock, buffer, strlen(buffer)+1, 10.) <= SOCKERR_UNSPEC){
      fprintf(stderr, "socketServer: socketSend() failed\n");
      return 0;  // bail out, but don't indicate abort
    }
        
    // attempt a clean disconnect
    if (socketFinal(sock, 2.) <= SOCKERR_UNSPEC){
      fprintf(stderr, "socketServer: socketFinal() handshake failed"
        " - expected\n");
    }else{
      fprintf(stderr, "socketServer: socketFinal() unexpected handshake\n");
      return 0;  // bail out, but don't indicate abort
    }
    
    // attempt to re-open and connect a server socket
    sock = socketServerInit(PORT, 30.);
    if (sock <= SOCKERR_UNSPEC){
      fprintf(stderr, "socketServer: socketServerInit() failed\n");
      return 0;  // bail out, but don't indicate abort
    }
    
    // send an integer back and forth in a fault tolerant way
    int data;
    int i;
    for (i=0; i<10; i++){
      if (socketRecv(sock, &data, sizeof(data), 1.) <= SOCKERR_UNSPEC){
        fprintf(stderr, "socketServer: socketRecv() failed - break loop\n");
        break;
      }
      fprintf(stderr, "socketServer: recv'd data = %d\n", data);
      ++data; // server increments the integer
      if (socketSend(sock, &data, sizeof(data), 1.) <= SOCKERR_UNSPEC){
        fprintf(stderr, "socketServer: socketSend() failed - break loop\n");
        break;
      }
      fprintf(stderr, "socketServer:   sent data = %d\n", data);

      // at iteration 5 take a long break which the other side will time out on
      if (i==5){
        fprintf(stderr, "socketServer: taking a 4s delay now...\n");
        VMK::wtimedelay(4.);
      }
    }
    
    if (i!=10){
      // the previous comm loop was interrupted due to connection issues
      if (socketFinal(sock, 5.) <= SOCKERR_UNSPEC){
        fprintf(stderr, "socketServer: socketFinal() handshake failed\n");
      }
      // attempt to re-open and connect a server socket
      sock = socketServerInit(PORT, 30.);
      if (sock <= SOCKERR_UNSPEC){
        fprintf(stderr, "socketServer: socketServerInit() failed\n");
      }
    }
    
    // again send an integer back and forth in a fault tolerant way
    for (i=0; i<10; i++){
      if (socketRecv(sock, &data, sizeof(data), 1.) <= SOCKERR_UNSPEC){
        fprintf(stderr, "socketServer: socketRecv() failed - break loop\n");
        break;
      }
      fprintf(stderr, "socketServer: recv'd data = %d\n", data);
      ++data; // server increments the integer
      if (socketSend(sock, &data, sizeof(data), 1.) <= SOCKERR_UNSPEC){
        fprintf(stderr, "socketServer: socketSend() failed - break loop\n");
        break;
      }
      fprintf(stderr, "socketServer:   sent data = %d\n", data);

      // at iteration 5 a catastrophic event happens - division by zero CRASH
      if (i==5){
        fprintf(stderr, "socketServer: oh no, divide by zero CRASH...\n");
        int b = 53/0; // trigger failure
      }
    }
#endif
    
    // return successfully
    return 0;
#endif
  }
  
  // ------------------------------------------------------------------------

  int socketClient(void){
    
    fprintf(stderr, "Hi there from socketClient()\n");
    
#ifdef ESMF_NO_SOCKETS
    fprintf(stderr, "ESMF was built with ESMF_NO_SOCKETS\n");
    return SOCKERR_UNSPEC;
#else

    FILE *fp = fopen(SERVERFILE, "r");
    if (fp == NULL){
      fprintf(stderr, "socketClient: failed opening SERVERFILE\n");
      return 0;   // bail out, but don't indicate abort
    }
    
    char serverName[80];
    fscanf(fp, "%s", serverName);
    fclose(fp);
    
    fprintf(stderr, "socketClient: connecting to server: %s\n", serverName);
    
    // attempt to open an connect a client socket
    int sock = socketClientInit(serverName, PORT, 60.);
    if (sock <= SOCKERR_UNSPEC){
      fprintf(stderr, "socketClient: socketClientInit() failed to connect\n");
      return 0;  // bail out, but don't indicate abort
    }
    
    // prepare buffer for following tests    
    int const bufferSize = 256;
    char buffer[bufferSize];
    int len;

    // simple ping-pong test using blocking send/recv
    if ((len=recv(sock, buffer, bufferSize, 0)) < 0){
      perror("client: recv()");
      return 0;   // bail out, but don't indicate abort
    }
    fprintf(stderr, "client received: len=%d :: %s\n", len, buffer);
    sprintf(buffer, "Hi, this is the PONG message!");
    if (send(sock, buffer, strlen(buffer)+1, 0) < 0){
      perror("client: send()");
      return 0;   // bail out, but don't indicate abort
    }

    // attempt a clean disconnect
    if (socketFinal(sock, 20.) <= SOCKERR_UNSPEC){
      fprintf(stderr, "socketClient: socketFinal() handshake failed\n");
      return 0;  // bail out, but don't indicate abort
    }
    fprintf(stderr, "client back from clean disconnect\n");

#ifdef MORE_EXHAUSTIVE_SOCK_TESTING
    // attempt to reconnect with server (while it isn't up)
    sock = socketClientInit(serverName, PORT, 10.);
    if (sock <= SOCKERR_UNSPEC){
      fprintf(stderr, "socketClient: socketClientInit() failed to connect"
        " - expected\n");
    }else{
      fprintf(stderr, "socketClient: socketClientInit() unexpected connect\n");
      return 0;  // bail out, but don't indicate abort
    }
    
    VMK::wtimedelay(20.);
    
    // attempt to reconnect with server (now it should be there)
    sock = socketClientInit(serverName, PORT, 10.);
    if (sock <= SOCKERR_UNSPEC){
      fprintf(stderr, "socketClient: socketClientInit() failed to connect\n");
      return 0;  // bail out, but don't indicate abort
    }
    
    // attempted recv with a timeout based on select
    if ((len=socketRecv(sock, buffer, bufferSize, 10.)) <= SOCKERR_UNSPEC){
      fprintf(stderr, "socketClient: socketRecv() failed\n");
      return 0;  // bail out, but don't indicate abort
    }
    fprintf(stderr, "socketClient: received %d bytes: %s\n", len, buffer);
    
    VMK::wtimedelay(10.);
    
    // attempt a clean disconnect
    if (socketFinal(sock, 2.) <= SOCKERR_UNSPEC){
      fprintf(stderr, "socketClient: socketFinal() handshake failed\n");
      return 0;  // bail out, but don't indicate abort
    }
    
    // attempt to reconnect with server
    sock = socketClientInit(serverName, PORT, 20.);
    if (sock <= SOCKERR_UNSPEC){
      fprintf(stderr, "socketClient: socketClientInit() failed to connect\n");
      return 0;  // bail out, but don't indicate abort
    }
    
    // send an integer back and forth in a fault tolerant way
    int data = 0; // initialize
    int i;
    for (i=0; i<10; i++){
      if (socketSend(sock, &data, sizeof(data), 1.) <= SOCKERR_UNSPEC){
        fprintf(stderr, "socketClient: socketSend() failed - break loop\n");
        break;
      }
      fprintf(stderr, "socketClient:   sent data = %d\n", data);
      if (socketRecv(sock, &data, sizeof(data), 1.) <= SOCKERR_UNSPEC){
        fprintf(stderr, "socketClient: socketRecv() failed - break loop\n");
        break;
      }
      fprintf(stderr, "socketClient: recv'd data = %d\n", data);
    }
    
    if (i!=10){
      // the previous comm loop was interrupted due to connection issues
      if (socketFinal(sock, 5.) <= SOCKERR_UNSPEC){
        fprintf(stderr, "socketClient: socketFinal() handshake failed\n");
      }
      // attempt to reconnect with server
      sock = socketClientInit(serverName, PORT, 20.);
      if (sock <= SOCKERR_UNSPEC){
        fprintf(stderr, "socketClient: socketClientInit() failed to connect\n");
      }
    }
    
    // again send an integer back and forth in a fault tolerant way
    data = 0; // initialize
    for (i=0; i<10; i++){
      if (socketSend(sock, &data, sizeof(data), 1.) <= SOCKERR_UNSPEC){
        fprintf(stderr, "socketClient: socketSend() failed - break loop\n");
        break;
      }
      fprintf(stderr, "socketClient:   sent data = %d\n", data);
      if (socketRecv(sock, &data, sizeof(data), 1.) <= SOCKERR_UNSPEC){
        fprintf(stderr, "socketClient: socketRecv() failed - break loop\n");
        break;
      }
      fprintf(stderr, "socketClient: recv'd data = %d\n", data);
    }
    
    if (i!=10){
      // the previous comm loop was interrupted due to connection issues
      if (socketFinal(sock, 5.) <= SOCKERR_UNSPEC){
        fprintf(stderr, "socketClient: socketFinal() handshake failed\n");
      }
    }
#endif
    
    // return successfully
    return 0;
#endif
  }
  
} // namespace ESMCI
//==============================================================================
