// $Id$
//
// Earth System Modeling Framework
// Copyright (c) 2002-2025, University Corporation for Atmospheric Research, 
// Massachusetts Institute of Technology, Geophysical Fluid Dynamics 
// Laboratory, University of Michigan, National Centers for Environmental 
// Prediction, Los Alamos National Laboratory, Argonne National Laboratory, 
// NASA Goddard Space Flight Center.
// Licensed under the University of Illinois-NCSA License.
//
//==============================================================================
// convert a 2D grid file (either structured or unstructured) in SCRIP format
// NetCDF file into a ESMF unstructured file format or a UGRID file format.
// A dual mesh is generated by mapping the center coordinates into nodes if
// the dualflag is set to 1.

#include <stdlib.h>
#include <stdio.h>
#include <math.h>
#include <time.h>
#include <string.h>
#include <ctype.h>

#ifndef MPICH_IGNORE_CXX_SEEK
#define MPICH_IGNORE_CXX_SEEK
#endif

#include "ESMC.h"
#include "Mesh/include/ESMCI_ClumpPnts.h"
#include "mpi.h"

#ifdef ESMF_NETCDF
#include "netcdf.h"
#endif

#if !defined (M_PI)
// for Windows...
#define M_PI 3.14159265358979323846
#endif

#define TOL 0.0000000001

// defined in ESMC_IOScrip2ESMF.C
// will be put in a header file for consistency later
bool handle_error(int status, int lineno);
void orderit(int index, double lon, double lat, int numedges, double *latlonbuf, int *next);
void convert3D(double lon, double lat, double *x, double *y);
void orderit2(int index, double lon, double lat, int numedges, double *latlonbuf, int *next);

int print_usage() {
      /* standard --help argument was specified */
      printf("ESMF_Scrip2Unstruct: Convert an unstructured grid file in SCRIP format into either a ESMF unstructured file or a UGRID file format.\n");
      printf("Usage: ESMF_Scrip2Unstruct [--help] [--version] [-V] inputfile outputfile dualflag [fileformat]\n");
      printf("    [--help]        Display this information and exit.\n");
      printf("    [--version]     Display ESMF version and license information "
        "and exit.\n");
      printf("    [-V]            Display ESMF version string and exit.\n");
      printf("    inputfile       input grid filename \n");
      printf("    outputfile      output filename \n");
      printf("    dualflag        1 to generate a dual mesh, 0 for non-dual mesh\n");
      printf("    [fileformat]    Either ESMF or UGRID, the default is ESMF\n");
      printf("\n");
      return 0;
}

// Create a ESMF unstructured grid file and define all the dimension, variables and attributes
int create_esmf(char* filename, char* infilename, int dualflag, size_t nnodes, size_t nelmts, size_t maxconnection, 
                int nocenter, int nomask, int noarea, 
                int orig_grid_rank, int *orig_grid_dims)
{
  int ncid1, ncid2;
  int vertdimid, celldimid, vpcdimid,vdimid;
  int vertexid, edgeid, ccoordid, cellid, caid, cmid;
  int ogr_dimid, ogd_id, areaid, maskid, attid;
  time_t tloc;
  int dims[2];
  int status, fillvalue;
  const char* strbuf;
  char * strbuf1;

#ifdef ESMF_NETCDF
  // create the output ESMF netcdf file
#ifdef NC_NETCDF4
  status = nc_create(filename, NC_CLOBBER|NC_NETCDF4, &ncid2);
  if (status == NC_ENOTNC) {
    status = nc_create(filename, NC_CLOBBER, &ncid2);
  } 
  if (status != NC_NOERR) handle_error(status,__LINE__);
#else
  status = nc_create(filename, NC_CLOBBER, &ncid2);
  if (status != NC_NOERR) handle_error(status,__LINE__);
#endif

  // Open the input file and get the long_name and units attribute from grid_area
  status = nc_open(infilename, NC_NOWRITE, &ncid1);  
  if (status != NC_NOERR) handle_error(status,__LINE__);
  
  // define the dimensions
  status = nc_def_dim(ncid2, "nodeCount", nnodes, &vertdimid);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  status = nc_def_dim(ncid2, "elementCount", nelmts, &celldimid);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  status = nc_def_dim(ncid2, "maxNodePElement", maxconnection, & vpcdimid);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  status = nc_def_dim(ncid2, "coordDim", 2L, &vdimid);
  if (status != NC_NOERR) handle_error(status,__LINE__);

  // Only add if originally 2D and not-dual 
  if ((orig_grid_rank == 2) && (dualflag == 0))  {
    status = nc_def_dim(ncid2, "origGridRank", orig_grid_rank, &ogr_dimid);
    if (status != NC_NOERR) handle_error(status,__LINE__);
    status = nc_def_var(ncid2,"origGridDims", NC_INT, 1, &ogr_dimid, &ogd_id);
    if (status != NC_NOERR) handle_error(status,__LINE__);
  }
  
  // define the variables
  dims[0]=vertdimid;
  dims[1]=vdimid;
  status = nc_def_var(ncid2,"nodeCoords", NC_DOUBLE, 2, dims, &vertexid);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  strbuf = "degrees";
  status = nc_put_att_text(ncid2, vertexid, "units", strlen(strbuf)+1, strbuf);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  dims[0]=celldimid;
  dims[1]=vpcdimid;
  status = nc_def_var(ncid2,"elementConn", NC_INT, 2, dims, &cellid);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  strbuf = "Node indices that define the element connectivity";
  status = nc_put_att_text(ncid2, cellid, "long_name", strlen(strbuf)+1, strbuf);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  fillvalue = -1;
  status = nc_put_att_int(ncid2, cellid, "_FillValue", NC_INT, 1, &fillvalue);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  status = nc_def_var(ncid2,"numElementConn", NC_INT, 1, dims, &edgeid);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  strbuf = "Number of nodes per element";
  status = nc_put_att_text(ncid2, edgeid, "long_name", strlen(strbuf)+1, strbuf);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  if (!nocenter) {
    dims[0]=celldimid;
    dims[1]=vdimid;
    status = nc_def_var(ncid2, "centerCoords", NC_DOUBLE, 2, dims, &ccoordid);
    if (status != NC_NOERR) handle_error(status,__LINE__);
    strbuf = "degrees";
    status = nc_put_att_text(ncid2, ccoordid, "units", strlen(strbuf)+1, strbuf);
    if (status != NC_NOERR) handle_error(status,__LINE__);
  }
  if (!noarea) {
    status = nc_def_var(ncid2, "elementArea", NC_DOUBLE, 1, dims, &caid);
    if (status != NC_NOERR) handle_error(status,__LINE__);
    // copy the units and long_name attributes if they exist in the input file
    status = nc_inq_varid(ncid1, "grid_area", &areaid);
    if (status != NC_NOERR) handle_error(status,__LINE__);
    status = nc_inq_attid(ncid1, areaid, "units", &attid);
    if (status == NC_NOERR) {
      status = nc_copy_att(ncid1, areaid, "units", ncid2, caid);
      if (status != NC_NOERR) handle_error(status,__LINE__);
    }    
    status = nc_inq_attid(ncid1, areaid, "_FillValue", &attid);
    if (status == NC_NOERR) {
      status = nc_copy_att(ncid1, areaid, "_FillValue", ncid2, caid);
      if (status != NC_NOERR) handle_error(status,__LINE__);
    }    
    status = nc_inq_attid(ncid1, areaid, "long_name", &attid);
    if (status == NC_NOERR) {
       status = nc_copy_att(ncid1, areaid, "long_name", ncid2, caid);
       if (status != NC_NOERR) handle_error(status,__LINE__);
    }
  }
  if (!nomask && dualflag==0) {
    status = nc_def_var(ncid2, "elementMask", NC_INT, 1, dims, &cmid);
    if (status != NC_NOERR) handle_error(status,__LINE__);
    status = nc_inq_varid(ncid1, "grid_imask", &maskid);
    if (status != NC_NOERR) handle_error(status,__LINE__);
    status = nc_inq_attid(ncid1, maskid, "units", &attid);
    if (status == NC_NOERR) {
      status = nc_copy_att(ncid1, maskid, "units", ncid2, cmid);
      if (status != NC_NOERR) handle_error(status,__LINE__);
    }    
  }
  if (!nomask && dualflag==1) {
    dims[0]=vertdimid;
    status = nc_def_var(ncid2, "nodeMask", NC_INT, 1, dims, &cmid);
    if (status != NC_NOERR) handle_error(status,__LINE__);
    status = nc_inq_varid(ncid1, "grid_imask", &maskid);
    if (status != NC_NOERR) handle_error(status,__LINE__);
    status = nc_inq_attid(ncid1, maskid, "units", &attid);
    if (status == NC_NOERR) {
      status = nc_copy_att(ncid1, maskid, "units", ncid2, cmid);
      if (status != NC_NOERR) handle_error(status,__LINE__);
    }    
  }
  
  // Global Attribute
  if (dualflag == 1) 
    strbuf = "unstructured dual mesh";
  else
    strbuf = "unstructured mesh";
  status = nc_put_att_text(ncid2, NC_GLOBAL, "gridType", strlen(strbuf), strbuf);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  strbuf = "0.9";
  status = nc_put_att_text(ncid2, NC_GLOBAL, "version", strlen(strbuf), strbuf);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  status = nc_put_att_text(ncid2, NC_GLOBAL, "inputFile", strlen(infilename), infilename);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  time(&tloc);
  strbuf1 = ctime(&tloc);
  strbuf1[strlen(strbuf1)-1] = '\0';
  status = nc_put_att_text(ncid2, NC_GLOBAL, "timeGenerated", strlen(strbuf1), strbuf1);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  
  status=nc_close(ncid2);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  status = nc_close(ncid1);
  if (status != NC_NOERR) handle_error(status,__LINE__);
#endif
  return 1;
}

// Create a UGRID file and define all the dimension, variables and attributes
int create_ugrid(char* filename, char* infilename, int dualflag, size_t nnodes, size_t nfaces, size_t maxconnection, int nocenter)
{
  int ncid2;
  int vertdimid, celldimid, vpcdimid;
  int varid;
  int dims[2];
  time_t tloc;
  int status, var;
  const char *strbuf;
  char *strbuf1;

#ifdef ESMF_NETCDF
#ifdef NC_NETCDF4
  status = nc_create(filename, NC_CLOBBER|NC_NETCDF4, &ncid2);
  if (status == NC_ENOTNC) {
    status = nc_create(filename, NC_CLOBBER, &ncid2);
  } 
  if (status != NC_NOERR) handle_error(status,__LINE__);
#else
  status = nc_create(filename, NC_CLOBBER, &ncid2);
  if (status != NC_NOERR) handle_error(status,__LINE__);
#endif

  status = nc_def_dim(ncid2, "nodeCount", nnodes, &vertdimid);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  status = nc_def_dim(ncid2, "faceCount", nfaces, &celldimid);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  status = nc_def_dim(ncid2, "maxNodePerFace", maxconnection, &vpcdimid);
  if (status != NC_NOERR) handle_error(status,__LINE__);
	
  // create node coordinates
  status = nc_def_var(ncid2, "node_x", NC_DOUBLE, 1, &vertdimid, &varid);
  if (status != NC_NOERR) handle_error(status,__LINE__);
	
  // add attribute
  strbuf = "node longitude";
  status = nc_put_att_text(ncid2, varid, "standard_name", strlen(strbuf)+1, strbuf);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  strbuf =  "degrees_east";
  status = nc_put_att_text(ncid2, varid, "units", strlen(strbuf)+1, strbuf);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  status = nc_def_var(ncid2, "node_y", NC_DOUBLE, 1, &vertdimid, &var);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  
  // add attribute
  strbuf = "node latitude";
  status = nc_put_att_text(ncid2, var, "standard_name", strlen(strbuf)+1, strbuf);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  strbuf =  "degrees_north";
  status = nc_put_att_text(ncid2, var, "units", strlen(strbuf)+1, strbuf);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  
  // create face coordinates
  if (!nocenter) {
    status = nc_def_var(ncid2, "face_x", NC_DOUBLE, 1, &celldimid, &varid);
    if (status != NC_NOERR) handle_error(status,__LINE__);
  
    // add attribute
    strbuf = "center longitude";
    status = nc_put_att_text(ncid2, varid, "standard_name", strlen(strbuf)+1, strbuf);
    if (status != NC_NOERR) handle_error(status,__LINE__);
    strbuf = "degrees_east";
    status = nc_put_att_text(ncid2, varid, "units", strlen(strbuf)+1, strbuf);
    if (status != NC_NOERR) handle_error(status,__LINE__);
    
    status = nc_def_var(ncid2, "face_y", NC_DOUBLE, 1, &celldimid, &varid);
    if (status != NC_NOERR) handle_error(status,__LINE__);
  
    // add attribute
    strbuf = "center latitude";
    status = nc_put_att_text(ncid2, varid, "standard_name", strlen(strbuf)+1, strbuf);
    if (status != NC_NOERR) handle_error(status,__LINE__);
    strbuf = "degrees_north";
    status = nc_put_att_text(ncid2, varid, "units", strlen(strbuf)+1, strbuf);
    if (status != NC_NOERR) handle_error(status,__LINE__);
  }

  // define face_node_connectivity variable
  dims[0]=celldimid;
  dims[1]=vpcdimid;
  
  status = nc_def_var(ncid2, "elementConn", NC_INT, 2, dims, &varid);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  
  // add attribute
  strbuf = "face_node_connectivity";
  status = nc_put_att_text(ncid2, varid, "standard_name", strlen(strbuf), strbuf);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  var = 1;
  status = nc_put_att_int(ncid2, varid, "start_index", NC_INT, 1, &var);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  var = -1;
  status = nc_put_att_int(ncid2, varid, "_FillValue", NC_INT, 1, &var);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  
  // Last, define the dummy variable
  status = nc_def_var(ncid2, "mesh", NC_INT, 0, 0, &varid);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  
  // add attributes
  strbuf = "mesh_topology";
  status = nc_put_att_text(ncid2, varid, "standard_name", strlen(strbuf), strbuf);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  status = nc_put_att_text(ncid2, varid, "cf_role", strlen(strbuf), strbuf);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  var = 2;
  // change dimension attribute to topology_dimension based on the v0.9.0 ugrid-convention
  status = nc_put_att_int(ncid2, varid, "topology_dimension", NC_INT, 1, &var);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  if (nocenter) 
    strbuf = "node";
  else
    strbuf = "face node";
  status = nc_put_att_text(ncid2, varid, "locations", strlen(strbuf), strbuf);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  strbuf = "node_x node_y";
  status = nc_put_att_text(ncid2, varid, "node_coordinates", strlen(strbuf), strbuf);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  if (!nocenter) {
    strbuf = "face_x face_y";
    status = nc_put_att_text(ncid2, varid, "face_coordinates", strlen(strbuf), strbuf);
    if (status != NC_NOERR) handle_error(status,__LINE__);
  }
  strbuf = "elementConn";
  status = nc_put_att_text(ncid2, varid, "face_node_connectivity", strlen(strbuf), strbuf);
  if (status != NC_NOERR) handle_error(status,__LINE__);

  // Global Attribute
  if (dualflag == 1) 
    strbuf = "unstructured dual mesh";
  else
    strbuf = "unstructured mesh";
  status = nc_put_att_text(ncid2, NC_GLOBAL, "gridType", strlen(strbuf), strbuf);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  strbuf = "0.9";
  status = nc_put_att_text(ncid2, NC_GLOBAL, "version", strlen(strbuf), strbuf);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  status = nc_put_att_text(ncid2, NC_GLOBAL, "inputFile", strlen(infilename), infilename);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  time(&tloc);
  strbuf1 = ctime(&tloc);
  strbuf1[strlen(strbuf)-1] = '\0';
  status = nc_put_att_text(ncid2, NC_GLOBAL, "timeGenerated", strlen(strbuf1), strbuf1);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  
  status=nc_close(ncid2);
  if (status != NC_NOERR) handle_error(status,__LINE__);
#endif
  return 1;
}

      
int main(int argc, char** argv)
{
  int ncid1, ncid2;
  int gsdimid, gcdimid, grdimid;
  size_t  gsdim, gcdim, grdim;
  int areaid, ctlatid, ctlonid, colatid, colonid, maskid;
  int status;
  int vertexid, cellid, edgeid, ccoordid, caid, cmid;
  int varid;
  double *cornerlats, *cornerlons, *nodelatlon;
  double *nodelats, *nodelons;
  double *inbuf, *inbuf1;
  int *inbuf2;
  int *dualcells;
  int *dualcellcounts;
  int *cells, temp[16];
  int numedges, *next;
  int *totalneighbors;
  int *edges;
  int i,i1, j, k, totalnodes, goodnodes, count;
  int *globalnodes;
  int noarea, nocenter, nomask;
  size_t starts[2], counts[2];
  int maxconnection;
  char *c_infile;
  char *c_outfile;
  char units[80];
  size_t len;
  double rad2deg = 180.0/M_PI;
  int dualflag;
  int ind, startind;
  double minlat, maxlat, startlat, endlat, part;
  int nprocs, myrank, npes;
  int *mycells;
  int alltotal, mypart, left, offset, mystart, mystartelement;
  size_t start1[1], count1[1], start2[2], count2[2];
  int doesmf = 1;
  double *coord1d;
  int totalsize;
  ESMC_VM vm;
  MPI_Comm mpi_comm;
  int argIndex;
  int argFlag;
  int pthreadflag, openmpflag;
  int rc;
#define MAX_GRID_RANK 2
  int grid_dims[MAX_GRID_RANK];
  int ogr_dimid, ogd_id;  

  ESMC_Initialize(&status, ESMC_ArgLast);
  vm = ESMC_VMGetGlobal(&status);
  status = ESMC_VMGet(vm, &myrank, &nprocs, &npes, &mpi_comm, &pthreadflag, &openmpflag);

  if (myrank == 0){
    argFlag = 0;
    int vFlag = 0;
    int versionFlag = 0;

    /* check for standard command line arguments */
    argIndex = ESMC_UtilGetArgIndex(argc, argv, "--help", &rc);
    if (argIndex >= 0){
      argFlag=1;
      print_usage();
    }
    argIndex = ESMC_UtilGetArgIndex(argc, argv, "--version", &rc);
    if (argIndex >= 0){
      argFlag=1;
      versionFlag = 1;
    }
    argIndex = ESMC_UtilGetArgIndex(argc, argv, "-V", &rc);
    if (argIndex >= 0){
      argFlag=1;
      vFlag = 1;
    }
    if (argFlag)
      ESMC_UtilVersionPrint (vFlag, versionFlag, &rc);
  }
  // broadcast the argIndex to all the processors
  MPI_Bcast(&argFlag, 1, MPI_INT, 0, mpi_comm);
  if (argFlag == 1) {
    ESMC_Finalize();
    exit(0);
  }
             
#ifdef ESMF_NETCDF
  if (argc < 4) {
    if (myrank == 0) {
      print_usage();
    }
    ESMC_Finalize();
    exit(1);
  }
  c_infile = argv[1];
  if (c_infile == NULL) {
    if (myrank == 0)
      fprintf(stderr, "SCRIP file name not valid \n");
    ESMC_Finalize();
    exit(1); // bail out
  }

  c_outfile=argv[2];
  if (c_outfile == NULL) {
    if (myrank == 0) 
      fprintf(stderr, "output file name from converter not valid \n");
    ESMC_Finalize();
    exit(1); // bail out
  }
  
  dualflag = atoi(argv[3]);
  // Open intput SCRIP file
  status = nc_open(c_infile, NC_NOWRITE, &ncid1);  
  if (status != NC_NOERR) handle_error(status,__LINE__);

  doesmf = 1;
  if (argc > 4) {
    if (strcmp(argv[4],"ESMF")==0 || strcmp(argv[4],"esmf")==0) {
      doesmf = 1;
    } else if (strcmp(argv[4],"UGRID")==0 || strcmp(argv[4],"ugrid")==0) {
      doesmf = 0;
    } else {
      if (myrank == 0) 
	fprintf(stderr, "File format %s is not supported, use ESMF or UGRID\n", argv[4]);
      ESMC_Finalize();
      exit(1);
    }
  }
  // inquire dimension ids
  status = nc_inq_dimid(ncid1, "grid_size", &gsdimid);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  status = nc_inq_dimid(ncid1, "grid_corners", &gcdimid);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  status = nc_inq_dimid(ncid1, "grid_rank", &grdimid);
  if (status != NC_NOERR) handle_error(status,__LINE__);

  // Get dimension values
  status = nc_inq_dimlen(ncid1, gsdimid, &gsdim);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  status = nc_inq_dimlen(ncid1, gcdimid, &gcdim);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  status = nc_inq_dimlen(ncid1, grdimid, &grdim);
  if (status != NC_NOERR) handle_error(status,__LINE__);


  // comment out the following check -- support the SCRIP file with grid_rank==1 and 2
  //  if (grdim > 1) {
  //  fprintf(stderr, "%s: grid_rank is greater than 1.  This program only convert grids with grid_rank=1.\n",c_infile);
  //  ESMC_Finalize();
  //  exit(1); // bail out
  //}

  // Add check back in, but for larger grid rank
  if (grdim > MAX_GRID_RANK) {
     fprintf(stderr, "%s: grid_rank is greater than 2. This program only converts grids with grid_rank <= 2.\n",c_infile);
     ESMC_Finalize();
     exit(1); // bail out
  }

  // Get grid dims
  status = nc_inq_varid(ncid1, "grid_dims", &varid);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  status = nc_get_var_int(ncid1, varid, grid_dims);
  if (status != NC_NOERR) handle_error(status,__LINE__);


#if 1
  noarea = 0;
  nocenter = 0;
  nomask = 0;
  // inquire variable ids
  status = nc_inq_varid(ncid1, "grid_area", &areaid);
  if (status != NC_NOERR) noarea = 1;
  status = nc_inq_varid(ncid1, "grid_center_lat", &ctlatid);
  if (status != NC_NOERR) nocenter = 1;
  status = nc_inq_varid(ncid1, "grid_center_lon", &ctlonid);
  if ((status != NC_NOERR && nocenter != 1) || (status == NC_NOERR && nocenter == 1)) {
    fprintf(stderr, "%s: Either grid_center_lat or grid_center_lon does not exist.\n",c_infile);
    ESMC_Finalize();
    exit(1); // bail out
  }
#else
  noarea = 1;
  nocenter = 1;
  nomask = 1;

#endif
  status = nc_inq_varid(ncid1, "grid_corner_lat", &colatid);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  status = nc_inq_varid(ncid1, "grid_corner_lon", &colonid);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  status = nc_inq_varid(ncid1, "grid_imask", &maskid);
  if (status != NC_NOERR) nomask = 1;

  // read in the corner lat/lon and extract unique node (vertex) list
  cornerlats = (double*)malloc(sizeof(double)*gcdim*gsdim);
  cornerlons = (double*)malloc(sizeof(double)*gcdim*gsdim);
  status = nc_get_var_double(ncid1, colatid, cornerlats);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  status = nc_get_var_double(ncid1, colonid, cornerlons);
  if (status != NC_NOERR) handle_error(status,__LINE__);

  // get units of grid_cornor_lon
  status = nc_inq_attlen(ncid1, colonid, "units", &len);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  status = nc_get_att_text(ncid1, colonid, "units", units);
  if (status != NC_NOERR) handle_error(status,__LINE__);
  units[len] = '\0';

  // convert radian to degree
  for (i=0; i<len; i++) {
    units[i]=tolower(units[i]);
  }
  if (strncmp(units, "degrees", 7) && strncmp(units, "radians", 7)) {
    fprintf(stderr, "%s: The units attribute for grid_corner_lon is not degrees nor radians.\n",c_infile);
    ESMC_Finalize();
    exit(1);
  }
  minlat=180;
  maxlat=-180;
  if (!strncmp(units, "radians", 7)) {
    for (i = 0; i < gcdim*gsdim; i++) {
      cornerlats[i] *= rad2deg;
      cornerlons[i] *= rad2deg;
    }
  }

  for (i = 0; i < gcdim*gsdim; i++) {
    if (cornerlats[i] < minlat) minlat=cornerlats[i];
    if (cornerlats[i] > maxlat) maxlat=cornerlats[i];
  }

  maxlat += 0.00001;

#if 1
  part = (maxlat-minlat)/nprocs;
  startlat = myrank*part+minlat;
  endlat = startlat+part;
#else
  // do some manual load balance for a unstructured land grid using 4PEs
  if (myrank==0) {
    startlat = minlat;
    endlat = 0;
  } else {
    part = (maxlat-0)/(nprocs-1);
    startlat = (myrank-1)*part;
    endlat = startlat+part;
  }
#endif
  //  startind = gsdim*gcdim*myrank;
  startind = 0;
  // printf("myrank %d latitude range: %lf, %lf, %d\n", myrank, startlat, endlat, startind);
  
  totalsize = gsdim * gcdim;
  cells = (int*)malloc(sizeof(int)*totalsize);
  ESMCI::ClumpPntsLL(totalsize, cornerlons, cornerlats, TOL, cells, &totalnodes,
		     &nodelons, &nodelats, &maxconnection, startlat, endlat, &status);

  // cells[] are 0-based index, if not filled, the null value is -1
  //
  // printf("PET %d: totalnodes %d, maxconnect %d, start and end lat %f %f\n", myrank, totalnodes, maxconnection, startlat, endlat);
  free(cornerlats);
  free(cornerlons);

  // create node table
  // also count the maximum cells that vertex belongs to, this
  // this value will decide the maximal edges of the dual mesh
  nodelatlon = (double*)malloc(sizeof(double)*totalnodes*2);
  totalneighbors=(int*)calloc(totalnodes, sizeof(int));
  // if the original longitude is in (-180, 180), conver it back
  for (i=0; i<totalnodes; i++) {
    nodelatlon[i*2]=nodelons[i];
    nodelatlon[i*2+1]=nodelats[i];
  }

  free(nodelons);
  free(nodelats);

  for (i=0; i<totalsize; i++) {
    cells[i]++;
    if (cells[i] > 0) {
      totalneighbors[cells[i]-1] += 1;
    }
  }

  //printf("PET %d: Finish creating node table\n",myrank);
  MPI_Barrier(mpi_comm);

  // broadcast totalnodes to all PETs, adjust the node index in cells[]
  globalnodes = (int*)malloc(sizeof(int)*nprocs);
  MPI_Allgather(&totalnodes, 1, MPI_INT, globalnodes, 1, MPI_INT, mpi_comm);
  mystart = 0;
  for (i=0; i<myrank; i++) {
    mystart += globalnodes[i];
  }
  for (i=0; i<gcdim*gsdim; i++) {
    if (cells[i]>0)  cells[i]+=mystart;
  }
  
  alltotal = 0;
  for (i=0; i<nprocs; i++) {
    alltotal += globalnodes[i];
  }
  //printf("PET %d: Total number of nodes: %d local nodes: %d starting at %d\n", myrank, alltotal, totalnodes,mystart );
  // consolidate cell table, each PET gets gsdim/nprocs cells
  // do global reduce for all the processors
#ifndef ESMF_MPIUNI
  MPI_Allreduce(MPI_IN_PLACE, cells, gsdim*gcdim, MPI_INT, MPI_SUM, mpi_comm);
#endif

  mypart = (int)(gsdim/nprocs);
  left = gsdim%nprocs;
  mystartelement = mypart*myrank;
  if (myrank == nprocs-1) {
    mypart += left;
  }
  mycells = &cells[mystartelement*gcdim]; 

#if 1
  // check for degenerated cells, remove duplicate nodes and fill cell_edges;
  edges = (int*)malloc(mypart*sizeof(int));
  int mycount = 0;
  for (i=0; i<gsdim; i++) {
    i1=i*gcdim;
    temp[0]=cells[i1];
    count = 1;

    for (j=1; j<gcdim; j++) {
      for (k=0; k<j; k++) {
	if (cells[i1+j]==cells[i1+k]) {
	  // the two vertices belong to one cell, over-counted
	  ind=cells[i1+j]-1-mystart;
	  if (ind >=0 && ind < totalnodes) {
	    totalneighbors[ind]--;
	    if (totalneighbors[ind] < 0) {
	      printf("PET %d: negative neighbors: %d %d %d %f %f\n", myrank, i, j, ind, nodelatlon[ind*2], nodelatlon[ind*2+1]);
	    }
	  }
	  //printf("duplicate vertex at %d: vertex %d\n", i, mycells[i1+j]);
	  break;
	}
      }
      if (k==j) {
	temp[count++]=cells[i1+j];
      }
    }
    // copy temp array back to cell, fill with unfilled space with -1
    if (i >= mystartelement && i < mystartelement+mypart) {
      edges[i-mystartelement]=count;
    }
    for (j=0; j<count; j++) {
      cells[i1+j]=temp[j];
    }
    for (j=count; j<gcdim; j++) {
      cells[i1+j]=-1;
    }
  }

  //printf("PET %d: my element starts at %d for %d counts %d elements have 5 edges\n", 
  //	 myrank, mystartelement, mypart, mycount);
#endif

  // find the maximal number of neighbors for all the vertices
  maxconnection = 0;
  for (i=0; i< totalnodes; i++) {
    if (totalneighbors[i]>maxconnection)
      maxconnection=totalneighbors[i];
  }

  // find the max of maxconnection using MPI_AllReduce
#ifndef ESMF_MPIUNI
  MPI_Allreduce(MPI_IN_PLACE, &maxconnection, 1, MPI_INT, MPI_MAX, mpi_comm);
#endif
  // global max to find the maxconnection
  //printf("Maximal connection per vertex is %d\n", maxconnection);

  // create output file at PET=0
  if (dualflag == 0) {
    if (myrank == 0) {
      if (doesmf) {
	// create the output ESMF netcdf file
	create_esmf(c_outfile, c_infile, dualflag, alltotal, gsdim, gcdim, nocenter, nomask, noarea, grdim, grid_dims); 
      } else {
        // Create UGRID file
        create_ugrid(c_outfile, c_infile, dualflag, alltotal, gsdim, gcdim, nocenter);
      }
    }

    MPI_Barrier(mpi_comm);
    
    // now write out node and elements in sequence
    for (i=0; i<nprocs; i++) {
      if (myrank == i) {
	status=nc_open(c_outfile, NC_WRITE, &ncid2);
	if (status != NC_NOERR) handle_error(status,__LINE__);
        //printf("%d: write nodeCoords from %d of total %d count\n", myrank, mystart, totalnodes);
	if (doesmf) {
	  start2[0]=mystart;
	  start2[1]=0;
	  count2[0]=totalnodes;
	  count2[1]=2;
	  status = nc_inq_varid(ncid2, "nodeCoords" ,&vertexid);
	  status = nc_put_vara_double(ncid2, vertexid, start2, count2, nodelatlon); 
	  if (status != NC_NOERR) handle_error(status,__LINE__);
	} else {
	  coord1d = (double*)malloc(totalnodes*sizeof(double));
	  for (j=0; j<totalnodes; j++) {
	    coord1d[j]=nodelatlon[j*2];
	  }
	  start1[0]=mystart;
	  count1[0]=totalnodes;
	  status = nc_inq_varid(ncid2, "node_x" ,&vertexid);
	  status = nc_put_vara_double(ncid2, vertexid, start1, count1, coord1d); 
	  if (status != NC_NOERR) handle_error(status,__LINE__);
	  for (j=0; j<totalnodes; j++) {
	    coord1d[j]=nodelatlon[j*2+1];
	  }
	  status = nc_inq_varid(ncid2, "node_y" ,&vertexid);
	  status = nc_put_vara_double(ncid2, vertexid, start1, count1, coord1d); 
	  if (status != NC_NOERR) handle_error(status,__LINE__);
	  free(coord1d);
	}
        start2[0]=mystartelement;
        start2[1]=0;
        count2[0]=mypart;
        count2[1]=gcdim;
        //printf("%d: write elementConn from %d of total %d count\n", myrank, mystartelement, mypart);
        status = nc_inq_varid(ncid2, "elementConn" ,&cellid);
	status = nc_put_vara_int(ncid2, cellid, start2, count2, mycells);
	if (status != NC_NOERR) handle_error(status,__LINE__);
	if (doesmf) {
	  status = nc_inq_varid(ncid2, "numElementConn" ,&edgeid);
	  start1[0]=mystartelement;
	  count1[0]=mypart;
	  status = nc_put_vara_int(ncid2, edgeid, start1, count1, edges);
	  if (status != NC_NOERR) handle_error(status,__LINE__);
	}
	status=nc_close(ncid2);
	if (status != NC_NOERR) handle_error(status,__LINE__);
	MPI_Barrier(mpi_comm);
      } else 
	MPI_Barrier(mpi_comm);
    }
    free(edges);
    free(nodelatlon);
  
    if (myrank == 0) {
	status=nc_open(c_outfile, NC_WRITE, &ncid2);
	if (status != NC_NOERR) handle_error(status,__LINE__);

        // Output original grid dims if 2D and esmf format
        if ((doesmf) && (grdim == 2)) {
          status = nc_inq_varid(ncid2, "origGridDims" ,&ogd_id);

          status = nc_put_var_int(ncid2,ogd_id, grid_dims);
          if (status != NC_NOERR) handle_error(status,__LINE__);
        }

	inbuf = (double*)malloc(sizeof(double)*gsdim);
	if (!nocenter) {
	  // get units of grid_center_lon
	  status = nc_inq_attlen(ncid1, ctlonid, "units", &len);
	  if (status != NC_NOERR) handle_error(status,__LINE__);
	  status = nc_get_att_text(ncid1, ctlonid, "units", units);
	  if (status != NC_NOERR) handle_error(status,__LINE__);
	  units[len] = '\0';
	  // convert radian to degree
	  for (i=0; i<len; i++) {
	    units[i]=tolower(units[i]);
	  }
	  if (strncmp(units, "degrees", 7) && strncmp(units, "radians", 7)) {
	    fprintf(stderr, "%s: The units attribute for grid_center_lon is not degrees nor radians.\n", c_infile);
	    ESMC_Finalize();
	    exit(1);
	  }
	  status = nc_get_var_double(ncid1, ctlatid, inbuf);
	  if (status != NC_NOERR) handle_error(status,__LINE__);
	  if (!strncmp(units, "radians", 7)) {
	    for (i=0; i<gsdim; i++) {
	      inbuf[i] *= rad2deg;
	    }
	  }
	  if (doesmf) {
	    // copy inbuf to inbuf1
	    inbuf1 = (double*)malloc(sizeof(double)*gsdim*2);
	    for (i=0; i<gsdim; i++) {
	      inbuf1[i*2+1]=inbuf[i];
	    }
	  } else {
	    status = nc_inq_varid(ncid2, "face_y", &varid);
	    if (status != NC_NOERR) handle_error(status,__LINE__);
	    status = nc_put_var_double(ncid2, varid, inbuf);
	    if (status != NC_NOERR) handle_error(status,__LINE__);
	  }

	  status = nc_get_var_double(ncid1, ctlonid, inbuf);
	  if (status != NC_NOERR) handle_error(status,__LINE__);
	  if (!strncmp(units, "radians", 7)) {
	    for (i=0; i<gsdim; i++) {
	      inbuf[i] *= rad2deg;
	    }
	  }
	  if (doesmf) {
	    // copy inbuf to inbuf1
	    for (i=0; i<gsdim; i++) {
	      inbuf1[i*2]=inbuf[i];
	    }
	  } else {
	    status = nc_inq_varid(ncid2, "face_x", &varid);
	    if (status != NC_NOERR) handle_error(status,__LINE__);
	    status = nc_put_var_double(ncid2, varid, inbuf);
	    if (status != NC_NOERR) handle_error(status,__LINE__);
	  }
	  if (doesmf) {
	    starts[0]=0;
	    starts[1]=0;
	    counts[0]=gsdim;
	    counts[1]=2;
	    status = nc_inq_varid(ncid2, "centerCoords", &ccoordid);
	    if (status != NC_NOERR) handle_error(status,__LINE__);
	    status = nc_put_vara_double(ncid2, ccoordid, starts, counts, inbuf1);
	    if (status != NC_NOERR) handle_error(status,__LINE__);
	    free(inbuf1);
	  }  
	  if (doesmf && !nomask) {
	    inbuf2=(int*)malloc(sizeof(int)*gsdim);
	    status = nc_get_var_int(ncid1, maskid, inbuf2);
	    if (status != NC_NOERR) handle_error(status,__LINE__);
	    status = nc_inq_varid(ncid2, "elementMask", &cmid);
	    if (status != NC_NOERR) handle_error(status,__LINE__);
	    status = nc_put_var_int(ncid2, cmid, inbuf2);
	    if (status != NC_NOERR) handle_error(status,__LINE__);
	    free(inbuf2);
	  }
	  if (doesmf && !noarea) {
	    status = nc_get_var_double(ncid1, areaid, inbuf);
	    if (status != NC_NOERR) handle_error(status,__LINE__);
	    status = nc_inq_varid(ncid2, "elementArea", &caid);
	    if (status != NC_NOERR) handle_error(status,__LINE__);
	    status = nc_put_var_double(ncid2, caid, inbuf);
	    if (status != NC_NOERR) handle_error(status,__LINE__);
	  }
	  free(inbuf);
	}
	nc_close(ncid2);
    }
    nc_close(ncid1);
    free(totalneighbors);
  } else {
    // Now create the dual mesh using the cell coordinates.  The
    // format is the same except that the num_verts = the original num_cells (gsdim), 
    // vert_coords will be the original center-coords. num_cells = the original
    // num_verts (totalnodes) mask is not
    // changed, and cell_verts will be generated here
    // 
    // for each vert in the original grid, find out which cell uses it, use the
    // the center of the cells to form a new cell
    // The dual mesh should have equal number of cells and vertices
    //
    // celltbl = (int*)malloc(sizeof(int)*maxconnection);
    // the new dual mesh may not have the same topology as the original mesh
    // it depends on how many edges are sharing a specific vertices
    // so, this has to be calculated as well
    
    // First, read in the center coordinates, I need all the center coordinates to calculate the order
    inbuf = (double*)malloc(sizeof(double)*gsdim);
    inbuf1 = (double*)malloc(sizeof(double)*gsdim*2);
    status = nc_inq_varid(ncid1, "grid_center_lat", &ctlatid);
    if (status != NC_NOERR) {
      fprintf(stderr, "grid_center_lat has to exist to create a dual mesh.\n");
      handle_error(status,__LINE__);
    }
    status = nc_get_var_double(ncid1, ctlatid, inbuf);
    if (status != NC_NOERR) handle_error(status,__LINE__);
    // copy inbuf to inbuf1
    for (i=0; i<gsdim; i++) {
      inbuf1[i*2+1]=inbuf[i];
    }
    status = nc_inq_varid(ncid1, "grid_center_lon", &ctlonid);
    if (status != NC_NOERR) {
      fprintf(stderr, "grid_center_lon has to exist to create a dual mesh.\n");
      handle_error(status,__LINE__);
    }
    status = nc_get_var_double(ncid1, ctlonid, inbuf);
    if (status != NC_NOERR) handle_error(status,__LINE__);
    // copy inbuf to inbuf1
    for (i=0; i<gsdim; i++) {
      inbuf1[i*2]=inbuf[i];
    }
    // get units of grid_center_lon
    status = nc_inq_attlen(ncid1, ctlonid, "units", &len);
    if (status != NC_NOERR) handle_error(status,__LINE__);
    status = nc_get_att_text(ncid1, ctlonid, "units", units);
    if (status != NC_NOERR) handle_error(status,__LINE__);
    units[len]='\0';
    // convert radian to degree
    for (i=0; i<len; i++) {
      units[i]=tolower(units[i]);
    }
    if (strncmp(units, "degrees", 7) && strncmp(units, "radians", 7)) {
      fprintf(stderr, "%s: The units attribute for grid_center_lon is not degrees nor radians.\n", c_infile);
      ESMC_Finalize();
      exit(1);
    }
    if (!strncmp(units, "radians", 7)) {
      for (i=0; i<gsdim*2; i++) {
	inbuf1[i] *= rad2deg;
      }
    }
    free(inbuf);

    dualcells = (int*)malloc(sizeof(int)*maxconnection*totalnodes);
    dualcellcounts = (int*)malloc(sizeof(int)*totalnodes);
    for (i=0; i<totalnodes; i++)
      dualcellcounts[i]=0;
    // initialize the values to -1
    for (i=0; i<maxconnection*totalnodes; i++)
      dualcells[i]=-1;
  
    // go through the cells table and put the cell id into the dualcell table
    int cnt=0;
    for (i=0,k=0; i<gsdim; i++) {
      for (j=0; j<gcdim; j++,k++) {
	i1 = cells[k]-1-mystart;
	// check if this node is local
	if (i1 >= 0 && i1 < totalnodes) { 
	  cnt++;
	  dualcells[i1*maxconnection+dualcellcounts[i1]]=i+1;
	  dualcellcounts[i1]++;
	  if (dualcellcounts[i1] > maxconnection) {
	    printf("Vertex %d exceed maximal connections %d\n", i1, maxconnection);
	    ESMC_Finalize();
	    exit(1); // bail out
	  }
	}
      }
    }

    // remove the cells with less than 3 edges in dualcells table
    // also remove them from the node coordinates table and the totalneighbors table 
    for (i=0, i1=0; i<totalnodes; i++) {
      if (dualcellcounts[i] >= 3) {
	if (i1 != i) {
	  for (k=0; k<maxconnection; k++) {
	    dualcells[i1*maxconnection+k]=dualcells[i*maxconnection+k];
	    totalneighbors[i1]=totalneighbors[i];
	    nodelatlon[i1*2]=nodelatlon[i*2];
	    nodelatlon[i1*2+1]=nodelatlon[i*2+1];
	  }
	}
	i1++;
      }
    }

    goodnodes = i1;
    // printf("Total nodes: %d, total non-degenerated nodes: %d\n", totalnodes, goodnodes);
    // broadcast goodnodes to get total number of good nodes and revise mystart and alltotal
    globalnodes = (int*)malloc(sizeof(int)*nprocs);
    MPI_Allgather(&goodnodes, 1, MPI_INT, globalnodes, 1, MPI_INT, mpi_comm);
    mystart = 0;
    for (i=0; i<myrank; i++) {
      mystart += globalnodes[i];
    }
    alltotal = 0;
    for (i=0; i<nprocs; i++) {
      alltotal += globalnodes[i];
    }

    // order the cell center coordinates in counter-clockwise order
    // lonbuf and latbuf contains the center vertex coordinates
    // next points to the cell_vertex location where we will fill
    // in the cell id in counter clockwise order
    for (i = 0; i < goodnodes; i++) {  
      next = &dualcells[i*maxconnection];
      numedges = totalneighbors[i];
      if (fabs(nodelatlon[i*2+1]) > 88.0) {
	orderit2(i+1, nodelatlon[i*2], nodelatlon[i*2+1], numedges, inbuf1,next);      
      } else {
	orderit(i+1, nodelatlon[i*2], nodelatlon[i*2+1], numedges, inbuf1,next);      
      }
    }

    free(dualcellcounts);
    // now write out the dual mesh in a netcdf file
    // create the output netcdf file

    totalnodes = goodnodes;

    if (myrank==0) { 
      if (doesmf == 1) {
	create_esmf(c_outfile, c_infile, dualflag, gsdim, alltotal, maxconnection, 0, nomask, 1, grdim, grid_dims); 
      } else {
	create_ugrid(c_outfile, c_infile, dualflag, gsdim, alltotal, maxconnection, 0);
      }
    }
 
    MPI_Barrier(mpi_comm);

    // now write out node and elements in sequence
    for (i=0; i<nprocs; i++) {
      if (myrank == i) {
	status=nc_open(c_outfile, NC_WRITE, &ncid2);
	if (status != NC_NOERR) handle_error(status,__LINE__);
	if (myrank == 0) {
	  if (doesmf) {
	    status = nc_inq_varid(ncid2, "nodeCoords" ,&vertexid);
	    if (status != NC_NOERR) handle_error(status,__LINE__);
	    status = nc_put_var_double(ncid2, vertexid, inbuf1); 
	    if (status != NC_NOERR) handle_error(status,__LINE__);
	  } else {
	    inbuf = (double*)malloc(sizeof(double)*gsdim);
	    for (j=0; j<gsdim;j++) {
	      inbuf[j]=inbuf1[j*2];
	    }  
	    status = nc_inq_varid(ncid2, "node_x" ,&vertexid);
	    if (status != NC_NOERR) handle_error(status,__LINE__);
	    status = nc_put_var_double(ncid2, vertexid, inbuf); 
	    if (status != NC_NOERR) handle_error(status,__LINE__);
	    for (j=0;  j<gsdim;j++) {
	      inbuf[j]=inbuf1[j*2+1];
	    }  
	    status = nc_inq_varid(ncid2, "node_y" ,&vertexid);
	    if (status != NC_NOERR) handle_error(status,__LINE__);
	    status = nc_put_var_double(ncid2, vertexid, inbuf); 
	    if (status != NC_NOERR) handle_error(status,__LINE__);
	    free(inbuf);
	  }
	}
	free(inbuf1);
	if (doesmf) {
	  start2[0]=mystart;
	  start2[1]=0;
	  count2[0]=totalnodes;
	  count2[1]=2;
	  // printf("%d: write centerCoords from %d of total %d count\n", myrank, mystart, totalnodes);
	  status = nc_inq_varid(ncid2, "centerCoords" ,&vertexid);
	  if (status != NC_NOERR) handle_error(status,__LINE__);
	  status = nc_put_vara_double(ncid2, vertexid, start2, count2, nodelatlon); 
	  if (status != NC_NOERR) handle_error(status,__LINE__);
	  // write out nodemask
	  if (!nomask) {
	    inbuf2=(int*)malloc(sizeof(int)*gsdim);
	    status = nc_get_var_int(ncid1, maskid, inbuf2);
	    if (status != NC_NOERR) handle_error(status,__LINE__);
	    status = nc_inq_varid(ncid2, "nodeMask", &cmid);
	    if (status != NC_NOERR) handle_error(status,__LINE__);
	    status = nc_put_var_int(ncid2, cmid, inbuf2);
	    if (status != NC_NOERR) handle_error(status,__LINE__);
	    free(inbuf2);
	  }
	} else {
	  start1[0]=mystart;
	  count1[0]=totalnodes;
	  inbuf=(double*)malloc(sizeof(double)*totalnodes);
	  for (j=0; j<totalnodes;j++) {
	    inbuf[j]=nodelatlon[j*2];
	  }  
	  status = nc_inq_varid(ncid2, "face_x" ,&vertexid);
	  if (status != NC_NOERR) handle_error(status,__LINE__);
	  status = nc_put_vara_double(ncid2, vertexid, start1, count1, nodelatlon); 
	  if (status != NC_NOERR) handle_error(status,__LINE__);
	  for (j=0; j<totalnodes;j++) {
	    inbuf[j]=nodelatlon[j*2+1];
	  }  
	  status = nc_inq_varid(ncid2, "face_y" ,&vertexid);
	  if (status != NC_NOERR) handle_error(status,__LINE__);
	  status = nc_put_vara_double(ncid2, vertexid, start1, count1, nodelatlon); 
	  if (status != NC_NOERR) handle_error(status,__LINE__);
	  free(inbuf);
	}
	free(nodelatlon);
	start2[0]=mystart;
	start2[1]=0;
	count2[0]=totalnodes;
	count2[1]=maxconnection;
	// printf("%d: write elementConn from %d of total %d count\n", myrank, mystart, totalnodes);
	status = nc_inq_varid(ncid2, "elementConn" ,&cellid);
	status = nc_put_vara_int(ncid2, cellid, start2, count2, dualcells);
	if (status != NC_NOERR) handle_error(status,__LINE__);
	if (doesmf) {
	  status = nc_inq_varid(ncid2, "numElementConn" ,&edgeid);
	  start1[0]=mystart;
	  count1[0]=totalnodes;
	  status = nc_put_vara_int(ncid2, edgeid, start1, count1, totalneighbors);
	  if (status != NC_NOERR) handle_error(status,__LINE__);
	}
	status=nc_close(ncid2);
	if (status != NC_NOERR) handle_error(status,__LINE__);
	MPI_Barrier(mpi_comm);
      } else {
	MPI_Barrier(mpi_comm);
      }
    }
    free(totalneighbors);
    free(dualcells);
    nc_close(ncid1);
  }   
  free(cells);
  if (myrank == 0) {
    printf("Done converting %s\n", c_infile);
  }
  ESMC_Finalize();
  
#else
  if (myrank==0) {
    fprintf(stderr, "Have to compile with ESMF_NETCDF environment variable defined\n");
  }
  ESMC_Finalize();
  exit(1);
#endif
}


