Actual source code: parmmgadapt.c

      1: #include "../mmgcommon.h"   /*I      "petscdmplex.h"   I*/
      2: #include <parmmg/libparmmg.h>

      4: PETSC_EXTERN PetscErrorCode DMAdaptMetric_ParMmg_Plex(DM dm, Vec vertexMetric, DMLabel bdLabel, DMLabel rgLabel, DM *dmNew)
      5: {
      6:   MPI_Comm           comm;
      7:   const char        *bdName = "_boundary_";
      8:   const char        *rgName = "_regions_";
      9:   DM                 udm, cdm;
     10:   DMLabel            bdLabelNew, rgLabelNew;
     11:   const char        *bdLabelName, *rgLabelName;
     12:   IS                 globalVertexNum;
     13:   PetscSection       coordSection;
     14:   Vec                coordinates;
     15:   PetscSF            sf;
     16:   const PetscScalar *coords, *met;
     17:   PetscReal         *vertices, *metric, *verticesNew, *verticesNewLoc, gradationFactor, hausdorffNumber;
     18:   PetscInt          *cells, *cellsNew, *cellTags, *cellTagsNew, *verTags, *verTagsNew;
     19:   PetscInt          *bdFaces, *faceTags, *facesNew, *faceTagsNew;
     20:   PetscInt          *corners, *requiredCells, *requiredVer, *ridges, *requiredFaces;
     21:   PetscInt           cStart, cEnd, c, numCells, fStart, fEnd, f, numFaceTags, vStart, vEnd, v, numVertices;
     22:   PetscInt           dim, off, coff, maxConeSize, bdSize, i, j, k, Neq, verbosity, numIter;
     23:   PetscInt          *numVerInterfaces, *ngbRanks, *verNgbRank, *interfaces_lv, *interfaces_gv, *intOffset;
     24:   PetscInt           niranks, nrranks, numNgbRanks, numVerNgbRanksTotal, count, sliceSize, p, r, n, lv, gv;
     25:   PetscInt          *gv_new, *owners, *verticesNewSorted, pStart, pEnd;
     26:   PetscInt           numCellsNew, numVerticesNew, numCornersNew, numFacesNew, numVerticesNewLoc;
     27:   const PetscInt    *gV, *ioffset, *irootloc, *roffset, *rmine, *rremote;
     28:   PetscBool          flg = PETSC_FALSE, noInsert, noSwap, noMove, noSurf, isotropic, uniform;
     29:   const PetscMPIInt *iranks, *rranks;
     30:   PetscMPIInt        numProcs, rank;
     31:   PMMG_pParMesh      parmesh = NULL;

     33:   PetscObjectGetComm((PetscObject) dm, &comm);
     34:   MPI_Comm_size(comm, &numProcs);
     35:   MPI_Comm_rank(comm, &rank);
     36:   if (bdLabel) {
     37:     PetscObjectGetName((PetscObject) bdLabel, &bdLabelName);
     38:     PetscStrcmp(bdLabelName, bdName, &flg);
     40:   }
     41:   if (rgLabel) {
     42:     PetscObjectGetName((PetscObject) rgLabel, &rgLabelName);
     43:     PetscStrcmp(rgLabelName, rgName, &flg);
     45:   }

     47:   /* Get mesh information */
     48:   DMGetDimension(dm, &dim);
     50:   Neq  = (dim*(dim+1))/2;
     51:   DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);
     52:   DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);
     53:   DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);
     54:   DMPlexUninterpolate(dm, &udm);
     55:   DMPlexGetMaxSizes(udm, &maxConeSize, NULL);
     56:   numCells    = cEnd - cStart;
     57:   numVertices = vEnd - vStart;

     59:   /* Get cell offsets */
     60:   PetscMalloc1(numCells*maxConeSize, &cells);
     61:   for (c = 0, coff = 0; c < numCells; ++c) {
     62:     const PetscInt *cone;
     63:     PetscInt        coneSize, cl;

     65:     DMPlexGetConeSize(udm, c, &coneSize);
     66:     DMPlexGetCone(udm, c, &cone);
     67:     for (cl = 0; cl < coneSize; ++cl) cells[coff++] = cone[cl] - vStart+1;
     68:   }

     70:   /* Get vertex coordinate array */
     71:   DMGetCoordinateDM(dm, &cdm);
     72:   DMGetLocalSection(cdm, &coordSection);
     73:   DMGetCoordinatesLocal(dm, &coordinates);
     74:   VecGetArrayRead(coordinates, &coords);
     75:   PetscMalloc2(numVertices*Neq, &metric, dim*numVertices, &vertices);
     76:   for (v = 0; v < vEnd-vStart; ++v) {
     77:     PetscSectionGetOffset(coordSection, v+vStart, &off);
     78:     for (i = 0; i < dim; ++i) vertices[dim*v+i] = PetscRealPart(coords[off+i]);
     79:   }
     80:   VecRestoreArrayRead(coordinates, &coords);

     82:   /* Get face tags */
     83:   if (!bdLabel) {
     84:     flg = PETSC_TRUE;
     85:     DMLabelCreate(PETSC_COMM_SELF, bdName, &bdLabel);
     86:     DMPlexMarkBoundaryFaces(dm, 1, bdLabel);
     87:   }
     88:   DMLabelGetBounds(bdLabel, &pStart, &pEnd);
     89:   for (f = pStart, bdSize = 0, numFaceTags = 0; f < pEnd; ++f) {
     90:     PetscBool hasPoint;
     91:     PetscInt *closure = NULL, closureSize, cl;

     93:     DMLabelHasPoint(bdLabel, f, &hasPoint);
     94:     if ((!hasPoint) || (f < fStart) || (f >= fEnd)) continue;
     95:     numFaceTags++;

     97:     DMPlexGetTransitiveClosure(dm, f, PETSC_TRUE, &closureSize, &closure);
     98:     for (cl = 0; cl < closureSize*2; cl += 2) {
     99:       if ((closure[cl] >= vStart) && (closure[cl] < vEnd)) ++bdSize;
    100:     }
    101:     DMPlexRestoreTransitiveClosure(dm, f, PETSC_TRUE, &closureSize, &closure);
    102:   }
    103:   PetscMalloc2(bdSize, &bdFaces, numFaceTags, &faceTags);
    104:   for (f = pStart, bdSize = 0, numFaceTags = 0; f < pEnd; ++f) {
    105:     PetscBool hasPoint;
    106:     PetscInt *closure = NULL, closureSize, cl;

    108:     DMLabelHasPoint(bdLabel, f, &hasPoint);
    109:     if ((!hasPoint) || (f < fStart) || (f >= fEnd)) continue;

    111:     DMPlexGetTransitiveClosure(dm, f, PETSC_TRUE, &closureSize, &closure);
    112:     for (cl = 0; cl < closureSize*2; cl += 2) {
    113:       if ((closure[cl] >= vStart) && (closure[cl] < vEnd)) bdFaces[bdSize++] = closure[cl] - vStart+1;
    114:     }
    115:     DMPlexRestoreTransitiveClosure(dm, f, PETSC_TRUE, &closureSize, &closure);
    116:     DMLabelGetValue(bdLabel, f, &faceTags[numFaceTags++]);
    117:   }

    119:   /* Get cell tags */
    120:   PetscCalloc2(numVertices, &verTags, numCells, &cellTags);
    121:   if (rgLabel) {
    122:     for (c = cStart; c < cEnd; ++c) DMLabelGetValue(rgLabel, c, &cellTags[c]);
    123:   }

    125:   /* Get metric */
    126:   VecViewFromOptions(vertexMetric, NULL, "-adapt_metric_view");
    127:   VecGetArrayRead(vertexMetric, &met);
    128:   DMPlexMetricIsIsotropic(dm, &isotropic);
    129:   DMPlexMetricIsUniform(dm, &uniform);
    130:   for (v = 0; v < (vEnd-vStart); ++v) {
    131:     for (i = 0, k = 0; i < dim; ++i) {
    132:       for (j = i; j < dim; ++j, ++k) {
    133:         if (isotropic) {
    134:           if (i == j) {
    135:             if (uniform) metric[Neq*v+k] = PetscRealPart(met[0]);
    136:             else metric[Neq*v+k] = PetscRealPart(met[v]);
    137:           } else metric[Neq*v+k] = 0.0;
    138:         } else metric[Neq*v+k] = PetscRealPart(met[dim*dim*v+dim*i+j]);
    139:       }
    140:     }
    141:   }
    142:   VecRestoreArrayRead(vertexMetric, &met);

    144:   /* Build ParMMG communicators: the list of vertices between two partitions  */
    145:   niranks = nrranks = 0;
    146:   numNgbRanks = 0;
    147:   if (numProcs > 1) {
    148:     DMGetPointSF(dm, &sf);
    149:     PetscSFSetUp(sf);
    150:     PetscSFGetLeafRanks(sf, &niranks, &iranks, &ioffset, &irootloc);
    151:     PetscSFGetRootRanks(sf, &nrranks, &rranks, &roffset, &rmine, &rremote);
    152:     PetscCalloc1(numProcs, &numVerInterfaces);

    154:     /* Count number of roots associated with each leaf */
    155:     for (r = 0; r < niranks; ++r) {
    156:       for (i=ioffset[r], count=0; i<ioffset[r+1]; ++i) {
    157:         if (irootloc[i] >= vStart && irootloc[i] < vEnd) count++;
    158:       }
    159:       numVerInterfaces[iranks[r]] += count;
    160:     }

    162:     /* Count number of leaves associated with each root */
    163:     for (r = 0; r < nrranks; ++r) {
    164:       for (i=roffset[r], count=0; i<roffset[r+1]; ++i) {
    165:         if (rmine[i] >= vStart && rmine[i] < vEnd) count++;
    166:       }
    167:       numVerInterfaces[rranks[r]] += count;
    168:     }

    170:     /* Count global number of ranks */
    171:     for (p = 0; p < numProcs; ++p) {
    172:       if (numVerInterfaces[p]) numNgbRanks++;
    173:     }

    175:     /* Provide numbers of vertex interfaces */
    176:     PetscMalloc2(numNgbRanks, &ngbRanks, numNgbRanks, &verNgbRank);
    177:     for (p = 0, n = 0; p < numProcs; ++p) {
    178:       if (numVerInterfaces[p]) {
    179:         ngbRanks[n] = p;
    180:         verNgbRank[n] = numVerInterfaces[p];
    181:         n++;
    182:       }
    183:     }
    184:     numVerNgbRanksTotal = 0;
    185:     for (p = 0; p < numNgbRanks; ++p) numVerNgbRanksTotal += verNgbRank[p];

    187:     /* For each neighbor, fill in interface arrays */
    188:     PetscMalloc3(numVerNgbRanksTotal, &interfaces_lv, numVerNgbRanksTotal, &interfaces_gv,  numNgbRanks+1, &intOffset);
    189:     intOffset[0] = 0;
    190:     for (p = 0, r = 0, i = 0; p < numNgbRanks; ++p) {
    191:       intOffset[p+1] = intOffset[p];

    193:       /* Leaf case */
    194:       if (iranks && iranks[i] == ngbRanks[p]) {

    196:         /* Add the right slice of irootloc at the right place */
    197:         sliceSize = ioffset[i+1]-ioffset[i];
    198:         for (j = 0, count = 0; j < sliceSize; ++j) {
    200:           v = irootloc[ioffset[i]+j];
    201:           if (v >= vStart && v < vEnd) {
    203:             interfaces_lv[intOffset[p+1]+count] = v-vStart;
    204:             count++;
    205:           }
    206:         }
    207:         intOffset[p+1] += count;
    208:         i++;
    209:       }

    211:       /* Root case */
    212:       if (rranks && rranks[r] == ngbRanks[p]) {

    214:         /* Add the right slice of rmine at the right place */
    215:         sliceSize = roffset[r+1]-roffset[r];
    216:         for (j = 0, count = 0; j < sliceSize; ++j) {
    218:           v = rmine[roffset[r]+j];
    219:           if (v >= vStart && v < vEnd) {
    221:             interfaces_lv[intOffset[p+1]+count] = v-vStart;
    222:             count++;
    223:           }
    224:         }
    225:         intOffset[p+1] += count;
    226:         r++;
    227:       }

    229:       /* Check validity of offsets */
    231:     }
    232:     DMPlexGetVertexNumbering(udm, &globalVertexNum);
    233:     ISGetIndices(globalVertexNum, &gV);
    234:     for (i = 0; i < numVerNgbRanksTotal; ++i) {
    235:       v = gV[interfaces_lv[i]];
    236:       interfaces_gv[i] = v < 0 ? -v-1 : v;
    237:       interfaces_lv[i] += 1;
    238:       interfaces_gv[i] += 1;
    239:     }
    240:     ISRestoreIndices(globalVertexNum, &gV);
    241:     PetscFree(numVerInterfaces);
    242:   }
    243:   DMDestroy(&udm);

    245:   /* Send the data to ParMmg and remesh */
    246:   DMPlexMetricNoInsertion(dm, &noInsert);
    247:   DMPlexMetricNoSwapping(dm, &noSwap);
    248:   DMPlexMetricNoMovement(dm, &noMove);
    249:   DMPlexMetricNoSurf(dm, &noSurf);
    250:   DMPlexMetricGetVerbosity(dm, &verbosity);
    251:   DMPlexMetricGetNumIterations(dm, &numIter);
    252:   DMPlexMetricGetGradationFactor(dm, &gradationFactor);
    253:   DMPlexMetricGetHausdorffNumber(dm, &hausdorffNumber);
    254:   PMMG_Init_parMesh(PMMG_ARG_start, PMMG_ARG_ppParMesh, &parmesh, PMMG_ARG_pMesh, PMMG_ARG_pMet, PMMG_ARG_dim, 3, PMMG_ARG_MPIComm, comm, PMMG_ARG_end);
    255:   PMMG_Set_meshSize(parmesh, numVertices, numCells, 0, numFaceTags, 0, 0);
    256:   PMMG_Set_iparameter(parmesh, PMMG_IPARAM_APImode, PMMG_APIDISTRIB_nodes);
    257:   PMMG_Set_iparameter(parmesh, PMMG_IPARAM_noinsert, noInsert);
    258:   PMMG_Set_iparameter(parmesh, PMMG_IPARAM_noswap, noSwap);
    259:   PMMG_Set_iparameter(parmesh, PMMG_IPARAM_nomove, noMove);
    260:   PMMG_Set_iparameter(parmesh, PMMG_IPARAM_nosurf, noSurf);
    261:   PMMG_Set_iparameter(parmesh, PMMG_IPARAM_verbose, verbosity);
    262:   PMMG_Set_iparameter(parmesh, PMMG_IPARAM_globalNum, 1);
    263:   PMMG_Set_iparameter(parmesh, PMMG_IPARAM_niter, numIter);
    264:   PMMG_Set_dparameter(parmesh, PMMG_DPARAM_hgrad, gradationFactor);
    265:   PMMG_Set_dparameter(parmesh, PMMG_DPARAM_hausd, hausdorffNumber);
    266:   PMMG_Set_vertices(parmesh, vertices, verTags);
    267:   PMMG_Set_tetrahedra(parmesh, cells, cellTags);
    268:   PMMG_Set_triangles(parmesh, bdFaces, faceTags);
    269:   PMMG_Set_metSize(parmesh, MMG5_Vertex, numVertices, MMG5_Tensor);
    270:   PMMG_Set_tensorMets(parmesh, metric);
    271:   PMMG_Set_numberOfNodeCommunicators(parmesh, numNgbRanks);
    272:   for (c = 0; c < numNgbRanks; ++c) {
    273:     PMMG_Set_ithNodeCommunicatorSize(parmesh, c, ngbRanks[c], intOffset[c+1]-intOffset[c]);
    274:     PMMG_Set_ithNodeCommunicator_nodes(parmesh, c, &interfaces_lv[intOffset[c]], &interfaces_gv[intOffset[c]], 1);
    275:   }
    276:   PMMG_parmmglib_distributed(parmesh);
    277:   PetscFree(cells);
    278:   PetscFree2(metric, vertices);
    279:   PetscFree2(bdFaces, faceTags);
    280:   PetscFree2(verTags, cellTags);
    281:   if (numProcs > 1) {
    282:     PetscFree2(ngbRanks, verNgbRank);
    283:     PetscFree3(interfaces_lv, interfaces_gv, intOffset);
    284:   }

    286:   /* Retrieve mesh from Mmg */
    287:   numCornersNew = 4;
    288:   PMMG_Get_meshSize(parmesh, &numVerticesNew, &numCellsNew, 0, &numFacesNew, 0, 0);
    289:   PetscMalloc4(dim*numVerticesNew, &verticesNew, numVerticesNew, &verTagsNew, numVerticesNew, &corners, numVerticesNew, &requiredVer);
    290:   PetscMalloc3((dim+1)*numCellsNew, &cellsNew, numCellsNew, &cellTagsNew, numCellsNew, &requiredCells);
    291:   PetscMalloc4(dim*numFacesNew, &facesNew, numFacesNew, &faceTagsNew, numFacesNew, &ridges, numFacesNew, &requiredFaces);
    292:   PMMG_Get_vertices(parmesh, verticesNew, verTagsNew, corners, requiredVer);
    293:   PMMG_Get_tetrahedra(parmesh, cellsNew, cellTagsNew, requiredCells);
    294:   PMMG_Get_triangles(parmesh, facesNew, faceTagsNew, requiredFaces);
    295:   PetscMalloc2(numVerticesNew, &owners, numVerticesNew, &gv_new);
    296:   PMMG_Set_iparameter(parmesh, PMMG_IPARAM_globalNum, 1);
    297:   PMMG_Get_verticesGloNum(parmesh, gv_new, owners);
    298:   for (i = 0; i < dim*numFacesNew; ++i) facesNew[i] -= 1;
    299:   for (i = 0; i < (dim+1)*numCellsNew; ++i) cellsNew[i] = gv_new[cellsNew[i]-1]-1;
    300:   for (i = 0, numVerticesNewLoc = 0; i < numVerticesNew; ++i) {
    301:     if (owners[i] == rank) numVerticesNewLoc++;
    302:   }
    303:   PetscMalloc2(numVerticesNewLoc*dim, &verticesNewLoc, numVerticesNew, &verticesNewSorted);
    304:   for (i = 0, c = 0; i < numVerticesNew; i++) {
    305:     if (owners[i] == rank) {
    306:       for (j=0; j<dim; ++j) verticesNewLoc[dim*c+j] = verticesNew[dim*i+j];
    307:       c++;
    308:     }
    309:   }

    311:   /* Reorder for consistency with DMPlex */
    312:   for (i = 0; i < numCellsNew; ++i) DMPlexInvertCell(DM_POLYTOPE_TETRAHEDRON, &cellsNew[4*i]);

    314:   /* Create new plex */
    315:   DMPlexCreateFromCellListParallelPetsc(comm, dim, numCellsNew, numVerticesNewLoc, PETSC_DECIDE, numCornersNew, PETSC_TRUE, cellsNew, dim, verticesNewLoc, NULL, &verticesNewSorted, dmNew);
    316:   PMMG_Free_all(PMMG_ARG_start, PMMG_ARG_ppParMesh, &parmesh, PMMG_ARG_end);
    317:   PetscFree4(verticesNew, verTagsNew, corners, requiredVer);

    319:   /* Get adapted mesh information */
    320:   DMPlexGetHeightStratum(*dmNew, 0, &cStart, &cEnd);
    321:   DMPlexGetHeightStratum(*dmNew, 1, &fStart, &fEnd);
    322:   DMPlexGetDepthStratum(*dmNew, 0, &vStart, &vEnd);

    324:   /* Rebuild boundary label */
    325:   DMCreateLabel(*dmNew, flg ? bdName : bdLabelName);
    326:   DMGetLabel(*dmNew, flg ? bdName : bdLabelName, &bdLabelNew);
    327:   for (i = 0; i < numFacesNew; i++) {
    328:     PetscBool       hasTag = PETSC_FALSE;
    329:     PetscInt        numCoveredPoints, numFaces = 0, facePoints[3];
    330:     const PetscInt *coveredPoints = NULL;

    332:     for (j = 0; j < dim; ++j) {
    333:       lv = facesNew[i*dim+j];
    334:       gv = gv_new[lv]-1;
    335:       PetscFindInt(gv, numVerticesNew, verticesNewSorted, &lv);
    336:       facePoints[j] = lv+vStart;
    337:     }
    338:     DMPlexGetFullJoin(*dmNew, dim, facePoints, &numCoveredPoints, &coveredPoints);
    339:     for (j = 0; j < numCoveredPoints; ++j) {
    340:       if (coveredPoints[j] >= fStart && coveredPoints[j] < fEnd) {
    341:         numFaces++;
    342:         f = j;
    343:       }
    344:     }
    346:     DMLabelHasStratum(bdLabel, faceTagsNew[i], &hasTag);
    347:     if (hasTag) DMLabelSetValue(bdLabelNew, coveredPoints[f], faceTagsNew[i]);
    348:     DMPlexRestoreJoin(*dmNew, dim, facePoints, &numCoveredPoints, &coveredPoints);
    349:   }
    350:   PetscFree4(facesNew, faceTagsNew, ridges, requiredFaces);
    351:   PetscFree2(owners, gv_new);
    352:   PetscFree2(verticesNewLoc, verticesNewSorted);
    353:   if (flg) DMLabelDestroy(&bdLabel);

    355:   /* Rebuild cell labels */
    356:   DMCreateLabel(*dmNew, rgLabel ? rgLabelName : rgName);
    357:   DMGetLabel(*dmNew, rgLabel ? rgLabelName : rgName, &rgLabelNew);
    358:   for (c = cStart; c < cEnd; ++c) DMLabelSetValue(rgLabelNew, c, cellTagsNew[c-cStart]);
    359:   PetscFree3(cellsNew, cellTagsNew, requiredCells);

    361:   return 0;
    362: }