Skip to content

Commit 6fead36

Browse files
asterioukonstantinosasterioukonstantinos
committed
Extended checkpointing functionality when combining preInlet along with interiorViscosity or solidify_mechanics
1 parent 3189621 commit 6fead36

File tree

4 files changed

+46
-49
lines changed

4 files changed

+46
-49
lines changed

‎core/hemoCell.cpp

+13-12
Original file line numberDiff line numberDiff line change
@@ -327,7 +327,6 @@ void HemoCell::iterate() {
327327
if(iter %cellfields->particleVelocityUpdateTimescale == 0) {
328328
// #### 3 #### IBM interpolation
329329
cellfields->interpolateFluidVelocity();
330-
331330
// ### 4 ### sync the particles
332331
cellfields->syncEnvelopes();
333332
}
@@ -482,18 +481,20 @@ void HemoCell::initializeLattice(MultiBlockManagement3D const & management) {
482481
totalNodes += preInlet->getNumberOfNodes();
483482
totalNodes += cellsInBoundingBox(management.getBoundingBox());
484483

485-
preInlet->nProcs = global::mpi().getSize()*(preInlet->getNumberOfNodes()/(T)totalNodes);
486-
if (preInlet->nProcs == 0) {
487-
preInlet->nProcs = 1;
484+
try { // Look for block management info in the config file
485+
plint preInlet_pABx = (*cfg)["preInlet"]["parameters"]["pABx"].read<plint>();
486+
plint preInlet_pABy = (*cfg)["preInlet"]["parameters"]["pABy"].read<plint>();
487+
plint preInlet_pABz = (*cfg)["preInlet"]["parameters"]["pABz"].read<plint>();
488+
preInlet->nProcs = preInlet_pABx*preInlet_pABy*preInlet_pABz;
489+
}
490+
catch (const std::invalid_argument& e) {
491+
preInlet->nProcs = global::mpi().getSize()*(preInlet->getNumberOfNodes()/(T)totalNodes);
492+
if (preInlet->nProcs == 0) {
493+
preInlet->nProcs = 1;
494+
}
488495
}
489-
490-
// JON addition: Try to read in number of processors allocated to preinlet from config file.
491-
// TODO: Do we still need this?
492-
// Just continue with value computed above if reading it from XML throws an exception because it does not exist
493-
try { preInlet->nProcs = (*cfg)["preInlet"]["parameters"]["nProcs"].read<int>(); }
494-
catch (const std::invalid_argument& e) {}
495496

496-
int nProcs = global::mpi().getSize()-preInlet->nProcs;
497+
int nProcs = global::mpi().getSize() - preInlet->nProcs;
497498

498499
//Assign processors to PreInlet or Domain
499500
unsigned int currentPreInlet = 0;
@@ -692,4 +693,4 @@ void HemoCell::sanityCheck() {
692693

693694

694695
sanityCheckDone = true;
695-
}
696+
}

‎helper/bindingField.cpp

+18-14
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
2525
#include "hemocell.h"
2626
#include "palabos3D.h"
2727
#include "palabos3D.hh"
28+
#include "preInlet.h"
2829

2930
namespace hemo {
3031
bindingFieldHelper::bindingFieldHelper(HemoCellFields * cellFields_) : cellFields(*cellFields_) {
@@ -36,26 +37,26 @@ namespace hemo {
3637

3738
//Create bindingfield with same properties as fluid field underlying the particleField.
3839
multiBindingField = new plb::MultiScalarField3D<bool>(
39-
MultiBlockManagement3D (
40-
*cellFields.hemocell.lattice->getSparseBlockStructure().clone(),
41-
cellFields.hemocell.lattice->getMultiBlockManagement().getThreadAttribution().clone(),
42-
cellFields.hemocell.lattice->getMultiBlockManagement().getEnvelopeWidth(),
43-
cellFields.hemocell.lattice->getMultiBlockManagement().getRefinementLevel()),
40+
MultiBlockManagement3D (
41+
*cellFields.hemocell.domain_lattice->getSparseBlockStructure().clone(),
42+
cellFields.hemocell.domain_lattice->getMultiBlockManagement().getThreadAttribution().clone(),
43+
cellFields.hemocell.domain_lattice->getMultiBlockManagement().getEnvelopeWidth(),
44+
cellFields.hemocell.domain_lattice->getMultiBlockManagement().getRefinementLevel()),
4445
defaultMultiBlockPolicy3D().getBlockCommunicator(),
4546
defaultMultiBlockPolicy3D().getCombinedStatistics(),
4647
defaultMultiBlockPolicy3D().getMultiScalarAccess<bool>(),
4748
0);
48-
multiBindingField->periodicity().toggle(0,cellFields.hemocell.lattice->periodicity().get(0));
49-
multiBindingField->periodicity().toggle(1,cellFields.hemocell.lattice->periodicity().get(1));
50-
multiBindingField->periodicity().toggle(2,cellFields.hemocell.lattice->periodicity().get(2));
49+
multiBindingField->periodicity().toggle(0,cellFields.hemocell.domain_lattice->periodicity().get(0));
50+
multiBindingField->periodicity().toggle(1,cellFields.hemocell.domain_lattice->periodicity().get(1));
51+
multiBindingField->periodicity().toggle(2,cellFields.hemocell.domain_lattice->periodicity().get(2));
5152

5253
multiBindingField->initialize();
53-
5454
//Make sure each particleField has access to its local scalarField
5555
for (const plint & bId : multiBindingField->getLocalInfo().getBlocks()) {
56-
HemoCellParticleField & pf = cellFields.immersedParticles->getComponent(bId);
56+
HemoCellParticleField & pf = cellFields.domain_immersedParticles->getComponent(bId);
5757
pf.bindingField = &multiBindingField->getComponent(bId);
5858
}
59+
5960
}
6061

6162
bindingFieldHelper::~bindingFieldHelper() {
@@ -67,15 +68,17 @@ namespace hemo {
6768
pcout << "(BindingField) Checkpoint called while global.enableSolidifyMechanics is not enabled, still checkpointing but this should not happen" << endl;
6869
return;
6970
}
71+
7072
std::string & outDir = hemo::global.checkpointDirectory;
7173
mkpath(outDir.c_str(), 0777);
72-
74+
7375
if (global::mpi().isMainProcessor()) {
7476
renameFileToDotOld(outDir + "bindingSites.dat");
7577
renameFileToDotOld(outDir + "bindingSites.plb");
7678
}
77-
79+
7880
plb::parallelIO::save(*multiBindingField, outDir + "bindingSites", true);
81+
7982
}
8083

8184
void bindingFieldHelper::restore(HemoCellFields & cellFields) {
@@ -119,8 +122,8 @@ namespace hemo {
119122
}
120123

121124
void bindingFieldHelper::refillBindingSites() {
122-
for (const plint & bId : cellFields.immersedParticles->getLocalInfo().getBlocks()) {
123-
HemoCellParticleField & pf = cellFields.immersedParticles->getComponent(bId);
125+
for (const plint & bId : cellFields.domain_immersedParticles->getLocalInfo().getBlocks()) {
126+
HemoCellParticleField & pf = cellFields.domain_immersedParticles->getComponent(bId);
124127
ScalarField3D<bool> & bf = *pf.bindingField;
125128
Box3D domain = bf.getBoundingBox();
126129
for (int x = domain.x0; x <= domain.x1 ; x++) {
@@ -134,4 +137,5 @@ namespace hemo {
134137
}
135138
}
136139
}
140+
137141
}

‎helper/interiorViscosity.cpp

+11-11
Original file line numberDiff line numberDiff line change
@@ -31,23 +31,23 @@ namespace hemo {
3131
//Create bindingfield with same properties as fluid field underlying the particleField.
3232
multiInteriorViscosityField = new plb::MultiScalarField3D<T>(
3333
MultiBlockManagement3D (
34-
*cellFields.hemocell.lattice->getSparseBlockStructure().clone(),
35-
cellFields.hemocell.lattice->getMultiBlockManagement().getThreadAttribution().clone(),
36-
cellFields.hemocell.lattice->getMultiBlockManagement().getEnvelopeWidth(),
37-
cellFields.hemocell.lattice->getMultiBlockManagement().getRefinementLevel()),
34+
*cellFields.hemocell.domain_lattice->getSparseBlockStructure().clone(),
35+
cellFields.hemocell.domain_lattice->getMultiBlockManagement().getThreadAttribution().clone(),
36+
cellFields.hemocell.domain_lattice->getMultiBlockManagement().getEnvelopeWidth(),
37+
cellFields.hemocell.domain_lattice->getMultiBlockManagement().getRefinementLevel()),
3838
defaultMultiBlockPolicy3D().getBlockCommunicator(),
3939
defaultMultiBlockPolicy3D().getCombinedStatistics(),
4040
defaultMultiBlockPolicy3D().getMultiScalarAccess<T>(),
4141
0);
42-
multiInteriorViscosityField->periodicity().toggle(0,cellFields.hemocell.lattice->periodicity().get(0));
43-
multiInteriorViscosityField->periodicity().toggle(1,cellFields.hemocell.lattice->periodicity().get(1));
44-
multiInteriorViscosityField->periodicity().toggle(2,cellFields.hemocell.lattice->periodicity().get(2));
42+
multiInteriorViscosityField->periodicity().toggle(0,cellFields.hemocell.domain_lattice->periodicity().get(0));
43+
multiInteriorViscosityField->periodicity().toggle(1,cellFields.hemocell.domain_lattice->periodicity().get(1));
44+
multiInteriorViscosityField->periodicity().toggle(2,cellFields.hemocell.domain_lattice->periodicity().get(2));
4545

4646
multiInteriorViscosityField->initialize();
4747

4848
//Make sure each particleField has access to its local scalarField
4949
for (const plint & bId : multiInteriorViscosityField->getLocalInfo().getBlocks()) {
50-
HemoCellParticleField & pf = cellFields.immersedParticles->getComponent(bId);
50+
HemoCellParticleField & pf = cellFields.domain_immersedParticles->getComponent(bId);
5151
pf.interiorViscosityField = &multiInteriorViscosityField->getComponent(bId);
5252
}
5353
}
@@ -119,8 +119,8 @@ namespace hemo {
119119
}
120120

121121
void InteriorViscosityHelper::refillBindingSites() {
122-
for (const plint & bId : cellFields.immersedParticles->getLocalInfo().getBlocks()) {
123-
HemoCellParticleField & pf = cellFields.immersedParticles->getComponent(bId);
122+
for (const plint & bId : cellFields.domain_immersedParticles->getLocalInfo().getBlocks()) {
123+
HemoCellParticleField & pf = cellFields.domain_immersedParticles->getComponent(bId);
124124
ScalarField3D<T> & bf = *pf.interiorViscosityField;
125125
Box3D domain = bf.getBoundingBox();
126126
for (int x = domain.x0; x <= domain.x1 ; x++) {
@@ -139,4 +139,4 @@ namespace hemo {
139139
}
140140
}
141141
}
142-
}
142+
}

‎helper/preInlet.cpp

+4-12
Original file line numberDiff line numberDiff line change
@@ -359,12 +359,7 @@ void PreInlet::applyPreInletVelocityBoundary() {
359359
int z_o = domain.getNz();
360360
int y_o = domain.getNy();
361361
int tag;
362-
363-
plb::Array<T,3> receiver;
364362
plb::Array<T,3> vel;
365-
std::vector<plb::Array<T,3>> buffer;
366-
std::vector<MPI_Request> requests;
367-
368363
for (int bId : hemocell->lattice->getLocalInfo().getBlocks()) {
369364
Box3D bulk = hemocell->lattice->getMultiBlockManagement().getBulk(bId);
370365
if (!intersect(domain,bulk,result)) { continue; }
@@ -382,25 +377,22 @@ void PreInlet::applyPreInletVelocityBoundary() {
382377
int dest = hemocell->domain_lattice_management->getThreadAttribution().getMpiProcess(hemocell->domain_lattice_management->getSparseBlockStructure().locate(x+loc.x,y+loc.y,z+loc.z));
383378

384379
// send velocity from preInlet boundary
385-
buffer.push_back(vel);
386-
requests.push_back(MPI_Request());
387-
MPI_Isend(&buffer.back()[0],3*sizeof(T),MPI_CHAR,dest,tag,MPI_COMM_WORLD,&requests.back());
380+
MPI_Send(&vel[0],3*sizeof(T),MPI_CHAR,dest,tag,MPI_COMM_WORLD);
388381
} else {
389382
Box3D point(x,x,y,y,z,z);
390383
int source = hemocell->preinlet_lattice_management->getThreadAttribution().getMpiProcess(hemocell->preinlet_lattice_management->getSparseBlockStructure().locate(x+loc.x,y+loc.y,z+loc.z));
391384

392385
// receive velocity from preInlet
393-
MPI_Recv(&receiver,3*sizeof(T),MPI_CHAR,source,tag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
386+
MPI_Recv(&vel[0],3*sizeof(T),MPI_CHAR,source,tag,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
394387

395388
// update velocity lattice main domain
396-
setBoundaryVelocity(hemocell->lattice->getComponent(bId),point,receiver);
389+
setBoundaryVelocity(hemocell->lattice->getComponent(bId),point,vel);
397390
}
398391
}
399392
}
400393
}
401394
}
402395
}
403-
MPI_Waitall(requests.size(), &requests[0], MPI_STATUS_IGNORE);
404396
global.statistics.getCurrent().stop();
405397
}
406398

@@ -996,4 +988,4 @@ void PreInlet::createBoundary() {
996988
PreInlet::CreatePreInletBoundingBox * PreInlet::CreatePreInletBoundingBox::clone() const { return new PreInlet::CreatePreInletBoundingBox(*this);}
997989
PreInlet::CreateDrivingForceFunctional * PreInlet::CreateDrivingForceFunctional::clone() const { return new PreInlet::CreateDrivingForceFunctional(*this);}
998990
PreInlet::FillFlagMatrix * PreInlet::FillFlagMatrix::clone() const { return new PreInlet::FillFlagMatrix(*this);}
999-
}
991+
}

0 commit comments

Comments
 (0)