VTK  9.2.5
vtkPStructuredGridConnectivity.h
Go to the documentation of this file.
1 /*=========================================================================
2 
3  Program: Visualization Toolkit
4  Module: vtkPStructuredGridConnectivity.h
5 
6  Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen
7  All rights reserved.
8  See Copyright.txt or http://www.kitware.com/Copyright.htm for details.
9 
10  This software is distributed WITHOUT ANY WARRANTY; without even
11  the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
12  PURPOSE. See the above copyright notice for more information.
13 
14  =========================================================================*/
34 #ifndef vtkPStructuredGridConnectivity_h
35 #define vtkPStructuredGridConnectivity_h
36 
37 // VTK include directives
38 #include "vtkFiltersParallelGeometryModule.h" // For export macro
39 #include "vtkMPICommunicator.h" // Needed for vtkMPICommunicator::Request
41 
42 // C++ include directives
43 #include <vector> // For STL vector
44 
45 // Forward declarations
47 class vtkMPIController;
49 // class vtkMPICommunicator::Request;
50 
51 class VTKFILTERSPARALLELGEOMETRY_EXPORT vtkPStructuredGridConnectivity
53 {
54 public:
57  void PrintSelf(ostream& os, vtkIndent indent) override;
58 
60 
64  vtkGetObjectMacro(Controller, vtkMultiProcessController);
66 
70  void SetNumberOfGrids(const unsigned int N) override;
71 
75  void RegisterGrid(const int gridID, int extents[6], vtkUnsignedCharArray* nodesGhostArray,
76  vtkUnsignedCharArray* cellGhostArray, vtkPointData* pointData, vtkCellData* cellData,
77  vtkPoints* gridNodes) override;
78 
83  int GetNumberOfLocalGrids() { return static_cast<int>(this->GridIds.size()); }
84 
91  int GetGridRank(const int gridID);
92 
96  bool IsGridRemote(const int gridID);
97 
101  bool IsGridLocal(const int gridID);
102 
108  void Initialize();
109 
115  void ComputeNeighbors() override;
116 
121  void CreateGhostLayers(const int N = 1) override;
122 
123 protected:
126 
128  int Rank;
130 
131  std::vector<int> GridRanks; // Corresponding rank for each grid
132  std::vector<int> GridIds; // List of GridIds, owned by this process
133 
134  // Data structures to store the remote ghost data of each grid for each one
135  // of its neighbors. The first index is the global grid index. The second
136  // is the neighbor index.
137  std::vector<std::vector<vtkPoints*>> RemotePoints;
138  std::vector<std::vector<vtkPointData*>> RemotePointData;
139  std::vector<std::vector<vtkCellData*>> RemoteCellData;
140 
141  // Data structures to store the send/receive buffer sizes and corresponding
142  // persistent buffers. The first index is the global grid index. The second
143  // index is the neighbor index for the given grid.
144  std::vector<std::vector<unsigned int>> SendBufferSizes;
145  std::vector<std::vector<unsigned int>> RcvBufferSizes;
146  std::vector<std::vector<unsigned char*>> SendBuffers;
147  std::vector<std::vector<unsigned char*>> RcvBuffers;
148 
152 
153  // Array of MPI requests
155 
159  bool GridExtentsAreEqual(int rhs[6], int lhs[6]);
160 
164  bool HasPointData(const int gridIdx);
165 
169  bool HasCellData(const int gridIdx);
170 
174  bool HasPoints(const int gridIdx);
175 
179  void InitializeMessageCounters();
180 
185  void ClearRemoteData();
186 
190  void ClearRawBuffers();
191 
196  void RegisterRemoteGrid(const int gridID, int extents[6], int process);
197 
203  const int gridIdx, const int nei, const vtkStructuredNeighbor& Neighbor);
204 
210  void TransferGhostDataFromNeighbors(const int gridID) override;
211 
216 
222 
227  void DeserializeBufferSizesForProcess(int* buffersizes, vtkIdType N, const int processId);
228 
233  void SerializeBufferSizes(int*& sizesbuf, vtkIdType& N);
234 
240 
247 
252  void PostReceives();
253 
258  void PostSends();
259 
266 
273 
278 
283  void SerializeGhostPoints(const int gridIdx, int ext[6], vtkMultiProcessStream& bytestream);
284 
289 
294  void SerializeFieldData(int sourceExtent[6], int targetExtent[6], vtkFieldData* fieldData,
295  vtkMultiProcessStream& bytestream);
296 
301  void SerializeGhostPointData(const int gridIdx, int ext[6], vtkMultiProcessStream& bytestream);
302 
307  void SerializeGhostCellData(const int gridIdx, int ext[6], vtkMultiProcessStream& bytestream);
308 
314  const int gridIdx, const int nei, int ext[6], vtkMultiProcessStream& bytestream);
315 
319  void DeserializeDataArray(vtkDataArray*& dataArray, const int dataType, const int numberOfTuples,
320  const int numberOfComponents, vtkMultiProcessStream& bytestream);
321 
326  void DeserializeFieldData(int ext[6], vtkFieldData* fieldData, vtkMultiProcessStream& bytestream);
327 
333  const int gridIdx, const int nei, int ext[6], vtkMultiProcessStream& bytestream);
334 
340  const int gridIdx, const int nei, int ext[6], vtkMultiProcessStream& bytestream);
341 
348  void SerializeGhostData(const int sndGridID, const int rcvGrid, int sndext[6],
349  unsigned char*& buffer, unsigned int& size);
350 
355  void DeserializeGhostData(const int gridID, const int neiListID, const int neiGridIdx,
356  int rcvext[6], unsigned char* buffer, unsigned int size);
357 
363 
368  void SerializeGridExtents(int*& sndbuffer, vtkIdType& N);
369 
374  void DeserializeGridExtentForProcess(int* rcvbuffer, vtkIdType& N, const int processId);
375 
376 private:
378  void operator=(const vtkPStructuredGridConnectivity&) = delete;
379 };
380 
381 //=============================================================================
382 // INLINE METHODS
383 //=============================================================================
384 
385 inline bool vtkPStructuredGridConnectivity::GridExtentsAreEqual(int rhs[6], int lhs[6])
386 {
387  for (int i = 0; i < 6; ++i)
388  {
389  if (rhs[i] != lhs[i])
390  {
391  return false;
392  }
393  }
394  return true;
395 }
396 
397 //------------------------------------------------------------------------------
398 inline bool vtkPStructuredGridConnectivity::HasPointData(const int gridIdx)
399 {
400  // Sanity check
401  assert("pre: grid index is out-of-bounds!" && (gridIdx >= 0) &&
402  (gridIdx < static_cast<int>(this->NumberOfGrids)));
403 
404  if ((this->GridPointData[gridIdx] != nullptr) &&
405  (this->GridPointData[gridIdx]->GetNumberOfArrays() > 0))
406  {
407  return true;
408  }
409  return false;
410 }
411 
412 //------------------------------------------------------------------------------
413 inline bool vtkPStructuredGridConnectivity::HasCellData(const int gridIdx)
414 {
415  // Sanity check
416  assert("pre: grid index is out-of-bounds!" && (gridIdx >= 0) &&
417  (gridIdx < static_cast<int>(this->NumberOfGrids)));
418 
419  if ((this->GridCellData[gridIdx] != nullptr) &&
420  (this->GridCellData[gridIdx]->GetNumberOfArrays() > 0))
421  {
422  return true;
423  }
424  return false;
425 }
426 
427 //------------------------------------------------------------------------------
428 inline bool vtkPStructuredGridConnectivity::HasPoints(const int gridIdx)
429 {
430  // Sanity check
431  assert("pre: grid index is out-of-bounds!" && (gridIdx >= 0) &&
432  (gridIdx < static_cast<int>(this->NumberOfGrids)));
433 
434  if (this->GridPoints[gridIdx] != nullptr)
435  {
436  return true;
437  }
438  return false;
439 }
440 
441 //------------------------------------------------------------------------------
443 {
444  this->TotalNumberOfMsgs = this->TotalNumberOfRcvs = this->TotalNumberOfSends = 0;
445 }
446 
447 //------------------------------------------------------------------------------
449 {
450  this->SendBufferSizes.clear();
451  this->RcvBufferSizes.clear();
452 
453  // STEP 0: Clear send buffers
454  for (unsigned int i = 0; i < this->SendBuffers.size(); ++i)
455  {
456  for (unsigned int j = 0; j < this->SendBuffers[i].size(); ++j)
457  {
458  delete[] this->SendBuffers[i][j];
459  } // END for all neighbors
460  this->SendBuffers[i].clear();
461  } // END for all grids
462  this->SendBuffers.clear();
463 
464  // STEP 1: Clear rcv buffers
465  for (unsigned int i = 0; i < this->RcvBuffers.size(); ++i)
466  {
467  for (unsigned int j = 0; j < this->RcvBuffers[i].size(); ++j)
468  {
469  delete[] this->RcvBuffers[i][j];
470  } // END for all neighbors
471  this->RcvBuffers[i].clear();
472  } // END for all grids
473  this->RcvBuffers.clear();
474 }
475 
476 //------------------------------------------------------------------------------
478 {
479  // STEP 0: Clear remote points
480  for (unsigned int i = 0; i < this->RemotePoints.size(); ++i)
481  {
482  for (unsigned int j = 0; j < this->RemotePoints[i].size(); ++j)
483  {
484  if (this->RemotePoints[i][j] != nullptr)
485  {
486  this->RemotePoints[i][j]->Delete();
487  }
488  } // END for all j
489  this->RemotePoints[i].clear();
490  } // END for all i
491  this->RemotePoints.clear();
492 
493  // STEP 1: Clear remote point data
494  for (unsigned int i = 0; i < this->RemotePointData.size(); ++i)
495  {
496  for (unsigned int j = 0; j < this->RemotePointData[i].size(); ++j)
497  {
498  if (this->RemotePointData[i][j] != nullptr)
499  {
500  this->RemotePointData[i][j]->Delete();
501  }
502  } // END for all j
503  this->RemotePointData[i].clear();
504  } // END for all i
505  this->RemotePointData.clear();
506 
507  // STEP 2: Clear remote cell data
508  for (unsigned int i = 0; i < this->RemoteCellData.size(); ++i)
509  {
510  for (unsigned int j = 0; j < this->RemoteCellData[i].size(); ++j)
511  {
512  if (this->RemoteCellData[i][j] != nullptr)
513  {
514  this->RemoteCellData[i][j]->Delete();
515  }
516  } // END for all j
517  this->RemoteCellData[i].clear();
518  }
519  this->RemoteCellData.clear();
520 }
521 
522 //------------------------------------------------------------------------------
523 inline bool vtkPStructuredGridConnectivity::IsGridRemote(const int gridID)
524 {
525  return (!this->IsGridLocal(gridID));
526 }
527 
528 //------------------------------------------------------------------------------
529 inline bool vtkPStructuredGridConnectivity::IsGridLocal(const int gridID)
530 {
531  assert("pre: Instance has not been initialized!" && this->Initialized);
532  assert("pre: gridID is out-of-bounds" && (gridID >= 0) &&
533  (gridID < static_cast<int>(this->NumberOfGrids)));
534  assert(
535  "pre: GridRanks is not properly allocated" && this->NumberOfGrids == this->GridRanks.size());
536  return ((this->GridRanks[gridID] == this->Rank));
537 }
538 
539 //------------------------------------------------------------------------------
541 {
542  assert("pre: Instance has not been initialized!" && this->Initialized);
543  assert("pre: gridID out-of-bounds!" &&
544  (gridID >= 0 && gridID < static_cast<int>(this->NumberOfGrids)));
545  return (this->GridRanks[gridID]);
546 }
547 #endif /* vtkPStructuredGridConnectivity_h */
std::vector< vtkCellData * > GridCellData
std::vector< vtkPointData * > GridPointData
std::vector< vtkPoints * > GridPoints
represent and manipulate cell attribute data
Definition: vtkCellData.h:151
abstract superclass for arrays of numeric data
Definition: vtkDataArray.h:165
represent and manipulate fields of data
Definition: vtkFieldData.h:172
a simple class to control print indentation
Definition: vtkIndent.h:119
Process communication using MPI.
Multiprocessing communication superclass.
stream used to pass data across processes using vtkMultiProcessController.
vtkPStructuredGridConnectivity inherits from vtkStructuredGridConnectivity and implements functionali...
int GetNumberOfLocalGrids()
Returns the number of local grids registers by the process that owns the current vtkPStructuredGridCo...
~vtkPStructuredGridConnectivity() override
std::vector< std::vector< unsigned int > > RcvBufferSizes
virtual void SetController(vtkMultiProcessController *)
Set & Get the process controller.
bool GridExtentsAreEqual(int rhs[6], int lhs[6])
Returns true if the two extents are equal, otherwise false.
void DeserializeGhostPoints(const int gridIdx, const int nei, int ext[6], vtkMultiProcessStream &bytestream)
Helper method to de-serialize the ghost points received from a remote process.
void PrintSelf(ostream &os, vtkIndent indent) override
Methods invoked by print to print information about the object including superclasses.
void SerializeDataArray(vtkDataArray *dataArray, vtkMultiProcessStream &bytestream)
Serializes the data array into a bytestream.
void DeserializeFieldData(int ext[6], vtkFieldData *fieldData, vtkMultiProcessStream &bytestream)
Helper method to de-serialize field data.
void DeserializeGhostPointData(const int gridIdx, const int nei, int ext[6], vtkMultiProcessStream &bytestream)
Helper method to de-serialize the ghost point data received from a remote process.
void ClearRawBuffers()
Clears all raw send/rcv buffers.
bool IsGridLocal(const int gridID)
Returns true iff the grid corresponding to the given gridID is local.
void DeserializeGridExtentForProcess(int *rcvbuffer, vtkIdType &N, const int processId)
Deserializes the received grid extent information to the GridExtents internal data-structures.
void ExchangeGhostDataPost()
Helper method for exchanging ghost data.
void DeserializeGhostData(const int gridID, const int neiListID, const int neiGridIdx, int rcvext[6], unsigned char *buffer, unsigned int size)
Given the raw buffer consisting of ghost data, this method deserializes the object and returns the gr...
int GetGridRank(const int gridID)
Returns the rank of the given gridID.
std::vector< std::vector< unsigned char * > > SendBuffers
std::vector< std::vector< vtkPoints * > > RemotePoints
vtkMPICommunicator::Request * MPIRequests
void ExchangeGhostData()
Exchanges ghost data of the grids owned by this process.
static vtkPStructuredGridConnectivity * New()
bool IsGridRemote(const int gridID)
Returns true iff the grid is remote, otherwise false.
void RegisterGrid(const int gridID, int extents[6], vtkUnsignedCharArray *nodesGhostArray, vtkUnsignedCharArray *cellGhostArray, vtkPointData *pointData, vtkCellData *cellData, vtkPoints *gridNodes) override
See vtkStructuredGridConnectivity::RegisterGrid.
void CreateGhostLayers(const int N=1) override
Creates ghost layers on the grids owned by this process using data from both local and remote block n...
void PostSends()
Helper method to communicate ghost data.
void CommunicateGhostData()
Helper method for exchanging ghost data.
void SerializeGhostPointData(const int gridIdx, int ext[6], vtkMultiProcessStream &bytestream)
Helper method to serialize ghost point data.
void SetNumberOfGrids(const unsigned int N) override
Sets the total number of domains distributed among processors.
void ExchangeGridExtents()
Exchanges the grid extents among all processes and fully populates the GridExtents vector.
void TransferGhostDataFromNeighbors(const int gridID) override
This method transfers the fields (point data and cell data) to the ghost extents from the neighboring...
void ClearRemoteData()
Clears all internal VTK data-structures that are used to store the remote ghost data.
void PackGhostData()
Helper method to pack all the ghost data into send buffers.
void ExchangeBufferSizes()
Helper method to exchange buffer sizes.Each process sends the send buffer size of each grid to each o...
void DeserializeGhostCellData(const int gridIdx, const int nei, int ext[6], vtkMultiProcessStream &bytestream)
Helper method to de-serialize the ghost cell data received from a remote process.
void DeserializeBufferSizesForProcess(int *buffersizes, vtkIdType N, const int processId)
Helper method to deserialize the buffer sizes coming from the given process.
bool HasCellData(const int gridIdx)
Returns true iff the grid corresponding to the given ID has cell data.
void RegisterRemoteGrid(const int gridID, int extents[6], int process)
Registers a remote grid with the given grid Id, structured extents and process.
void UnpackGhostData()
Helper method to unpack the raw ghost data from the receive buffers in to the VTK remote point data-s...
void SerializeGridExtents(int *&sndbuffer, vtkIdType &N)
Serializes the grid extents and information in a buffer to send over MPI The data is serialized as fo...
void DeserializeDataArray(vtkDataArray *&dataArray, const int dataType, const int numberOfTuples, const int numberOfComponents, vtkMultiProcessStream &bytestream)
Helper method to deserialize the data array from a bytestream.
void SerializeBufferSizes(int *&sizesbuf, vtkIdType &N)
Helper method to serialize the buffer sizes for the grids of this process to neighboring grids.
std::vector< std::vector< unsigned int > > SendBufferSizes
void ExchangeGhostDataInit()
Helper method for exchanging ghost data.
void SerializeGhostData(const int sndGridID, const int rcvGrid, int sndext[6], unsigned char *&buffer, unsigned int &size)
Given a grid ID and the corresponding send extent, this method serializes the grid and data within th...
void TransferRemoteNeighborData(const int gridIdx, const int nei, const vtkStructuredNeighbor &Neighbor)
This method transfers all the remote neighbor data to the ghosted grid instance of the grid correspon...
std::vector< std::vector< vtkCellData * > > RemoteCellData
void SerializeFieldData(int sourceExtent[6], int targetExtent[6], vtkFieldData *fieldData, vtkMultiProcessStream &bytestream)
Helper method to serialize field data.
void ComputeNeighbors() override
Computes the neighboring topology of a distributed structured grid data-set.
void SerializeGhostPoints(const int gridIdx, int ext[6], vtkMultiProcessStream &bytestream)
Helper method to serialize the ghost points to send to a remote process.
void PostReceives()
Helper method to communicate ghost data.
void Initialize()
Initializes this instance of vtkPStructuredGridConnectivity, essentially, the acquires the local proc...
bool HasPointData(const int gridIdx)
Returns true iff the grid corresponding to the given ID has point data.
void SerializeGhostCellData(const int gridIdx, int ext[6], vtkMultiProcessStream &bytestream)
Helper method to serialize ghost cell data.
void InitializeMessageCounters()
Sets all message counters to 0.
std::vector< std::vector< vtkPointData * > > RemotePointData
bool HasPoints(const int gridIdx)
Returns true iff the grid corresponding to the given ID has points.
std::vector< std::vector< unsigned char * > > RcvBuffers
represent and manipulate point attribute data
Definition: vtkPointData.h:151
represent and manipulate 3D points
Definition: vtkPoints.h:149
vtkStructuredGridConnectivity is a concrete instance of vtkObject that implements functionality for c...
An internal, light-weight class used to store neighbor information.
dynamic, self-adjusting array of unsigned char
@ size
Definition: vtkX3D.h:259
int vtkIdType
Definition: vtkType.h:332