Feature: boundary condition handler MPI communication

src/t8.h

@@ -161,6 +161,7 @@ typedef enum {
   T8_MPI_CMESH_UNIFORM_BOUNDS_END,      /**< Used for cmesh uniform bounds computation. */
   T8_MPI_TEST_ELEMENT_PACK_TAG,         /**< Used for testing mpi pack and unpack functionality */
   T8_MPI_PFC_TAG,                       /**< Used for data exchange during partition for coarsening. */
+  T8_MPI_BOUNDARY_CONDITION_SYNC_TAG,   /**< Used for the global synchronization of boundary conditions. */
   T8_MPI_TAG_LAST                       /**< Dummy last MPI tag. */
 } t8_MPI_tag_t;
 

src/t8_cmesh/t8_cmesh_boundary_conditions/internal/t8_cmesh_boundary_condition_handler.cxx

@@ -25,11 +25,14 @@
  * Implementation context of \ref t8_cmesh_boundary_condition_handler.hxx
  */
 
+#include <t8.h>
 #include <t8_cmesh/t8_cmesh_boundary_conditions/internal/t8_cmesh_boundary_condition_handler.hxx>
 #include <t8_cmesh/t8_cmesh_boundary_conditions/internal/t8_cmesh_boundary_condition_handler_types.h>
 #include <t8_cmesh/t8_cmesh_internal/t8_cmesh_types.h>
 #include <t8_cmesh/t8_cmesh_internal/t8_cmesh_stash.h>
 
+#include <cstring>
+
 using namespace detail;
 
 #if T8_ENABLE_DEBUG
@@ -91,3 +94,121 @@ t8_cmesh_boundary_condition_handler::get_boundary_condition_attribute_key () con
 {
   return T8_CMESH_BOUNDARY_CONDITION_ATTRIBUTE_KEY;
 }
+
+std::vector<char>
+t8_cmesh_boundary_condition_handler::serialize_map () const
+{
+  std::vector<char> serial_data;
+  /* Fill serial_data with strings only. The hashes can be re-generated locally. */
+  for (const auto &[key, string] : t8_cmesh_boundary_condition_handler::m_boundary_conditions) {
+    serial_data.insert (serial_data.end (), string.begin (), string.end ());
+    /* Null terminate strings to be able to split them again later. */
+    serial_data.push_back ('\0');
+  }
+  serial_data.shrink_to_fit ();
+  return serial_data;
+}
+
+void
+t8_cmesh_boundary_condition_handler::unpack_map (std::vector<char> &serial_data, bool overwrite)
+{
+  if (overwrite) {
+    m_boundary_conditions.clear ();
+  }
+
+  /* Iterate over stings. */
+  for (const char *string = serial_data.data (); string < serial_data.data () + serial_data.size ();
+       string += std::strlen (string) + 1) {
+    /* Interpret c string as std::string, rehash it and insert it. */
+    std::string value (string);
+    t8_cmesh_boundary_condition_handler::m_boundary_conditions.try_emplace (
+      t8_cmesh_boundary_condition_handler::hash_boundary_condition_name (value), std::move (value));
+  }
+}
+
+void
+t8_cmesh_boundary_condition_handler::synchronize (sc_MPI_Comm comm)
+{
+  /* Use a bottom-up binomial tree merge approach instead of an MPI_Allgatherv to secure O(log(p)) scaling.
+   * In every level of the merge tree each rank with rank = rank & ~mask merges all information of rank = rank | mask.
+   * The rank = rank && ~mask then drops out of the communication pattern.
+   * The receiving rank also checks if there is a sender in the first place (src >= mpisize) for non-power of 2 mpisizes.
+   *
+   * After all data is collected, rank 0 broadcasts the collected data.
+   */
+
+  T8_ASSERT (comm != sc_MPI_COMM_NULL);
+
+  int rank, mpisize;
+  sc_MPI_Comm_rank (comm, &rank);
+  sc_MPI_Comm_size (comm, &mpisize);
+
+  if (mpisize == 1) {
+    /* Nothing to do. */
+    return;
+  }
+
+  /* Iterate over all levels of the merge tree. */
+  for (int mask = 1; mask < mpisize; mask <<= 1) {
+    /* This rank receives a message. */
+    if ((rank & mask) == 0) {
+      const int src = rank | mask;
+      if (src >= mpisize) {
+        /* There is no sender, so we have nothing to do on this level. */
+        continue;
+      }
+      /* Probe the size of the incoming message. */
+      sc_MPI_Status status;
+      sc_MPI_Probe (src, T8_MPI_BOUNDARY_CONDITION_SYNC_TAG, comm, &status);
+      int incoming_bytes;
+      sc_MPI_Get_count (&status, sc_MPI_BYTE, &incoming_bytes);
+
+      /* Prepare buffer and receive data. */
+      std::vector<char> incoming (incoming_bytes);
+      sc_MPI_Recv (incoming.data (), incoming_bytes, sc_MPI_BYTE, src, T8_MPI_BOUNDARY_CONDITION_SYNC_TAG, comm,
+                   sc_MPI_STATUS_IGNORE);
+      t8_cmesh_boundary_condition_handler::unpack_map (incoming, false);
+    }
+    /* This process sends a message and then drops out. */
+    else {
+      const int dst = rank & ~mask;
+      const std::vector<char> outgoing = serialize_map ();
+      sc_MPI_Send (const_cast<char *> (outgoing.data ()), static_cast<int> (outgoing.size ()), sc_MPI_BYTE, dst,
+                   T8_MPI_BOUNDARY_CONDITION_SYNC_TAG, comm);
+      /* Drop out. */
+      break;
+    }
+  }
+
+  t8_cmesh_boundary_condition_handler::bcast (0, comm);
+}
+
+void
+t8_cmesh_boundary_condition_handler::bcast (int main_rank, sc_MPI_Comm comm)
+{
+  int rank;
+  sc_MPI_Comm_rank (comm, &rank);
+
+  /* Buffer for sending and receiving */
+  std::vector<char> buffer;
+
+  /* Serialize data. */
+  if (rank == main_rank) {
+    buffer = t8_cmesh_boundary_condition_handler::serialize_map ();
+  }
+
+  /* Prepare buffer length. */
+  int size = static_cast<int> (buffer.size ());
+  sc_MPI_Bcast (&size, 1, sc_MPI_INT, main_rank, comm);
+  if (rank != main_rank) {
+    buffer.resize (size);
+  }
+
+  /* Broadcast data */
+  sc_MPI_Bcast (buffer.data (), size, sc_MPI_BYTE, main_rank, comm);
+
+  /* Add data to local map. */
+  if (rank != main_rank) {
+    t8_cmesh_boundary_condition_handler::unpack_map (buffer, true);
+  }
+}

src/t8_cmesh/t8_cmesh_boundary_conditions/internal/t8_cmesh_boundary_condition_handler.hxx

@@ -73,6 +73,8 @@ struct t8_cmesh_boundary_condition_handler
   using boundary_condition_hash = T8Type<size_t, boundary_condition_hash_tag, EqualityComparable>;
 
  public:
+  /**************************************** CONSTRUCTORS & ASSIGNMENT OPERATORS ****************************************/
+
   /**
    * Standard constructor. Associates the handler with a cmesh
    * \param [in] cmesh
@@ -81,6 +83,42 @@ struct t8_cmesh_boundary_condition_handler
   {
   }
 
+  /**
+   * Copy constructor.
+   * \param [in]  other  The other.
+   */
+  t8_cmesh_boundary_condition_handler (const t8_cmesh_boundary_condition_handler &other) = default;
+
+  /**
+   * Move constructor.
+   * \param [in]  other  The other.
+   */
+  t8_cmesh_boundary_condition_handler (t8_cmesh_boundary_condition_handler &&other) noexcept = default;
+
+  /**
+   * Copy assignment operator.
+   * \param [in]  other  The other.
+   * \return      A copy of this.
+   */
+  t8_cmesh_boundary_condition_handler &
+  operator= (const t8_cmesh_boundary_condition_handler &other)
+    = default;
+
+  /**
+   * Move assignment operator.
+   * \param [in]  other  The other.
+   * \return      A reference to a moved version of this.
+   */
+  t8_cmesh_boundary_condition_handler &
+  operator= (t8_cmesh_boundary_condition_handler &&other) noexcept
+    = default;
+
+  /**
+   * The destructor.
+   */
+  ~t8_cmesh_boundary_condition_handler () = default;
+
+  /**************************************** BOUNDARY CONDITION SETUP ****************************************/
   /**
    * Applies boundary conditions to the faces of a cmesh cell.
    *
@@ -98,13 +136,61 @@ struct t8_cmesh_boundary_condition_handler
     hashes.reserve (std::size (boundary_conditions));
     for (const auto &boundary_condition : boundary_conditions) {
       const boundary_condition_hash hash = hash_boundary_condition_name (boundary_condition);
-      m_boundary_conditions.try_emplace (hash, boundary_condition);
+      const auto inserted = m_boundary_conditions.try_emplace (hash, boundary_condition);
+#if T8_ENABLE_DEBUG
+      if (inserted.second) {
+        const std::string_view boundary_condition_view = boundary_condition;
+        t8_debugf ("Registered boundary condition %.*s\n", static_cast<int> (boundary_condition_view.size ()),
+                   boundary_condition_view.data ());
+      }
+#endif
       hashes.emplace_back (std::move (hash));
     }
     t8_cmesh_set_attribute (m_cmesh, gtreeid, t8_get_package_id (), get_boundary_condition_attribute_key (),
                             hashes.data (), sizeof (boundary_condition_hash) * hashes.size (), 0);
   }
 
+  /**
+   * Adds a boundary condition name to this handler without registering it to a tree.
+   * Mostly needed for debugging and testing reasons.
+   * Boundary conditions added via \ref add_boundary_conditions do not need to be registered explicitly.
+   * \tparam TString                  A string-like object.
+   * \param [in]  boundary_condition  The name of the boundary condition.
+   */
+  template <typename TString>
+    requires std::convertible_to<TString, std::string_view>
+  inline void
+  register_boundary_condition (TString &&boundary_condition)
+  {
+    const std::string boundary_condition_string { boundary_condition };
+    const boundary_condition_hash hash = hash_boundary_condition_name (boundary_condition_string);
+
+    const auto inserted = m_boundary_conditions.try_emplace (hash, std::move (boundary_condition_string));
+
+#if T8_ENABLE_DEBUG
+    if (inserted.second) {
+      const std::string_view boundary_condition_view = boundary_condition;
+      t8_debugf ("Registered boundary condition %.*s\n", static_cast<int> (boundary_condition_view.size ()),
+                 boundary_condition_view.data ());
+    }
+#endif
+  }
+
+  /**
+   * Updates the internal cmesh. The boundary condition handler can only be given to uncommitted cmeshes.
+   * \param [in] new_cmesh  The new cmesh.
+   */
+  inline void
+  set_cmesh (t8_cmesh_t new_cmesh)
+  {
+    T8_ASSERT (t8_cmesh_is_initialized (new_cmesh));
+    T8_ASSERTF (t8_cmesh_is_committed (new_cmesh, 0),
+                "The boundary condition handler can only be set for uncommitted cmeshes.\n");
+    m_cmesh = new_cmesh;
+  }
+
+  /**************************************** BOUNDARY CONDITION RETRIEVAL ****************************************/
+
   /**
    * Retrieves the boundary conditions of a cmesh cell.
    *
@@ -195,6 +281,23 @@ struct t8_cmesh_boundary_condition_handler
     return std::nullopt;
   }
 
+  /**
+   * Get all registered boundary condition names.
+   * \return A vector containing all registered boundary condition names.
+   */
+  inline std::vector<std::string_view>
+  get_registered_boundary_conditions () const
+  {
+    std::vector<std::string_view> boundary_conditions;
+    boundary_conditions.reserve (m_boundary_conditions.size ());
+    for (const auto &boundary_condition : m_boundary_conditions) {
+      boundary_conditions.push_back (boundary_condition.second);
+    }
+    return boundary_conditions;
+  }
+
+  /**************************************** HELPER FUNCTIONS ****************************************/
+
 #if T8_ENABLE_DEBUG
   /** Verifies the proper attribution of boundary conditions. Can only be called on a cmesh
    * during commit.
@@ -267,6 +370,45 @@ struct t8_cmesh_boundary_condition_handler
     return m_boundary_conditions.at (hash);
   }
 
+  /**************************************** MPI HELPER FUNCTIONS ****************************************/
+
+ public:
+  /**
+   * Synchronizes the contents of the boundary condition handler across all processes.
+   * \param [in]  comm      The communicator to use.
+   */
+  void
+  synchronize (sc_MPI_Comm comm);
+
+  /**
+   * Broadcasts the boundary conditions from \a main_rank to all other ranks.
+   * \param [in]  main_rank The main rank from which to broadcast.
+   * \param [in]  comm      The communicator to use.
+   */
+  void
+  bcast (int main_rank, sc_MPI_Comm comm);
+
+  /**
+   * Converts the contents of m_boundary_conditions into a serial vector of chars.
+   * The keys are omitted and only the strings are serialized.
+   * In the serialized vector, the individual strings are null-terminated.
+   * \return The serialized map.
+   */
+  std::vector<char>
+  serialize_map () const;
+
+  /**
+   * Unpacks and integrates the \a serial_data into m_boundary_conditions.
+   * It either merges the already existing data or overwrites the complete map if \a overwrite is set to true.
+   * \param [in]  serial_data   The data to unpack and integrate.
+   * \param [in]  overwrite     Overwrites the data in this handler if true. Merges the data with the existing data on false.
+   */
+  void
+  unpack_map (std::vector<char> &serial_data, bool overwrite);
+
+  /**************************************** MEMBERS ****************************************/
+
+ private:
   /** The associated cmesh of this struct */
   t8_cmesh_t m_cmesh;
 

src/t8_cmesh/t8_cmesh_internal/t8_cmesh_commit.cxx

@@ -36,6 +36,7 @@
 #include <t8_cmesh/t8_cmesh_geometry.hxx>
 #include <t8_geometry/t8_geometry_handler.hxx>
 #include <t8_cmesh/t8_cmesh_vertex_connectivity/t8_cmesh_vertex_connectivity.hxx>
+#include <t8_cmesh/t8_cmesh_boundary_conditions/internal/t8_cmesh_boundary_condition_handler.hxx>
 
 /**
  * A struct to hold the information about a ghost facejoin.
@@ -583,6 +584,15 @@ t8_cmesh_commit (t8_cmesh_t cmesh, sc_MPI_Comm comm)
       cmesh->geometry_handler->ref ();
     }
 
+    /* Copy the boundary condition handler if available. */
+    T8_ASSERT (cmesh->boundary_condition_handler == nullptr);
+    if (cmesh->set_from->boundary_condition_handler != nullptr) {
+      cmesh->boundary_condition_handler
+        = new detail::t8_cmesh_boundary_condition_handler (*cmesh->set_from->boundary_condition_handler);
+      /* Assign handler to new cmesh. */
+      cmesh->boundary_condition_handler->set_cmesh (cmesh);
+    }
+
 #if T8_ENABLE_DEBUG
     /* Copy negative volume check from set_from */
     cmesh->negative_volume_check = cmesh->set_from->negative_volume_check;

test/CMakeLists.txt

@@ -139,7 +139,7 @@ add_t8_cpp_test( NAME t8_gtest_compute_first_element_serial                 SOUR
 add_t8_cpp_test( NAME t8_gtest_multiple_attributes_parallel                 SOURCES t8_cmesh/t8_gtest_multiple_attributes.cxx )
 add_t8_cpp_test( NAME t8_gtest_attribute_gloidx_array_serial                SOURCES t8_cmesh/t8_gtest_attribute_gloidx_array.cxx )
 add_t8_cpp_test( NAME t8_gtest_cmesh_bounding_box_serial                    SOURCES t8_cmesh/t8_gtest_cmesh_bounding_box.cxx )
-add_t8_cpp_test( NAME t8_gtest_cmesh_boundary_conditions_serial             SOURCES t8_cmesh/t8_gtest_cmesh_boundary_conditions.cxx )
+add_t8_cpp_test( NAME t8_gtest_cmesh_boundary_conditions_parallel           SOURCES t8_cmesh/t8_gtest_cmesh_boundary_conditions.cxx )
 
 add_t8_cpp_test( NAME t8_gtest_shmem_parallel  SOURCES t8_data/t8_gtest_shmem.cxx )
 add_t8_cpp_test( NAME t8_gtest_data_pack_parallel       SOURCES t8_data/t8_gtest_data_handler.cxx t8_data/t8_data_handler_specs.cxx)