diff --git a/src/t8.h b/src/t8.h index 6528ce6e13..8a0bfd5b0a 100644 --- a/src/t8.h +++ b/src/t8.h @@ -161,6 +161,7 @@ typedef enum { T8_MPI_CMESH_UNIFORM_BOUNDS_END, /**< Used for cmesh uniform bounds computation. */ T8_MPI_TEST_ELEMENT_PACK_TAG, /**< Used for testing mpi pack and unpack functionality */ T8_MPI_PFC_TAG, /**< Used for data exchange during partition for coarsening. */ + T8_MPI_BOUNDARY_CONDITION_SYNC_TAG, /**< Used for the global synchronization of boundary conditions. */ T8_MPI_TAG_LAST /**< Dummy last MPI tag. */ } t8_MPI_tag_t; diff --git a/src/t8_cmesh/t8_cmesh_boundary_conditions/internal/t8_cmesh_boundary_condition_handler.cxx b/src/t8_cmesh/t8_cmesh_boundary_conditions/internal/t8_cmesh_boundary_condition_handler.cxx index 20800df38e..dac349623d 100644 --- a/src/t8_cmesh/t8_cmesh_boundary_conditions/internal/t8_cmesh_boundary_condition_handler.cxx +++ b/src/t8_cmesh/t8_cmesh_boundary_conditions/internal/t8_cmesh_boundary_condition_handler.cxx @@ -25,11 +25,14 @@ * Implementation context of \ref t8_cmesh_boundary_condition_handler.hxx */ +#include #include #include #include #include +#include + using namespace detail; #if T8_ENABLE_DEBUG @@ -91,3 +94,121 @@ t8_cmesh_boundary_condition_handler::get_boundary_condition_attribute_key () con { return T8_CMESH_BOUNDARY_CONDITION_ATTRIBUTE_KEY; } + +std::vector +t8_cmesh_boundary_condition_handler::serialize_map () const +{ + std::vector serial_data; + /* Fill serial_data with strings only. The hashes can be re-generated locally. */ + for (const auto &[key, string] : t8_cmesh_boundary_condition_handler::m_boundary_conditions) { + serial_data.insert (serial_data.end (), string.begin (), string.end ()); + /* Null terminate strings to be able to split them again later. */ + serial_data.push_back ('\0'); + } + serial_data.shrink_to_fit (); + return serial_data; +} + +void +t8_cmesh_boundary_condition_handler::unpack_map (std::vector &serial_data, bool overwrite) +{ + if (overwrite) { + m_boundary_conditions.clear (); + } + + /* Iterate over stings. */ + for (const char *string = serial_data.data (); string < serial_data.data () + serial_data.size (); + string += std::strlen (string) + 1) { + /* Interpret c string as std::string, rehash it and insert it. */ + std::string value (string); + t8_cmesh_boundary_condition_handler::m_boundary_conditions.try_emplace ( + t8_cmesh_boundary_condition_handler::hash_boundary_condition_name (value), std::move (value)); + } +} + +void +t8_cmesh_boundary_condition_handler::synchronize (sc_MPI_Comm comm) +{ + /* Use a bottom-up binomial tree merge approach instead of an MPI_Allgatherv to secure O(log(p)) scaling. + * In every level of the merge tree each rank with rank = rank & ~mask merges all information of rank = rank | mask. + * The rank = rank && ~mask then drops out of the communication pattern. + * The receiving rank also checks if there is a sender in the first place (src >= mpisize) for non-power of 2 mpisizes. + * + * After all data is collected, rank 0 broadcasts the collected data. + */ + + T8_ASSERT (comm != sc_MPI_COMM_NULL); + + int rank, mpisize; + sc_MPI_Comm_rank (comm, &rank); + sc_MPI_Comm_size (comm, &mpisize); + + if (mpisize == 1) { + /* Nothing to do. */ + return; + } + + /* Iterate over all levels of the merge tree. */ + for (int mask = 1; mask < mpisize; mask <<= 1) { + /* This rank receives a message. */ + if ((rank & mask) == 0) { + const int src = rank | mask; + if (src >= mpisize) { + /* There is no sender, so we have nothing to do on this level. */ + continue; + } + /* Probe the size of the incoming message. */ + sc_MPI_Status status; + sc_MPI_Probe (src, T8_MPI_BOUNDARY_CONDITION_SYNC_TAG, comm, &status); + int incoming_bytes; + sc_MPI_Get_count (&status, sc_MPI_BYTE, &incoming_bytes); + + /* Prepare buffer and receive data. */ + std::vector incoming (incoming_bytes); + sc_MPI_Recv (incoming.data (), incoming_bytes, sc_MPI_BYTE, src, T8_MPI_BOUNDARY_CONDITION_SYNC_TAG, comm, + sc_MPI_STATUS_IGNORE); + t8_cmesh_boundary_condition_handler::unpack_map (incoming, false); + } + /* This process sends a message and then drops out. */ + else { + const int dst = rank & ~mask; + const std::vector outgoing = serialize_map (); + sc_MPI_Send (const_cast (outgoing.data ()), static_cast (outgoing.size ()), sc_MPI_BYTE, dst, + T8_MPI_BOUNDARY_CONDITION_SYNC_TAG, comm); + /* Drop out. */ + break; + } + } + + t8_cmesh_boundary_condition_handler::bcast (0, comm); +} + +void +t8_cmesh_boundary_condition_handler::bcast (int main_rank, sc_MPI_Comm comm) +{ + int rank; + sc_MPI_Comm_rank (comm, &rank); + + /* Buffer for sending and receiving */ + std::vector buffer; + + /* Serialize data. */ + if (rank == main_rank) { + buffer = t8_cmesh_boundary_condition_handler::serialize_map (); + } + + /* Prepare buffer length. */ + int size = static_cast (buffer.size ()); + sc_MPI_Bcast (&size, 1, sc_MPI_INT, main_rank, comm); + if (rank != main_rank) { + buffer.resize (size); + } + + /* Broadcast data */ + sc_MPI_Bcast (buffer.data (), size, sc_MPI_BYTE, main_rank, comm); + + /* Add data to local map. */ + if (rank != main_rank) { + t8_cmesh_boundary_condition_handler::unpack_map (buffer, true); + } +} diff --git a/src/t8_cmesh/t8_cmesh_boundary_conditions/internal/t8_cmesh_boundary_condition_handler.hxx b/src/t8_cmesh/t8_cmesh_boundary_conditions/internal/t8_cmesh_boundary_condition_handler.hxx index 167bd3dba4..acbc5ba591 100644 --- a/src/t8_cmesh/t8_cmesh_boundary_conditions/internal/t8_cmesh_boundary_condition_handler.hxx +++ b/src/t8_cmesh/t8_cmesh_boundary_conditions/internal/t8_cmesh_boundary_condition_handler.hxx @@ -73,6 +73,8 @@ struct t8_cmesh_boundary_condition_handler using boundary_condition_hash = T8Type; public: + /**************************************** CONSTRUCTORS & ASSIGNMENT OPERATORS ****************************************/ + /** * Standard constructor. Associates the handler with a cmesh * \param [in] cmesh @@ -81,6 +83,42 @@ struct t8_cmesh_boundary_condition_handler { } + /** + * Copy constructor. + * \param [in] other The other. + */ + t8_cmesh_boundary_condition_handler (const t8_cmesh_boundary_condition_handler &other) = default; + + /** + * Move constructor. + * \param [in] other The other. + */ + t8_cmesh_boundary_condition_handler (t8_cmesh_boundary_condition_handler &&other) noexcept = default; + + /** + * Copy assignment operator. + * \param [in] other The other. + * \return A copy of this. + */ + t8_cmesh_boundary_condition_handler & + operator= (const t8_cmesh_boundary_condition_handler &other) + = default; + + /** + * Move assignment operator. + * \param [in] other The other. + * \return A reference to a moved version of this. + */ + t8_cmesh_boundary_condition_handler & + operator= (t8_cmesh_boundary_condition_handler &&other) noexcept + = default; + + /** + * The destructor. + */ + ~t8_cmesh_boundary_condition_handler () = default; + + /**************************************** BOUNDARY CONDITION SETUP ****************************************/ /** * Applies boundary conditions to the faces of a cmesh cell. * @@ -98,13 +136,61 @@ struct t8_cmesh_boundary_condition_handler hashes.reserve (std::size (boundary_conditions)); for (const auto &boundary_condition : boundary_conditions) { const boundary_condition_hash hash = hash_boundary_condition_name (boundary_condition); - m_boundary_conditions.try_emplace (hash, boundary_condition); + const auto inserted = m_boundary_conditions.try_emplace (hash, boundary_condition); +#if T8_ENABLE_DEBUG + if (inserted.second) { + const std::string_view boundary_condition_view = boundary_condition; + t8_debugf ("Registered boundary condition %.*s\n", static_cast (boundary_condition_view.size ()), + boundary_condition_view.data ()); + } +#endif hashes.emplace_back (std::move (hash)); } t8_cmesh_set_attribute (m_cmesh, gtreeid, t8_get_package_id (), get_boundary_condition_attribute_key (), hashes.data (), sizeof (boundary_condition_hash) * hashes.size (), 0); } + /** + * Adds a boundary condition name to this handler without registering it to a tree. + * Mostly needed for debugging and testing reasons. + * Boundary conditions added via \ref add_boundary_conditions do not need to be registered explicitly. + * \tparam TString A string-like object. + * \param [in] boundary_condition The name of the boundary condition. + */ + template + requires std::convertible_to + inline void + register_boundary_condition (TString &&boundary_condition) + { + const std::string boundary_condition_string { boundary_condition }; + const boundary_condition_hash hash = hash_boundary_condition_name (boundary_condition_string); + + const auto inserted = m_boundary_conditions.try_emplace (hash, std::move (boundary_condition_string)); + +#if T8_ENABLE_DEBUG + if (inserted.second) { + const std::string_view boundary_condition_view = boundary_condition; + t8_debugf ("Registered boundary condition %.*s\n", static_cast (boundary_condition_view.size ()), + boundary_condition_view.data ()); + } +#endif + } + + /** + * Updates the internal cmesh. The boundary condition handler can only be given to uncommitted cmeshes. + * \param [in] new_cmesh The new cmesh. + */ + inline void + set_cmesh (t8_cmesh_t new_cmesh) + { + T8_ASSERT (t8_cmesh_is_initialized (new_cmesh)); + T8_ASSERTF (t8_cmesh_is_committed (new_cmesh, 0), + "The boundary condition handler can only be set for uncommitted cmeshes.\n"); + m_cmesh = new_cmesh; + } + + /**************************************** BOUNDARY CONDITION RETRIEVAL ****************************************/ + /** * Retrieves the boundary conditions of a cmesh cell. * @@ -195,6 +281,23 @@ struct t8_cmesh_boundary_condition_handler return std::nullopt; } + /** + * Get all registered boundary condition names. + * \return A vector containing all registered boundary condition names. + */ + inline std::vector + get_registered_boundary_conditions () const + { + std::vector boundary_conditions; + boundary_conditions.reserve (m_boundary_conditions.size ()); + for (const auto &boundary_condition : m_boundary_conditions) { + boundary_conditions.push_back (boundary_condition.second); + } + return boundary_conditions; + } + + /**************************************** HELPER FUNCTIONS ****************************************/ + #if T8_ENABLE_DEBUG /** Verifies the proper attribution of boundary conditions. Can only be called on a cmesh * during commit. @@ -267,6 +370,45 @@ struct t8_cmesh_boundary_condition_handler return m_boundary_conditions.at (hash); } + /**************************************** MPI HELPER FUNCTIONS ****************************************/ + + public: + /** + * Synchronizes the contents of the boundary condition handler across all processes. + * \param [in] comm The communicator to use. + */ + void + synchronize (sc_MPI_Comm comm); + + /** + * Broadcasts the boundary conditions from \a main_rank to all other ranks. + * \param [in] main_rank The main rank from which to broadcast. + * \param [in] comm The communicator to use. + */ + void + bcast (int main_rank, sc_MPI_Comm comm); + + /** + * Converts the contents of m_boundary_conditions into a serial vector of chars. + * The keys are omitted and only the strings are serialized. + * In the serialized vector, the individual strings are null-terminated. + * \return The serialized map. + */ + std::vector + serialize_map () const; + + /** + * Unpacks and integrates the \a serial_data into m_boundary_conditions. + * It either merges the already existing data or overwrites the complete map if \a overwrite is set to true. + * \param [in] serial_data The data to unpack and integrate. + * \param [in] overwrite Overwrites the data in this handler if true. Merges the data with the existing data on false. + */ + void + unpack_map (std::vector &serial_data, bool overwrite); + + /**************************************** MEMBERS ****************************************/ + + private: /** The associated cmesh of this struct */ t8_cmesh_t m_cmesh; diff --git a/src/t8_cmesh/t8_cmesh_internal/t8_cmesh_commit.cxx b/src/t8_cmesh/t8_cmesh_internal/t8_cmesh_commit.cxx index 289b9a0b3d..6e752480aa 100644 --- a/src/t8_cmesh/t8_cmesh_internal/t8_cmesh_commit.cxx +++ b/src/t8_cmesh/t8_cmesh_internal/t8_cmesh_commit.cxx @@ -36,6 +36,7 @@ #include #include #include +#include /** * A struct to hold the information about a ghost facejoin. @@ -583,6 +584,15 @@ t8_cmesh_commit (t8_cmesh_t cmesh, sc_MPI_Comm comm) cmesh->geometry_handler->ref (); } + /* Copy the boundary condition handler if available. */ + T8_ASSERT (cmesh->boundary_condition_handler == nullptr); + if (cmesh->set_from->boundary_condition_handler != nullptr) { + cmesh->boundary_condition_handler + = new detail::t8_cmesh_boundary_condition_handler (*cmesh->set_from->boundary_condition_handler); + /* Assign handler to new cmesh. */ + cmesh->boundary_condition_handler->set_cmesh (cmesh); + } + #if T8_ENABLE_DEBUG /* Copy negative volume check from set_from */ cmesh->negative_volume_check = cmesh->set_from->negative_volume_check; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index fdc8409356..8eb1989447 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -139,7 +139,7 @@ add_t8_cpp_test( NAME t8_gtest_compute_first_element_serial SOUR add_t8_cpp_test( NAME t8_gtest_multiple_attributes_parallel SOURCES t8_cmesh/t8_gtest_multiple_attributes.cxx ) add_t8_cpp_test( NAME t8_gtest_attribute_gloidx_array_serial SOURCES t8_cmesh/t8_gtest_attribute_gloidx_array.cxx ) add_t8_cpp_test( NAME t8_gtest_cmesh_bounding_box_serial SOURCES t8_cmesh/t8_gtest_cmesh_bounding_box.cxx ) -add_t8_cpp_test( NAME t8_gtest_cmesh_boundary_conditions_serial SOURCES t8_cmesh/t8_gtest_cmesh_boundary_conditions.cxx ) +add_t8_cpp_test( NAME t8_gtest_cmesh_boundary_conditions_parallel SOURCES t8_cmesh/t8_gtest_cmesh_boundary_conditions.cxx ) add_t8_cpp_test( NAME t8_gtest_shmem_parallel SOURCES t8_data/t8_gtest_shmem.cxx ) add_t8_cpp_test( NAME t8_gtest_data_pack_parallel SOURCES t8_data/t8_gtest_data_handler.cxx t8_data/t8_data_handler_specs.cxx) diff --git a/test/t8_cmesh/t8_gtest_cmesh_boundary_conditions.cxx b/test/t8_cmesh/t8_gtest_cmesh_boundary_conditions.cxx index aac4d4836d..f7181d3fb1 100644 --- a/test/t8_cmesh/t8_gtest_cmesh_boundary_conditions.cxx +++ b/test/t8_cmesh/t8_gtest_cmesh_boundary_conditions.cxx @@ -324,3 +324,104 @@ TEST (t8_gtest_cmesh_boundary_conditions, test_boundary_condition_c_interface) t8_forest_unref (&forest); } + +/** + * Checks if the registered boundary conditions of the \a handler match with some \a testing_conditions. + * \param [in] handler The handler to check. + * \param [in] testing_conditions The conditions which should be registered in \a handler. + */ +static void +check_boundary_conditions (detail::t8_cmesh_boundary_condition_handler &handler, + std::vector testing_conditions) +{ + auto retrieved_conditions = handler.get_registered_boundary_conditions (); + ASSERT_EQ (retrieved_conditions.size (), testing_conditions.size ()); + + std::ranges::sort (retrieved_conditions); + std::ranges::sort (testing_conditions); + for (size_t i_bc = 0; i_bc < retrieved_conditions.size (); ++i_bc) { + EXPECT_EQ (retrieved_conditions[i_bc], testing_conditions[i_bc]); + } +} + +/** + * This test registers boundary conditions on rank 0 and broadcasts them to the other ranks. + * In the end we check, if every rank hast the boundary conditions. + */ +TEST (t8_gtest_cmesh_boundary_conditions, test_boundary_condition_broadcast) +{ + /* Get MPI info. */ + sc_MPI_Comm comm = sc_MPI_COMM_WORLD; + int rank; + sc_MPI_Comm_rank (comm, &rank); + + /* We create the same vector with 20 boundary conditions on each rank. */ + std::vector testing_boundary_conditions; + size_t num_boundary_conditions = 20; + testing_boundary_conditions.reserve (num_boundary_conditions); + for (size_t i_bc = 0; i_bc < num_boundary_conditions; ++i_bc) { + testing_boundary_conditions.emplace_back ("testing_boundary_condition_" + std::to_string (i_bc)); + } + + /* We create a handler and register the boundary conditions only on rank 0. */ + detail::t8_cmesh_boundary_condition_handler handler (nullptr); + if (rank == 0) { + for (const auto &boundary_condition : testing_boundary_conditions) { + handler.register_boundary_condition (boundary_condition); + } + } + + /* Broadcast the conditions. */ + handler.bcast (0, comm); + + /* Check if every rank has all conditions. */ + check_boundary_conditions (handler, testing_boundary_conditions); +} + +/** + * This test creates boundary conditions for each rank. + * Then we synchronize all bcs and check that all boundary conditions are available + * on all ranks. + */ +TEST (t8_gtest_cmesh_boundary_conditions, test_boundary_condition_synchronize) +{ + /* Get MPI info */ + sc_MPI_Comm comm = sc_MPI_COMM_WORLD; + int rank, mpisize; + sc_MPI_Comm_rank (comm, &rank); + sc_MPI_Comm_size (comm, &mpisize); + + /* Create the boundary conditions. + * For each rank, we create 6 bcs. + * The last rank gets no boundary conditions as a special testing case. + * Each rank gets his own 6 boundary conditions as well as 2 boundary conditions + * of the next rank. This way each rank (except the first, last and second to last) + * has 2 shared bcs with rank - 1, 2 unique bcs and 2 shared bcs with rank + 1. + * Rank 0 and mpisize - 2 have 4 unique bcs and 2 shared ones (if there are more than 2 ranks). + * Rank mpisize - 1 has no boundary conditions. */ + std::vector testing_boundary_conditions; + const size_t num_boundary_conditions_per_rank = 6; + /* Since the last rank gets no bcs we use mpisize - 1 and since every rank gets 2 additional bcs we also add 2. */ + size_t num_boundary_conditions = (mpisize - 1) * num_boundary_conditions_per_rank + 2; + testing_boundary_conditions.reserve (num_boundary_conditions); + for (size_t i_bc = 0; i_bc < num_boundary_conditions; ++i_bc) { + testing_boundary_conditions.emplace_back ("testing_boundary_condition_" + std::to_string (i_bc)); + } + + /* Assign all bcs. */ + detail::t8_cmesh_boundary_condition_handler handler (nullptr); + size_t bc_min = rank * num_boundary_conditions_per_rank; + size_t bc_max = (rank + 1) * num_boundary_conditions_per_rank + 2; + /* All ranks except the last one assign bcs. */ + if (rank != mpisize - 1) { + for (size_t i_bc = bc_min; i_bc < bc_max; ++i_bc) { + handler.register_boundary_condition (testing_boundary_conditions[i_bc]); + } + } + + /* Synchronize the conditions. */ + handler.synchronize (comm); + + /* Check if every rank has all conditions. */ + check_boundary_conditions (handler, testing_boundary_conditions); +}