STK: Snapshot 09-25-24 16:01 from Sierra 5.21.5-241-g354b4bbc

trilinos · Sep 25, 2024 · e903475 · e903475
1 parent 10005bd
commit e903475
Show file tree

Hide file tree

Showing 104 changed files with 3,013 additions and 1,126 deletions.
diff --git a/packages/stk/CHANGELOG.md b/packages/stk/CHANGELOG.md
@@ -1,5 +1,9 @@
 # CHANGELOG
 
+5.21.5 (STK_VERSION 5210500) 9/25/2024
+   general: Fixed MI300A unified memory build errors.
+   stk_search: Turned off sorted results by default.
+
 5.21.4-1 (STK_VERSION 5210401) 9/04/2024
    Fix cmake configuration errors that occurred on AMD MI300A platform
 

diff --git a/packages/stk/stk_balance/stk_balance/balanceUtils.hpp b/packages/stk/stk_balance/stk_balance/balanceUtils.hpp
@@ -34,7 +34,6 @@
 #define STK_BALANCE_UTILS
 
 #include "mpi.h"
-#include <stk_mesh/base/BulkData.hpp>
 #include <stk_mesh/base/Entity.hpp>
 #include <stk_topology/topology.hpp>
 #include "stk_mesh/base/Field.hpp"  // for field_data
@@ -44,6 +43,8 @@
 
 namespace stk
 {
+namespace mesh { class BulkData; }
+
 namespace balance
 {
 

diff --git a/packages/stk/stk_balance/stk_balance/internal/NodeBalancer.cpp b/packages/stk/stk_balance/stk_balance/internal/NodeBalancer.cpp
@@ -80,10 +80,8 @@ void
 NodeBalancer::getGlobalLoadImbalance(double &loadFactor, int& numLocallyOwnedNodes)
 {
   stk::mesh::Selector localSelector = m_metaData.locally_owned_part();
-  stk::mesh::EntityVector ownedNodes;
-  stk::mesh::get_entities(m_bulkData, stk::topology::NODE_RANK, localSelector, ownedNodes);
+  numLocallyOwnedNodes = stk::mesh::count_entities(m_bulkData, stk::topology::NODE_RANK, localSelector);
 
-  numLocallyOwnedNodes = ownedNodes.size();
   int maxLocallyOwned = 0;
   int minLocallyOwned = 0;
   stk::all_reduce_max(m_bulkData.parallel(), &numLocallyOwnedNodes, &maxLocallyOwned, 1);

diff --git a/packages/stk/stk_balance/stk_balance/internal/StkBalanceUtils.cpp b/packages/stk/stk_balance/stk_balance/internal/StkBalanceUtils.cpp
@@ -175,7 +175,11 @@ SearchElemPairs getBBIntersectionsForFacesParticles(stk::mesh::BulkData& stkMesh
 
   stk::balance::internal::SearchElemPairs searchResults;
 
-  stk::search::coarse_search(faceBoxes, faceBoxes, stk::search::KDTREE, stkMeshBulkData.parallel(), searchResults);
+  bool enforceSearchResultSymmetry = true;
+  bool autoSwapDomainAndRange = true;
+  bool sortSearchResults = true;
+  stk::search::coarse_search(faceBoxes, faceBoxes, stk::search::KDTREE, stkMeshBulkData.parallel(), searchResults,
+                            enforceSearchResultSymmetry, autoSwapDomainAndRange, sortSearchResults);
 
   stk::balance::internal::SearchElemPairs::iterator iter = std::unique(searchResults.begin(), searchResults.end());
   searchResults.resize(iter - searchResults.begin());

diff --git a/packages/stk/stk_balance/stk_balance/io/BalanceIO.hpp b/packages/stk/stk_balance/stk_balance/io/BalanceIO.hpp
@@ -37,7 +37,6 @@
 #include "mpi.h"
 
 #include "stk_mesh/base/MetaData.hpp"
-#include "stk_mesh/base/BulkData.hpp"
 #include "stk_io/StkMeshIoBroker.hpp"
 
 #include "stk_balance/balanceUtils.hpp"
@@ -46,6 +45,9 @@
 #include <memory>
 
 namespace stk {
+
+namespace mesh { class BulkData; }
+
 namespace balance {
 
 class BalanceIO

diff --git a/packages/stk/stk_balance/stk_balance/mesh/BalanceMesh.cpp b/packages/stk/stk_balance/stk_balance/mesh/BalanceMesh.cpp
@@ -32,6 +32,7 @@
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "BalanceMesh.hpp"
+#include <stk_mesh/base/BulkData.hpp>
 
 namespace stk {
 namespace balance {

diff --git a/packages/stk/stk_balance/stk_balance/mesh/BalanceMesh.hpp b/packages/stk/stk_balance/stk_balance/mesh/BalanceMesh.hpp
@@ -34,10 +34,11 @@
 #ifndef STK_BALANCE_BALANCE_MESH_HPP
 #define STK_BALANCE_BALANCE_MESH_HPP
 
-#include "stk_mesh/base/MetaData.hpp"
-#include "stk_mesh/base/BulkData.hpp"
-
 namespace stk {
+
+namespace mesh { class MetaData; }
+namespace mesh { class BulkData; }
+
 namespace balance {
 
 class BalanceMesh

diff --git a/packages/stk/stk_doc_tests/stk_io/restartTestUtils.hpp b/packages/stk/stk_doc_tests/stk_io/restartTestUtils.hpp
@@ -39,11 +39,12 @@
 #include <vector>
 #include <string>
 #include <stk_mesh/base/MetaData.hpp>
-#include <stk_mesh/base/BulkData.hpp>
 #include <stk_mesh/base/Field.hpp>
 #include <stk_mesh/base/Types.hpp>
 #include <stk_mesh/base/GetEntities.hpp>
 
+namespace stk { namespace mesh { class BulkData; } }
+
 inline void checkFileForNodalVarNames(const std::string &exodusFilename, const std::vector<std::string>& nodalVarNames)
 {
   Ioss::DatabaseIO *iossDb = Ioss::IOFactory::create("exodus", exodusFilename, Ioss::READ_MODEL, MPI_COMM_WORLD);
@@ -95,25 +96,6 @@ inline void putDataOnTestField(stk::mesh::BulkData &stkMeshBulkData, const doubl
   }
 }
 
-//inline void putDataOnTriStateField(stk::mesh::BulkData &bulkData, stk::mesh::FieldBase *triStateField,
-//        const double stateNp1Value,
-//        const double stateNValue,
-//        const double stateNm1Value)
-//{
-//    stk::mesh::FieldBase *statedFieldNp1 =
-//            triStateField->field_state(stk::mesh::StateNP1);
-//    putDataOnTestField(bulkData, stateNp1Value,
-//                       *statedFieldNp1);
-//    stk::mesh::FieldBase *statedFieldN =
-//            triStateField->field_state(stk::mesh::StateN);
-//    putDataOnTestField(bulkData, stateNValue,
-//                       *statedFieldN);
-//    stk::mesh::FieldBase *statedFieldNm1 =
-//            triStateField->field_state(stk::mesh::StateNM1);
-//    putDataOnTestField(bulkData, stateNm1Value,
-//                       *statedFieldNm1);
-//}
-
 inline void testDataOnField(stk::mesh::BulkData &stkMeshBulkData, const double goldValue, stk::mesh::FieldBase &field)
 {
   std::vector<stk::mesh::Entity> nodes;

diff --git a/packages/stk/stk_doc_tests/stk_mesh/stkMeshTestUtils.hpp b/packages/stk/stk_doc_tests/stk_mesh/stkMeshTestUtils.hpp
@@ -36,11 +36,12 @@
 #define stkMeshTestUtilsHpp
 
 #include <gtest/gtest.h>
-#include <stk_mesh/base/BulkData.hpp>
 #include <stk_mesh/base/ForEachEntity.hpp>
 #include <stk_mesh/base/Field.hpp>
 #include <stk_topology/topology.hpp>
 
+namespace stk { namespace mesh { class BulkData; } }
+
 namespace testUtils
 {
 

diff --git a/packages/stk/stk_doc_tests/stk_transfer/howToUseCopyTransfer.cpp b/packages/stk/stk_doc_tests/stk_transfer/howToUseCopyTransfer.cpp
@@ -72,14 +72,14 @@ void change_mesh_decomposition(stk::mesh::BulkData& mesh)
   int myProc = mesh.parallel_rank();
   int otherProc = 1-myProc;
 
-  stk::mesh::for_each_entity_run(mesh, stk::topology::ELEM_RANK,
-                                 mesh.mesh_meta_data().locally_owned_part(),
-                                 [&entityProcPairs, &otherProc](const stk::mesh::BulkData& bulkData, const stk::mesh::Entity& elem)
+  stk::mesh::for_each_entity_run_no_threads(mesh, stk::topology::ELEM_RANK,
+                                            mesh.mesh_meta_data().locally_owned_part(),
+                                            [&entityProcPairs, &otherProc](const stk::mesh::BulkData& bulkData, const stk::mesh::Entity& elem)
   {
     entityProcPairs.emplace_back(elem, otherProc);
   });
 
-  stk::mesh::for_each_entity_run(mesh, stk::topology::NODE_RANK,
+  stk::mesh::for_each_entity_run_no_threads(mesh, stk::topology::NODE_RANK,
                                  mesh.mesh_meta_data().locally_owned_part(),
                                  [&entityProcPairs, &otherProc](const stk::mesh::BulkData& bulkData, const stk::mesh::Entity& node)
   {

diff --git a/packages/stk/stk_expreval/stk_expreval/Eval.cpp b/packages/stk/stk_expreval/stk_expreval/Eval.cpp
@@ -377,12 +377,9 @@ Eval::initialize_function_map()
   m_functionMap["weibull_pdf"] = FunctionType::WEIBULL_PDF;
   m_functionMap["gamma_pdf"] = FunctionType::GAMMA_PDF;
 
-  m_functionMap["rand"] = FunctionType::RAND;
-  m_functionMap["srand"] = FunctionType::SRAND;
-  m_functionMap["random"] = FunctionType::RANDOM;
+
   m_functionMap["ts_random"] = FunctionType::TS_RANDOM;
   m_functionMap["ts_normal"] = FunctionType::TS_NORMAL;
-  m_functionMap["time"] = FunctionType::TIME;
 }
 
 Eval &

diff --git a/packages/stk/stk_expreval/stk_expreval/Function.hpp b/packages/stk/stk_expreval/stk_expreval/Function.hpp
@@ -111,24 +111,12 @@ enum class FunctionType {
   WEIBULL_PDF,
   GAMMA_PDF,
 
-  RAND,
-  SRAND,
-  RANDOM,
   TS_RANDOM,
   TS_NORMAL,
-  TIME,
 
   UNDEFINED
 };
 
-constexpr bool is_function_supported_on_device(FunctionType type)
-{
-  return type != FunctionType::RAND &&
-         type != FunctionType::SRAND &&
-         type != FunctionType::RANDOM &&
-         type != FunctionType::TIME &&
-         type != FunctionType::UNDEFINED;
-}
 
 KOKKOS_INLINE_FUNCTION
 double cycloidal_ramp(double t, double t1, double t2)

diff --git a/packages/stk/stk_expreval/stk_expreval/NgpNode.hpp b/packages/stk/stk_expreval/stk_expreval/NgpNode.hpp
@@ -627,30 +627,6 @@ class NgpNode
       STK_NGP_ThrowErrorMsg("Incorrect number of arguments for gamma_pdf function");
       break;
     }
-    case FunctionType::RAND : {
-      if (argumentCount == 0) {
-        return real_rand();
-      }
-      STK_NGP_ThrowErrorMsg("Incorrect number of arguments for rand function");
-      break;
-    }
-    case FunctionType::SRAND : {
-      if (argumentCount == 1) {
-        return real_srand(arguments[0]);
-      }
-      STK_NGP_ThrowErrorMsg("Incorrect number of arguments for srand function");
-      break;
-    }
-    case FunctionType::RANDOM : {
-      if (argumentCount == 0) {
-        return random0();
-      }
-      else if (argumentCount == 1) {
-        return random1(arguments[0]);
-      }
-      STK_NGP_ThrowErrorMsg("Incorrect number of arguments for random function");
-      break;
-    }
     case FunctionType::TS_RANDOM : {
       if (argumentCount == 4) {
         return time_space_random(arguments[0], arguments[1], arguments[2], arguments[3]);
@@ -666,13 +642,6 @@ class NgpNode
       STK_NGP_ThrowErrorMsg("Incorrect number of arguments for ts_normal function");
       break;
     }
-    case FunctionType::TIME : {
-      if (argumentCount == 0) {
-        return current_time();
-      }
-      STK_NGP_ThrowErrorMsg("Incorrect number of arguments for time function");
-      break;
-    }
     case FunctionType::UNDEFINED : {
       STK_NGP_ThrowErrorMsg("Undefined function type");
       break;

diff --git a/packages/stk/stk_expreval/stk_expreval/ParsedEval.hpp b/packages/stk/stk_expreval/stk_expreval/ParsedEval.hpp
@@ -73,6 +73,8 @@ class ParsedEval : public ParsedEvalBase
       m_hostNodes(i) = NgpNode(*(eval.get_node(i)));
     }
     Kokkos::deep_copy(m_deviceNodes, m_hostNodes);
+
+    check_for_errors();
   }
 
   KOKKOS_DEFAULTED_FUNCTION ParsedEval(const ParsedEval&) = default;
@@ -81,20 +83,13 @@ class ParsedEval : public ParsedEvalBase
 
   virtual int get_result_buffer_size() override { return RESULT_BUFFER_SIZE; }
 
-  void check_for_errors(bool will_run_on_device) const override
+  STK_DEPRECATED_MSG("check_for_errors is now called by the constructor.  No need to call it yourself")
+  void check_for_errors(bool /*will_run_on_device*/) const override
   {
-    for (size_t i=0; i < m_hostNodes.size(); ++i)
-    {
-      const NgpNode& node = m_hostNodes(i);
-      if (node.m_opcode == OPCODE_FUNCTION)
-      {
-        FunctionType funcType = node.m_data.function.functionType;
-        STK_ThrowRequireMsg(funcType != FunctionType::UNDEFINED, "user defined functions are not supported by ParsedEval");
-        STK_ThrowRequireMsg(will_run_on_device && is_function_supported_on_device(funcType), "random number generation and time functions not supported on device");
-      }
-    }
+    check_for_errors();
   }
 
+
   KOKKOS_INLINE_FUNCTION
   int get_num_variables() const { return m_numVariables; }
 
@@ -136,6 +131,19 @@ class ParsedEval : public ParsedEvalBase
   template <int MAX_BOUND_VARIABLES>
   friend class DeviceVariableMap;
 
+  void check_for_errors() const
+  {
+    for (size_t i=0; i < m_hostNodes.size(); ++i)
+    {
+      const NgpNode& node = m_hostNodes(i);
+      if (node.m_opcode == OPCODE_FUNCTION)
+      {
+        FunctionType funcType = node.m_data.function.functionType;
+        STK_ThrowRequireMsg(funcType != FunctionType::UNDEFINED, "user defined functions and system functions (rand(), time() etc.) are not supported by ParsedEval");
+      }
+    }
+  }
+
   int m_numVariables;
   int m_requiredResultBufferSize;
   Variable::ArrayOffset m_arrayOffsetType;

diff --git a/packages/stk/stk_expreval/stk_expreval/ParsedEvalBase.hpp b/packages/stk/stk_expreval/stk_expreval/ParsedEvalBase.hpp
@@ -36,6 +36,7 @@
 #define PARSEDEVALBASE_HPP
 
 #include <Kokkos_Core.hpp>
+#include "stk_util/stk_config.h"
 
 namespace stk {
 namespace expreval {
@@ -51,6 +52,8 @@ class ParsedEvalBase
 
   virtual int get_result_buffer_size() = 0;
 
+  // remove on 10/29/2024
+  STK_DEPRECATED_MSG("check_for_errors is now called by the constructor.  No need to call it yourself")
   virtual void check_for_errors(bool will_run_on_device) const = 0;
 };
 

diff --git a/packages/stk/stk_integration_tests/cmake_install_test/build_stk_no_stk_mesh_using_cmake b/packages/stk/stk_integration_tests/cmake_install_test/build_stk_no_stk_mesh_using_cmake
@@ -64,7 +64,7 @@ if [ $? -ne 0 ] ; then
 fi
 
 printf "Now building trilinos/stk using make...\n";
-exe "make VERBOSE=1 -j8 >& ${stk_make_log}";
+exe "make VERBOSE=1 -j16 >& ${stk_make_log}";
 if [ $? -ne 0 ] ; then
   echo "!! error in make, check output in ${stk_make_log} !!";
   exit 1;

diff --git a/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk b/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk
@@ -77,7 +77,6 @@ cmake \
 -DKokkos_ENABLE_CUDA_UVM:BOOL=OFF \
 -DKokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE:BOOL=ON \
 -DKokkos_ARCH_VOLTA70=${cuda_on_or_off} \
--DTrilinos_ENABLE_KokkosKernels:BOOL=ON \
 -DTrilinos_ENABLE_Fortran:BOOL=ON \
 -DCMAKE_CXX_STANDARD:STRING=17 \
 -DCMAKE_CXX_FLAGS:STRING="-D${fortran_macro} ${cmake_cxx_flags} -Werror=dangling-else" \

diff --git a/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk_no_stk_mesh b/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk_no_stk_mesh
@@ -34,6 +34,7 @@ cmake \
 -DTPL_ENABLE_BLAS=ON \
 -DTPL_ENABLE_LAPACK=ON \
 -DSTK_ENABLE_TESTS:BOOL=ON \
+-DTrilinos_ENABLE_Intrepid2:BOOL=ON \
 -DTrilinos_ENABLE_STK:BOOL=ON \
 -DTrilinos_ENABLE_STKMesh:BOOL=OFF \
 -DTrilinos_ENABLE_STKUtil:BOOL=ON \

diff --git a/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk_standalone b/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk_standalone
@@ -47,7 +47,6 @@ cmake \
 -DShards_DIR=/fgs/william/Trilinos/install_gcc/lib64/cmake/Shards \
 -DZoltan2Core_DIR=/fgs/william/Trilinos/install_gcc/lib64/cmake/Zoltan2Core \
 -DKokkos_DIR=/fgs/william/Trilinos/install_gcc/lib64/cmake/Kokkos \
--DKokkosKernels_DIR=/fgs/william/kokkos-kernels/install/lib64/cmake/KokkosKernels \
 -DGTest_DIR=/fgs/william/googletest/install/lib64/cmake/GTest \
 ${stk_src_dir}/
 
diff --git a/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk_user_facing b/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk_user_facing
@@ -50,7 +50,6 @@ cmake \
 -DKokkos_ENABLE_RELOCATABLE_DEVICE_CODE:BOOL=OFF \
 -DKokkos_ARCH_VOLTA70=${cuda_on_or_off} \
 -DTpetra_ENABLE_CUDA:BOOL=${cuda_on_or_off} \
--DTrilinos_ENABLE_KokkosKernels:BOOL=ON \
 -DTrilinos_ENABLE_Zoltan:BOOL=ON \
 -DTrilinos_ENABLE_Fortran:BOOL=OFF \
 -DCMAKE_CXX_STANDARD:STRING=17 \

diff --git a/packages/stk/stk_io/stk_io/IOHelpers.hpp b/packages/stk/stk_io/stk_io/IOHelpers.hpp
@@ -43,7 +43,6 @@
 #include <stk_io/DatabasePurpose.hpp>              // for DatabasePurpose
 #include <stk_io/IossBridge.hpp>
 #include <stk_io/MeshField.hpp>                    // for MeshField, etc
-#include <stk_mesh/base/BulkData.hpp>              // for BulkData
 #include <stk_mesh/base/Selector.hpp>              // for Selector
 #include <stk_util/parallel/Parallel.hpp>          // for ParallelMachine
 #include <stk_util/util/ParameterList.hpp>         // for Type