Skip to content

rocm build on tuo #1477

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 18 commits into
base: develop
Choose a base branch
from
Draft

rocm build on tuo #1477

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
195ec83
rocm build on tuo
chapman39 Oct 2, 2025
bf23830
Merge remote-tracking branch 'origin/develop' into task/chapman39/tos…
chapman39 Oct 6, 2025
23ada41
Merge remote-tracking branch 'origin/develop' into task/chapman39/tos…
chapman39 Oct 13, 2025
4150fd7
update mfem, tribol, hypre
chapman39 Oct 13, 2025
dfa3a1c
update axom
chapman39 Oct 14, 2025
fd6208b
update axom package
chapman39 Oct 15, 2025
3232cca
update uberenv
chapman39 Oct 15, 2025
d761498
update cray mpich version
chapman39 Oct 15, 2025
2c457d2
cleanup
chapman39 Oct 15, 2025
18cba34
Merge remote-tracking branch 'origin/develop' into task/chapman39/tos…
chapman39 Oct 27, 2025
9734b5a
update versions, update axom package to lastest
chapman39 Oct 29, 2025
4b435d5
Merge remote-tracking branch 'origin/develop' into task/chapman39/tos…
chapman39 Oct 30, 2025
f7a98c9
fix enzyme package
chapman39 Nov 3, 2025
cda3c73
Merge remote-tracking branch 'origin/develop' into task/chapman39/tos…
chapman39 Nov 5, 2025
291b664
something else was defined 4 underscores
chapman39 Nov 12, 2025
267ee9d
fix some build errors
chapman39 Nov 12, 2025
4009f37
rzadams host config
chapman39 Nov 13, 2025
54f4a17
tuo host config
chapman39 Nov 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .uberenv_config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"package_final_phase" : "initconfig",
"package_source_dir" : "../..",
"spack_url": "https://github.com/spack/spack.git",
"spack_commit": "b4b1c5f271c75e918ce922ff1bcc77542eda3ec9",
"spack_commit": "734c5db2121b01c373eed6538e452f18887e9e44",
"spack_packages_url": "https://github.com/spack/spack-packages.git",
"spack_packages_commit": "a75a7f75182ffc7a51c6ca7f0fec4bf9b2705be8",
"spack_configs_path": "scripts/spack/configs",
Expand Down
2 changes: 1 addition & 1 deletion axom
Open in desktop
Submodule axom updated 282 files
2 changes: 1 addition & 1 deletion cmake/thirdparty/SetupSmithThirdParty.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,7 @@ if (NOT SMITH_THIRD_PARTY_LIBRARIES_FOUND)
"Additional packages required by STRUMPACK.")
set(STRUMPACK_TARGET_NAMES STRUMPACK::strumpack CACHE STRING "")
endif()
set(MFEM_USE_UMPIRE OFF CACHE BOOL "")
set(MFEM_USE_UMPIRE ${SMITH_ENABLE_HIP} CACHE BOOL "")
set(MFEM_USE_ZLIB ON CACHE BOOL "")

#### MFEM Configuration Options
Expand Down
148 changes: 148 additions & 0 deletions host-configs/[email protected]_hip.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
#------------------------------------------------------------------------------
# !!!! This is a generated file, edit at own risk !!!!
#------------------------------------------------------------------------------
# CMake executable path: /usr/tce/bin/cmake
#------------------------------------------------------------------------------

set(CMAKE_PREFIX_PATH "/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib_cray/2025_11_13_11_17_03/llvm-amdgpu-6.4.2/caliper-2.12.1-tdt2eikgly5djuxkxoaeoora46pkhg3q;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib_cray/2025_11_13_11_17_03/llvm-amdgpu-6.4.2/tribol-0.1.0.23-lsokqdf2gqzplhzf3rzctavbczz5i4ah;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib_cray/2025_11_13_11_17_03/llvm-amdgpu-6.4.2/adiak-0.4.1-3pedcs7kfkltks4rbuimessyn2ftkfb7;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib_cray/2025_11_13_11_17_03/llvm-amdgpu-6.4.2/libunwind-1.8.1-ijbnecr6mrnowuwv3khy34jidzh3yyj3;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib_cray/2025_11_13_11_17_03/llvm-amdgpu-6.4.2/axom-0.12.0.0-xnz7rzvxkvy5ujy7hsffpqyrjjbbecyg;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib_cray/2025_11_13_11_17_03/llvm-amdgpu-6.4.2/conduit-0.9.5-fr6nlivg2q44p2fm7d3twivtsitkrwp4;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib_cray/2025_11_13_11_17_03/llvm-amdgpu-6.4.2/mfem-4.9.0.2-z7vfceuxqpfvnmta32y3cdk7inyhzowx;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib_cray/2025_11_13_11_17_03/llvm-amdgpu-6.4.2/enzyme-0.0.180-55na6ly2nalovxe2t7d2lihtwlakxnaz;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib_cray/2025_11_13_11_17_03/llvm-amdgpu-6.4.2/hypre-2.33.0-smuq4sfalzokt3xbj3afxcyxdyr737yr;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib_cray/2025_11_13_11_17_03/llvm-amdgpu-6.4.2/netcdf-c-4.7.4-hdvzf6k35jgfv4fto44neg6oka65trbb;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib_cray/2025_11_13_11_17_03/llvm-amdgpu-6.4.2/raja-2025.09.0-mlvvn7h257hluzfwmkgk5qyvvdgcimzr;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib_cray/2025_11_13_11_17_03/llvm-amdgpu-6.4.2/strumpack-8.0.0-hs2yx7rpgcfmescp4tgang2enurkzoyj;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib_cray/2025_11_13_11_17_03/llvm-amdgpu-6.4.2/superlu-dist-8.1.2-j4sdkhw4zt5g7zogpylucmb2adzufpps;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib_cray/2025_11_13_11_17_03/llvm-amdgpu-6.4.2/umpire-2025.09.0-7qtxj2a5ihxb7metqdjot6dxmfof47oe;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib_cray/2025_11_13_11_17_03/llvm-amdgpu-6.4.2/hdf5-1.8.23-hi7eotv4olqrfedxqzy56p3mgfldbpyg;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib_cray/2025_11_13_11_17_03/llvm-amdgpu-6.4.2/netlib-scalapack-2.2.2-a5ef54cb4mvrj7vaia2nssjjiy42fuy6;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib_cray/2025_11_13_11_17_03/llvm-amdgpu-6.4.2/parmetis-4.0.3-5prdk7nqp4ujhc3suytx66t2jkhunia4;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib_cray/2025_11_13_11_17_03/llvm-amdgpu-6.4.2/camp-2025.09.2-3fxmbhyimpvimplr2qvt32giqaom7ymy;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib_cray/2025_11_13_11_17_03/llvm-amdgpu-6.4.2/fmt-11.0.2-7obrzgdsm7vuixhdva7kcl7ut637gmfw;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib_cray/2025_11_13_11_17_03/llvm-amdgpu-6.4.2/metis-5.1.0-rvjhpveexfgalsyntmwhkwm6icozu7zg;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib_cray/2025_11_13_11_17_03/none-none/compiler-wrapper-1.0-gp57b4mbq4ctshuybmbzq62fjybcm5tp;/usr/tce;/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.29-rocmcc-6.4.2;/opt/rocm-6.4.2;/opt/rocm-6.4.2;/opt/rocm-6.4.2;/opt/rocm-6.4.2/llvm;/opt/rocm-6.4.2;/opt/rocm-6.4.2;/opt/rocm-6.4.2;/opt/rocm-6.4.2;/opt/rocm-6.4.2;/opt/rocm-6.4.2;/opt/rocm-6.4.2" CACHE STRING "")

set(CMAKE_INSTALL_RPATH_USE_LINK_PATH "ON" CACHE STRING "")

set(CMAKE_BUILD_RPATH "/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib_cray/2025_11_13_11_17_03/llvm-amdgpu-6.4.2/smith-develop-efnfuu7vhtjb42racje2yz3xxkzwjvaf/lib;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib_cray/2025_11_13_11_17_03/llvm-amdgpu-6.4.2/smith-develop-efnfuu7vhtjb42racje2yz3xxkzwjvaf/lib64;;/opt/rh/gcc-toolset-13/root/usr/lib/gcc/x86_64-redhat-linux/13" CACHE STRING "")

set(CMAKE_INSTALL_RPATH "/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib_cray/2025_11_13_11_17_03/llvm-amdgpu-6.4.2/smith-develop-efnfuu7vhtjb42racje2yz3xxkzwjvaf/lib;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib_cray/2025_11_13_11_17_03/llvm-amdgpu-6.4.2/smith-develop-efnfuu7vhtjb42racje2yz3xxkzwjvaf/lib64;;/opt/rh/gcc-toolset-13/root/usr/lib/gcc/x86_64-redhat-linux/13" CACHE STRING "")

set(CMAKE_BUILD_TYPE "Release" CACHE STRING "")

#------------------------------------------------------------------------------
# Compilers
#------------------------------------------------------------------------------
# Compiler Spec: [email protected]/5r3ij6vqblq4roanzix4i7qiteyl6gjj
#------------------------------------------------------------------------------
if(DEFINED ENV{SPACK_CC})

set(CMAKE_C_COMPILER "/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib_cray/2025_11_13_11_17_03/none-none/compiler-wrapper-1.0-gp57b4mbq4ctshuybmbzq62fjybcm5tp/libexec/spack/rocmcc/amdclang" CACHE PATH "")

set(CMAKE_CXX_COMPILER "/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib_cray/2025_11_13_11_17_03/none-none/compiler-wrapper-1.0-gp57b4mbq4ctshuybmbzq62fjybcm5tp/libexec/spack/rocmcc/amdclang++" CACHE PATH "")

set(CMAKE_Fortran_COMPILER "/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib_cray/2025_11_13_11_17_03/none-none/compiler-wrapper-1.0-gp57b4mbq4ctshuybmbzq62fjybcm5tp/libexec/spack/rocmcc/amdflang" CACHE PATH "")

else()

set(CMAKE_C_COMPILER "/opt/rocm-6.4.2/llvm/bin/amdclang" CACHE PATH "")

set(CMAKE_CXX_COMPILER "/opt/rocm-6.4.2/llvm/bin/amdclang++" CACHE PATH "")

set(CMAKE_Fortran_COMPILER "/opt/rocm-6.4.2/llvm/bin/amdflang" CACHE PATH "")

endif()

#------------------------------------------------------------------------------
# MPI
#------------------------------------------------------------------------------

set(MPI_C_COMPILER "/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.29-rocmcc-6.4.2/bin/mpicc" CACHE PATH "")

set(MPI_CXX_COMPILER "/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.29-rocmcc-6.4.2/bin/mpicxx" CACHE PATH "")

set(MPI_Fortran_COMPILER "/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.29-rocmcc-6.4.2/bin/mpif90" CACHE PATH "")

set(MPIEXEC_NUMPROC_FLAG "-n" CACHE STRING "")

set(ENABLE_MPI ON CACHE BOOL "")

set(MPIEXEC_EXECUTABLE "/usr/global/tools/flux_wrappers/bin/srun" CACHE PATH "")

#------------------------------------------------------------------------------
# Hardware
#------------------------------------------------------------------------------

#------------------------------------------------
# ROCm
#------------------------------------------------

set(ROCM_PATH "/opt/rocm-6.4.2" CACHE PATH "")

set(CMAKE_HIP_ARCHITECTURES "gfx90a;gfx942" CACHE STRING "")

set(CMAKE_HIP_COMPILER "/opt/rocm-6.4.2/llvm/bin/amdclang++" CACHE FILEPATH "")

set(HIP_ROOT_DIR "/opt/rocm-6.4.2" CACHE PATH "")

set(AMDGPU_TARGETS "gfx90a;gfx942" CACHE STRING "")

set(GPU_TARGETS "gfx90a;gfx942" CACHE STRING "")

set(ENABLE_OPENMP OFF CACHE BOOL "")

set(ENABLE_HIP ON CACHE BOOL "")

set(ROCM_ROOT_DIR "/opt/rocm-6.4.2" CACHE PATH "")

set(CMAKE_EXE_LINKER_FLAGS "-lxpmem -L/opt/cray/pe/mpich/8.1.29/gtl/lib -Wl,-rpath,/opt/cray/pe/mpich/8.1.29/gtl/lib -lmpi_gtl_hsa -L/opt/rocm-6.4.2/lib/llvm/lib -Wl,-rpath,/opt/rocm-6.4.2/lib/llvm/lib -L/opt/rocm-6.4.2/lib -Wl,-rpath,/opt/rocm-6.4.2/lib -Wl,--disable-new-dtags -lflang -lflangrti -lamdhip64 -lhsakmt -lhsa-runtime64 -lamd_comgr -lpgmath -lhipblas" CACHE STRING "")

#------------------------------------------------------------------------------
# TPLs
#------------------------------------------------------------------------------

set(TPL_ROOT "/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib_cray/2025_11_13_11_17_03/llvm-amdgpu-6.4.2" CACHE PATH "")

set(AXOM_DIR "${TPL_ROOT}/axom-0.12.0.0-xnz7rzvxkvy5ujy7hsffpqyrjjbbecyg" CACHE PATH "")

set(CAMP_DIR "${TPL_ROOT}/camp-2025.09.2-3fxmbhyimpvimplr2qvt32giqaom7ymy" CACHE PATH "")

set(CONDUIT_DIR "${TPL_ROOT}/conduit-0.9.5-fr6nlivg2q44p2fm7d3twivtsitkrwp4" CACHE PATH "")

set(LUA_DIR "/usr" CACHE PATH "")

set(MFEM_DIR "${TPL_ROOT}/mfem-4.9.0.2-z7vfceuxqpfvnmta32y3cdk7inyhzowx" CACHE PATH "")

set(HDF5_DIR "${TPL_ROOT}/hdf5-1.8.23-hi7eotv4olqrfedxqzy56p3mgfldbpyg" CACHE PATH "")

set(HYPRE_DIR "${TPL_ROOT}/hypre-2.33.0-smuq4sfalzokt3xbj3afxcyxdyr737yr" CACHE PATH "")

set(METIS_DIR "${TPL_ROOT}/metis-5.1.0-rvjhpveexfgalsyntmwhkwm6icozu7zg" CACHE PATH "")

set(PARMETIS_DIR "${TPL_ROOT}/parmetis-4.0.3-5prdk7nqp4ujhc3suytx66t2jkhunia4" CACHE PATH "")

set(NETCDF_DIR "${TPL_ROOT}/netcdf-c-4.7.4-hdvzf6k35jgfv4fto44neg6oka65trbb" CACHE PATH "")

set(SUPERLUDIST_DIR "${TPL_ROOT}/superlu-dist-8.1.2-j4sdkhw4zt5g7zogpylucmb2adzufpps" CACHE PATH "")

set(ADIAK_DIR "${TPL_ROOT}/adiak-0.4.1-3pedcs7kfkltks4rbuimessyn2ftkfb7" CACHE PATH "")

# AMGX not built

set(CALIPER_DIR "${TPL_ROOT}/caliper-2.12.1-tdt2eikgly5djuxkxoaeoora46pkhg3q" CACHE PATH "")

set(ENZYME_DIR "${TPL_ROOT}/enzyme-0.0.180-55na6ly2nalovxe2t7d2lihtwlakxnaz" CACHE PATH "")

# PETSC not built

set(RAJA_DIR "${TPL_ROOT}/raja-2025.09.0-mlvvn7h257hluzfwmkgk5qyvvdgcimzr" CACHE PATH "")

# SLEPC not built

set(STRUMPACK_DIR "${TPL_ROOT}/strumpack-8.0.0-hs2yx7rpgcfmescp4tgang2enurkzoyj" CACHE PATH "")

# SUNDIALS not built

set(UMPIRE_DIR "${TPL_ROOT}/umpire-2025.09.0-7qtxj2a5ihxb7metqdjot6dxmfof47oe" CACHE PATH "")

set(TRIBOL_DIR "${TPL_ROOT}/tribol-0.1.0.23-lsokqdf2gqzplhzf3rzctavbczz5i4ah" CACHE PATH "")

#------------------------------------------------------------------------------
# Devtools
#------------------------------------------------------------------------------

# Code checks disabled due to disabled devtools

set(SMITH_ENABLE_CODE_CHECKS OFF CACHE BOOL "")

set(ENABLE_CLANGFORMAT OFF CACHE BOOL "")

set(ENABLE_CLANGTIDY OFF CACHE BOOL "")

set(ENABLE_DOCS OFF CACHE BOOL "")


Loading