Openmp cce

.github/workflows/bench.yml

@@ -62,6 +62,14 @@ jobs:
             device: gpu
             interface: acc
             build_script: "bash .github/workflows/frontier/build.sh gpu acc bench"
+          - cluster: frontier
+            name: Oak Ridge | Frontier (CCE)
+            group: phoenix
+            labels: frontier
+            flag: f
+            device: gpu
+            interface: omp
+            build_script: "bash .github/workflows/frontier/build.sh gpu omp bench"
     runs-on:
       group: ${{ matrix.group }}
       labels: ${{ matrix.labels }}

.github/workflows/test.yml

@@ -111,10 +111,6 @@ jobs:
           - device: 'cpu'
             interface: 'none'
             lbl: 'frontier'
-        exclude:
-          - device: 'gpu'
-            interface: 'omp'
-            lbl: 'frontier'
     runs-on:
       group:  phoenix
       labels: ${{ matrix.lbl }}

CMakeLists.txt

@@ -29,6 +29,7 @@ option(MFC_SYSCHECK      "Build syscheck"                                    OFF
 option(MFC_DOCUMENTATION "Build documentation"                               OFF)
 option(MFC_ALL           "Build everything"                                  OFF)
 option(MFC_SINGLE_PRECISION "Build single precision"                         OFF)
+option(MFC_MIXED_PRECISION "Build mixed precision"                           OFF)
 
 if (MFC_ALL)
     set(MFC_PRE_PROCESS   ON FORCE)
@@ -43,6 +44,10 @@ else()
     add_compile_definitions(MFC_DOUBLE_PRECISION)
 endif()
 
+if (MFC_MIXED_PRECISION)
+    add_compile_definitions(MFC_MIXED_PRECISION)
+endif()
+
 
 # CMake Library Imports
 
@@ -196,7 +201,7 @@ elseif ((CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") OR (CMAKE_Fortran_COMPILER_
     add_compile_options(
         $<$<COMPILE_LANGUAGE:Fortran>:-Mfreeform>
         $<$<COMPILE_LANGUAGE:Fortran>:-cpp>
-	      $<$<COMPILE_LANGUAGE:Fortran>:-Minfo=inline>
+        $<$<COMPILE_LANGUAGE:Fortran>:-Minfo=inline>
         $<$<COMPILE_LANGUAGE:Fortran>:-Minfo=accel>
     )
 
@@ -474,10 +479,10 @@ function(MFC_SETUP_TARGET)
             endif()
         endif()
 
-				if (ARGS_LAPACK)
-					find_package(LAPACK REQUIRED)
-					target_link_libraries(${a_target} PRIVATE LAPACK::LAPACK)
-				endif()
+        if (ARGS_LAPACK)
+            find_package(LAPACK REQUIRED)
+            target_link_libraries(${a_target} PRIVATE LAPACK::LAPACK)
+        endif()
 
         if ((MFC_OpenACC AND ARGS_OpenACC) OR (MFC_OpenMP AND ARGS_OpenMP))
             if ((MFC_OpenACC AND ARGS_OpenACC))
@@ -573,7 +578,13 @@ function(MFC_SETUP_TARGET)
                 find_package(hipfort COMPONENTS hip CONFIG REQUIRED)
                 target_link_libraries(${a_target} PRIVATE hipfort::hip hipfort::hipfort-amdgcn)
             elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
-                find_package(hipfort COMPONENTS hip CONFIG REQUIRED)
+
+                if (MFC_Unified)
+                    target_compile_options(${ARGS_TARGET}
+                        PRIVATE -DFRONTIER_UNIFIED)
+                endif()
+
+    		find_package(hipfort COMPONENTS hip CONFIG REQUIRED)
                 target_link_libraries(${a_target} PRIVATE hipfort::hip hipfort::hipfort-amdgcn flang_rt.hostdevice)
             endif()
         elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")

examples/3D_IGR_33jet/case.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+import math
+import json
+
+# Domain parameters
+D = 2.5  # Jet diameter
+Nd = 711  # Cells per jet diameter
+
+x0 = 0  # x_beg coordinate
+x1 = 19 * D  # x_end coordinate
+y0 = -22 * D / 2  # y_beg coordinate
+y1 = 22 * D / 2  # y_end coordinate
+z0 = -22 * D / 2  # y_beg coordinate
+z1 = 22 * D / 2  # y_end coordinate
+Nx = int(Nd * (x1 - x0) / D) - 1
+Ny = int(Nd * (y1 - y0) / D) - 1
+Nz = int(Nd * (z1 - z0) / D) - 1
+
+time_end = 5
+igrIters = 5
+
+dx = D / Nd
+dt = dx / 36
+
+Nt = int(time_end / dt)
+
+# Configuring case dictionary
+print(
+    json.dumps(
+        {
+            # Logistics
+            "run_time_info": "T",
+            # Computational Domain Parameters
+            "x_domain%beg": x0,
+            "x_domain%end": x1,
+            "y_domain%beg": y0,
+            "y_domain%end": y1,
+            "z_domain%beg": z0,
+            "z_domain%end": z1,
+            "m": int(Nx),
+            "n": int(Ny),
+            "p": int(Nz),
+            "dt": dt,
+            "t_step_start": 0,
+            "t_step_stop": 10,  # Nt,
+            "t_step_save": 10,  # int(Nt/50),
+            # Simulation Algorithm Parameters
+            "num_patches": 1,
+            "num_bc_patches": 0,
+            "model_eqns": 2,
+            "alt_soundspeed": "F",
+            "num_fluids": 1,
+            "mpp_lim": "F",
+            "time_stepper": 3,
+            "igr": "T",
+            "igr_order": 3,
+            "igr_pres_lim": "T",
+            "igr_iter_solver": 1,
+            "num_igr_iters": igrIters,
+            "num_igr_warm_start_iters": 10 * igrIters,
+            "alf_factor": 10,
+            "bc_x%beg": -17,
+            "bc_x%end": -3,
+            "bc_y%beg": -3,
+            "bc_y%end": -3,
+            "bc_z%beg": -3,
+            "bc_z%end": -3,
+            # Formatted Database Files Structure Parameters
+            "format": 1,
+            "precision": 1,
+            "prim_vars_wrt": "T",
+            "file_per_process": "T",
+            "parallel_io": "T",
+            "down_sample": "T",
+            # Patch
+            "patch_icpp(1)%geometry": 9,
+            "patch_icpp(1)%x_centroid": (x1 + x0) / 2,
+            "patch_icpp(1)%y_centroid": (y1 + y0) / 2,
+            "patch_icpp(1)%z_centroid": (z1 + z0) / 2,
+            "patch_icpp(1)%length_x": 2 * (x1 - x0),
+            "patch_icpp(1)%length_y": 2 * (y1 - y0),
+            "patch_icpp(1)%length_z": 2 * (z1 - z0),
+            "patch_icpp(1)%hcid": 303,
+            "patch_icpp(1)%vel(1)": 0.0e00,
+            "patch_icpp(1)%vel(2)": 0.0e00,
+            "patch_icpp(1)%vel(3)": 0.0e00,
+            "patch_icpp(1)%pres": 1.0e00,
+            "patch_icpp(1)%alpha_rho(1)": 1.0,
+            "patch_icpp(1)%alpha(1)": 1.0,
+            # Perturbation of velocity field
+            "simplex_perturb": "T",
+            "simplex_params%perturb_vel(1)": "T",
+            "simplex_params%perturb_vel_freq(1)": 3,
+            "simplex_params%perturb_vel_scale(1)": 0.02,
+            "simplex_params%perturb_vel_offset(1,1)": 12.3,
+            "simplex_params%perturb_vel_offset(1,2)": -11.3,
+            "simplex_params%perturb_vel_offset(1,3)": 34.6,
+            "simplex_params%perturb_vel(2)": "T",
+            "simplex_params%perturb_vel_freq(2)": 2,
+            "simplex_params%perturb_vel_scale(2)": 0.02,
+            "simplex_params%perturb_vel_offset(2,1)": -70.3,
+            "simplex_params%perturb_vel_offset(2,2)": 33.4,
+            "simplex_params%perturb_vel_offset(2,3)": -34.6,
+            "simplex_params%perturb_vel(3)": "T",
+            "simplex_params%perturb_vel_freq(3)": 2,
+            "simplex_params%perturb_vel_scale(3)": 0.02,
+            "simplex_params%perturb_vel_offset(3,1)": 123.3,
+            "simplex_params%perturb_vel_offset(3,2)": -654.3,
+            "simplex_params%perturb_vel_offset(3,3)": -64.5,
+            # Fluids Physical Parameters
+            "fluid_pp(1)%gamma": 1.0e00 / (1.4e00 - 1.0e00),
+            "fluid_pp(1)%pi_inf": 0.0,
+            "viscous": "T",
+            "fluid_pp(1)%Re(1)": 5e5,
+        },
+        indent=4,
+    )
+)

examples/3D_IGR_33jet/jets.csv

@@ -0,0 +1,33 @@
+3.000000,0.000000,0.001
+-1.500000,2.598076,0.001
+-1.500000,-2.598076,0.001
+9.000000,0.000000,0.001
+7.281153,5.290067,0.001
+2.781153,8.559509,0.001
+-2.781153,8.559509,0.001
+-7.281153,5.290067,0.001
+-9.000000,0.000000,0.001
+-7.281153,-5.290067,0.001
+-2.781153,-8.559509,0.001
+2.781153,-8.559509,0.001
+7.281153,-5.290067,0.001
+15.000000,0.000000,0.001
+14.265848,4.635255,0.001
+12.135255,8.816779,0.001
+8.816779,12.135255,0.001
+4.635255,14.265848,0.001
+0.000000,15.000000,0.001
+-4.635255,14.265848,0.001
+-8.816779,12.135255,0.001
+-12.135255,8.816779,0.001
+-14.265848,4.635255,0.001
+-15.000000,0.000000,0.001
+-14.265848,-4.635255,0.001
+-12.135255,-8.816779,0.001
+-8.816779,-12.135255,0.001
+-4.635255,-14.265848,0.001
+-0.000000,-15.000000,0.001
+4.635255,-14.265848,0.001
+8.816779,-12.135255,0.001
+12.135255,-8.816779,0.001
+14.265848,-4.635255,0.001

examples/3D_IGR_33jet/njet.txt

@@ -0,0 +1 @@
+33

examples/3D_IGR_jet_1fluid/case.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+import json
+
+# Domain parameters
+alfFactor = 10
+igrIters = 3
+
+NPx = 2
+NPy = 2
+NPz = 2
+
+x0 = 0
+x1 = 2 * NPx
+y0 = -1.0 * NPy
+y1 = 1.0 * NPy
+z0 = -1.0 * NPz
+z1 = 1.0 * NPz
+
+N = 1383
+
+Nx = N * NPx - 1
+Ny = N * NPy - 1
+Nz = N * NPz - 1
+
+dx = (x1 - x0) / Nx
+dt = dx / 20000
+
+# Configuring case dictionary
+print(
+    json.dumps(
+        {
+            # Logistics
+            "run_time_info": "F",
+            # Computational Domain Parameters
+            "x_domain%beg": x0,
+            "x_domain%end": x1,
+            "y_domain%beg": y0,
+            "y_domain%end": y1,
+            "z_domain%beg": z0,
+            "z_domain%end": z1,
+            "m": int(Nx),
+            "n": int(Ny),
+            "p": int(Nz),
+            "dt": dt,
+            "t_step_start": 0,
+            "t_step_stop": 10,  # Nt,
+            "t_step_save": 10,  # int(Nt / 20),
+            # Simulation Algorithm Parameters
+            "num_patches": 1,
+            "num_bc_patches": 0,
+            "model_eqns": 2,
+            "alt_soundspeed": "F",
+            "num_fluids": 1,
+            "mpp_lim": "F",
+            "time_stepper": 3,
+            "igr": "T",
+            "igr_order": 5,
+            "igr_pres_lim": "T",
+            "igr_iter_solver": 1,
+            "num_igr_iters": igrIters,
+            "num_igr_warm_start_iters": igrIters,
+            "alf_factor": 10,
+            "bc_x%beg": -17,
+            "bc_x%end": -3,
+            "bc_y%beg": -3,
+            "bc_y%end": -3,
+            "bc_z%beg": -3,
+            "bc_z%end": -3,
+            # Formatted Database Files Structure Parameters
+            "format": 1,
+            "precision": 2,
+            "prim_vars_wrt": "T",
+            "c_wrt": "F",
+            "parallel_io": "T",
+            "file_per_process": "T",
+            "down_sample": "F",
+            # Background
+            "patch_icpp(1)%geometry": 9,
+            "patch_icpp(1)%x_centroid": (x1 + x0) / 2,
+            "patch_icpp(1)%y_centroid": (y1 + y0) / 2,
+            "patch_icpp(1)%z_centroid": (z1 + z0) / 2,
+            "patch_icpp(1)%length_x": (x1 - x0),
+            "patch_icpp(1)%length_y": (y1 - y0),
+            "patch_icpp(1)%length_z": (z1 - z0),
+            "patch_icpp(1)%hcid": 302,
+            "patch_icpp(1)%vel(1)": 1.0,
+            "patch_icpp(1)%vel(2)": 0.0e00,
+            "patch_icpp(1)%vel(3)": 0.0e00,
+            "patch_icpp(1)%pres": 1.0,
+            "patch_icpp(1)%alpha_rho(1)": 1.0e00,
+            "patch_icpp(1)%alpha(1)": 1.0e00,
+            # Fluids Physical Parameters
+            "fluid_pp(1)%gamma": 1.0e00 / (1.4e00 - 1.0e00),
+            "fluid_pp(1)%pi_inf": 0.0,
+            "viscous": "T",
+            "fluid_pp(1)%Re(1)": 5e4,
+        },
+        indent=4,
+    )
+)

load_amd.sh

@@ -0,0 +1,7 @@
+module use /ccs/home/bcornille/afar-drops/modulefiles/Core/
+module load rocm-afar-drop mpich cray-python
+module load cmake
+#export OMPX_APU_MAPS=0
+#export HSA_XNACK=0
+#export LIBOMPTARGET_INFO=0
+#export AMD_LOG_LEVEL=1