Summit fixes

Author: henryleberre

CMakeLists.txt

@@ -94,11 +94,19 @@ ELSEIF ((CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") OR
         (CMAKE_Fortran_COMPILER_ID STREQUAL "PGI"))
 
     ADD_COMPILE_OPTIONS(-r8 -cpp -Mfreeform -Minfo=accel -Mr8intrinsics)
+    
+    IF (MFC_WITH_OPEN_ACC)
+        IF     (CMAKE_BUILD_TYPE STREQUAL "Release")
+            ADD_COMPILE_OPTIONS(-gpu=keep,ptxinfo,lineinfo)
+        ELSEIF (CMAKE_BUILD_TYPE STREQUAL "Debug")
+            ADD_COMPILE_OPTIONS(-gpu=keep,ptxinfo,lineinfo,autocompare,debug)
+        ENDIF()
+    ENDIF()
 
 ENDIF()
 
 
-# === CMake Build Configurations
+# === Release Optimizations
 IF (CMAKE_BUILD_TYPE STREQUAL "Release")
 
     # === === Processor tuning
@@ -235,15 +243,36 @@ IF (MFC_BUILD_SIMULATION)
 
 
     IF (CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC" OR CMAKE_Fortran_COMPILER_ID STREQUAL "PGI")
-        TARGET_LINK_LIBRARIES(simulation PRIVATE CUDA::nvToolsExt cutensor)
+        TARGET_LINK_LIBRARIES(simulation PRIVATE CUDA::nvToolsExt)
     ENDIF()
 
 
     IF (MFC_WITH_OPEN_ACC)
         TARGET_LINK_LIBRARIES(simulation PRIVATE OpenACC::OpenACC_Fortran)
 
         IF (CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC" OR CMAKE_Fortran_COMPILER_ID STREQUAL "PGI")
-            TARGET_LINK_LIBRARIES(simulation PRIVATE CUDA::cufft)
+            TARGET_LINK_LIBRARIES(simulation PRIVATE CUDA::cudart CUDA::cufft cutensor)
+
+            # In the case of the NVHPC SDK, cuTENSOR can be found at a path similar to:
+            # /opt/nvidia/hpc_sdk/Linux_x86_64/xx.yy/math_libs/lib64/ on a regular installation.
+            # CUDAToolkit_LIBRARY_ROOT would then point to /opt/nvidia/hpc_sdk/Linux_x86_64/xx.yy/cuda/zz.ww.
+            # Since the linker doesn't usually consider math_libs/ (for it wasn't told to),
+            # we attempt to add it as a link directory.
+
+            SET(_warn_msg_begin "Can't automatically determine the location of cuTENSOR.")
+            SET(_warn_msg_end   "If linking fails, please explicitly specify a link directory.")
+
+            IF (EXISTS "${CUDAToolkit_LIBRARY_ROOT}/../../math_libs/lib")
+                TARGET_LINK_DIRECTORIES(simulation PRIVATE "${CUDAToolkit_LIBRARY_ROOT}/../../math_libs/lib")
+
+                MESSAGE(WARNING "${_warn_msg_begin} Gave linker ${CUDAToolkit_LIBRARY_ROOT}/../../math_libs/lib   as a potential search path for cuTENSOR (based on a regular NVHPC installation). ${_warn_msg_end} Fortunately, linking will likely succeed.")
+            ELSEIF (EXISTS "${CUDAToolkit_LIBRARY_ROOT}/../../math_libs/lib64")
+                TARGET_LINK_DIRECTORIES(simulation PRIVATE "${CUDAToolkit_LIBRARY_ROOT}/../../math_libs/lib64")
+
+                MESSAGE(WARNING "${_warn_msg_begin} Gave linker ${CUDAToolkit_LIBRARY_ROOT}/../../math_libs/lib64 as a potential search path for cuTENSOR (based on a regular NVHPC installation). ${_warn_msg_end} Fortunately, linking will likely succeed.")
+            ELSE()
+                MESSAGE(WARNING "${_warn_msg_begin} No common potential search directories were found. ${_warn_msg_end}")
+            ENDIF()
         ENDIF()
     ENDIF()
 
@@ -261,7 +290,7 @@ IF (MFC_BUILD_POST_PROCESS)
 
     ADD_EXECUTABLE(post_process "${post_process_srcs}" "${common_srcs}")
 
-    TARGET_LINK_LIBRARIES(post_process PRIVATE HDF5::HDF5 SILO::SILO FFTW::FFTW)
+    TARGET_LINK_LIBRARIES(post_process PRIVATE SILO::SILO HDF5::HDF5 FFTW::FFTW)
 
     IF (MFC_WITH_MPI)
         TARGET_COMPILE_DEFINITIONS(post_process PRIVATE MFC_MPI)

mfc.sh

@@ -64,7 +64,7 @@ if (($?)); then
 fi
 
 
-if ! command -v pip3 > /dev/null 2>&1; then
+if ! command -v pip3 > /dev/null 2>&1 && [ ! -f "$(pwd)/build/venv/bin/activate" ]; then
     wget -O "$(pwd)/build/get-pip.py" https://bootstrap.pypa.io/pip/get-pip.py
     if (($?)); then
         echo "[mfc.sh] Error: Couldn't download get-pip.py."

toolchain/cmake/FindFFTW.cmake

@@ -14,6 +14,7 @@ FIND_PATH(FFTW_INCLUDE_DIR
 FIND_LIBRARY(FFTW_LIBRARY
     NAMES         fftw3
     PATH_SUFFIXES fftw fftw3
+    NAMES_PER_DIR
 )
 
 

toolchain/cmake/FindHDF5.cmake

@@ -14,6 +14,7 @@ FIND_PATH(HDF5_INCLUDE_DIR
 FIND_LIBRARY(HDF5_LIBRARY
     NAMES         libhdf5 hdf5
     PATH_SUFFIXES hdf5
+    NAMES_PER_DIR
 )
 
 

toolchain/cmake/FindSILO.cmake

@@ -14,6 +14,7 @@ FIND_PATH(SILO_INCLUDE_DIR
 FIND_LIBRARY(SILO_LIBRARY
     NAMES         siloh5 silo
     PATH_SUFFIXES silo
+    NAMES_PER_DIR
 )
 
 

toolchain/mfc/tests/case.py

@@ -112,10 +112,10 @@ def run(self, args: dict) -> subprocess.CompletedProcess:
         filepath          = f'"{self.get_dirpath()}/case.py"'
         mode              = f'-m "{args["mode"]}"'
         tasks             = f"-n {self.ppn} "
-        jobs              = f"-j {args['jobs']}"    if args["case_optimization"]  else ""
-        gpus              = f"-g {self.ppn}"        if "gpu" in args["mode"]      else ""
-        binary_option     = f"-b {args['binary']}"  if args["binary"] is not None else ""
-        case_optimization =   "--case-optimization" if args["case_optimization"]  else "--no-build"
+        jobs              = f"-j {args['jobs']}"   if args["case_optimization"]  else ""
+        gpus              = f"-g {self.ppn}"       if "gpu" in args["mode"]      else ""
+        binary_option     = f"-b {args['binary']}" if args["binary"] is not None else ""
+        case_optimization =  "--case-optimization" if args["case_optimization"]  else "--no-build"
         no_mpi            = f"--no-mpi" if args["no_mpi"] else ""
 
         mfc_script = ".\mfc.bat" if os.name == 'nt' else "./mfc.sh"
@@ -166,6 +166,10 @@ def __getitem__(self, key: str) -> str:
 
     def __setitem__(self, key: str, val: str):
         self.params[key] = val
+    
+    def __str__(self) -> str:
+        return f"tests/[bold magenta]{self.get_uuid()}[/bold magenta]: {self.trace}"
+
 
 
 @dataclasses.dataclass

toolchain/mfc/tests/pack.py

@@ -120,7 +120,7 @@ def generate(case: case.Case) -> Pack:
 
         for double in doubles:
             if math.isnan(double):
-                raise MFCException(f"A NaN was found while generating a pack file for test with UUID [bold magenta]{case.get_uuid()}[/bold magenta].")
+                raise MFCException(f"Test {case}: A NaN was found while generating a pack file.")
 
         entries.append(PackEntry(short_filepath,doubles))
 
@@ -152,21 +152,21 @@ def check_tolerance(case: case.Case, candidate: Pack, golden: Pack, tol: float)
 
     # Compare entry-count
     if len(candidate.entries) != len(golden.entries):
-        raise MFCException(f"{os.sep.join(['tests', uuid])}: Line count didn't match.")
+        raise MFCException(f"Test {case}: Line count didn't match.")
 
     # For every entry in the golden's pack
     for gIndex, gEntry in enumerate(golden.entries):
         # Find the corresponding entry in the candidate's pack
         cIndex, cEntry = common.find(lambda i, e: e.filepath == gEntry.filepath, candidate.entries)
 
         if cIndex == None:
-            raise MFCException(f"{os.sep.join(['tests', uuid])}: No reference of {gEntry.filepath} in the candidate's pack.")
+            raise MFCException(f"Test {case}: No reference of {gEntry.filepath} in the candidate's pack.")
 
         filepath: str = gEntry.filepath
 
         # Compare variable-count
         if len(gEntry.doubles) != len(cEntry.doubles):
-            raise MFCException(f"{os.sep.join(['tests', uuid])}: Variable count didn't match for {filepath}.")
+            raise MFCException(f"Test {case}: Variable count didn't match for {filepath}.")
 
         # Check if each variable is within tolerance
         for valIndex, (gVal, cVal) in enumerate(zip(gEntry.doubles, cEntry.doubles)):
@@ -176,7 +176,7 @@ def check_tolerance(case: case.Case, candidate: Pack, golden: Pack, tol: float)
 
             def raise_err(msg: str):
                 raise MFCException(f"""\
-{os.sep.join(['tests', uuid])}: Variable n°{valIndex+1} (1-indexed) in {filepath} {msg}:
+Test {case}: Variable n°{valIndex+1} (1-indexed) in {filepath} {msg}:
   - Description: {case.trace}
   - Candidate:   {cVal}
   - Golden:      {gVal}

toolchain/mfc/tests/tests.py

@@ -131,7 +131,7 @@ def handle_case(self, test: Case):
             if test.params.get("qbmm", 'F') == 'T':
                 tol = 1e-7
             elif test.params.get("bubbles", 'F') == 'T':
-                tol = 1e-10
+                tol = 1e-6
             else:
                 tol = 1e-12
 
@@ -144,8 +144,8 @@ def handle_case(self, test: Case):
             if cmd.returncode != 0:
                 cons.print(cmd.stdout)
                 raise MFCException(f"""\
-    {os.sep.join(['tests', test.get_uuid()])}: Failed to execute MFC [{test.trace}]. Above is the output of MFC.
-    You can find the output in {out_filepath}, and the case dictionary in {os.path.join(test.get_dirpath(), "case.py")}.""")
+Test {test}: Failed to execute MFC. Above is the output of MFC.
+You can find the output in {out_filepath}, and the case dictionary in {os.path.join(test.get_dirpath(), "case.py")}.""")
 
             pack = packer.generate(test)
             pack.save(os.path.join(test.get_dirpath(), "pack.txt"))
@@ -157,7 +157,7 @@ def handle_case(self, test: Case):
                 pack.save(golden_filepath)
             else:
                 if not os.path.isfile(golden_filepath):
-                    raise MFCException(f"{os.sep.join(['tests', test.get_uuid()])}: Golden file doesn't exist! To generate golden files, use the '-g' flag. [{test.trace}]")
+                    raise MFCException(f"Test {test}: Golden file doesn't exist! To generate golden files, use the '-g' flag.")
 
                 packer.check_tolerance(test, pack, packer.load(golden_filepath), tol)
 
@@ -168,5 +168,5 @@ def handle_case(self, test: Case):
             if not self.mfc.args["relentless"]:
                 raise exc
 
-            cons.print(f"[bold red]Failed Test [bold magenta]{test.get_uuid()}[/bold magenta] ! [/bold red] ({test.trace})")
+            cons.print(f"[bold red]Failed test {test}.[/bold red]")
             cons.print(f"{exc}")