#!/bin/bash -e

#
# Global config
#

set -o pipefail

# Determine current machine.

print_help() {
  echo "test_all_sandia <ARGS> <OPTIONS>:"
  echo "--kokkoskernels-path=/Path/To/KokkosKernels: Path to the KokkosKernels root directory"
  echo "    Defaults to root repo containing this script"
  echo "--kokkos-path=/Path/To/Kokkos: Path to the Kokkos root directory"
  echo "    Defaults to KokkosKernelsPath/../kokkos"
  echo ""
  echo "--kokkoskernels-branch=<Branch>: Branch to test. Only relevant for Spack builds."
  echo "    No default. Spack otherwise does dev-build from source."
  echo "--spack: Run spack builds rather than direct CMake tests"
  echo ""
  echo "--debug: Run tests in debug. Defaults to False"
  echo "--deprecated-code: Enable deprecated code (disabled by default)"
  echo "--boundscheck: Enable Kokkos_ENABLE_DEBUG_BOUNDS_CHECK to check View accesses within bounds."
  echo "--test-script: Test this script, not Kokkos"
  echo "--skip-hwloc: Do not do hwloc tests"
  echo "--num=N: Number of jobs to run in parallel"
  echo "--spot-check: Minimal test set to issue pull request"
  echo "--spot-check-tpls: Minimal test set enabling blas and lapack tpls"
  echo "--timeout: Max time before ctest timeout (in seconds)"
  echo "--dry-run: Just print what would be executed"
  echo "--build-only: Just do builds, don't run anything"
  echo "--opt-flag=FLAG: Optimization flag (default: -O3)"
  echo "--cxxflags-extra=FLAGS: Extra flags to be added to CXX_FLAGS"
  echo "--ldflags-extra=FLAGS: Extra flags to be added to LD_FLAGS"
  echo ""
  echo "--arch=ARCHITECTURE: overwrite architecture flags"
  echo "                     Provide a comma-separated list of arch codes (see available at link below):"
  echo "                       https://github.com/kokkos/kokkos/wiki/Compiling#table-43-architecture-variables"
  echo ""
  echo "--with-cuda-options=OPT: set KOKKOS_CUDA_OPTIONS"
  echo "                         Provide a comma-separated list from the following valid items:"
  echo "                           force_uvm,use_ldg,enable_lambda,rdc"
  echo ""
  echo "--with-options=OPT: set KOKKOS_OPTIONS"
  echo "                    Provide a comma-separated list from the following valid items:"
  echo "                      compiler_warnings"
  echo "                      aggressive_vectorization = add ivdep on loops"
  echo "                      disable_profiling = do not compile with profiling hooks"
  echo ""
  echo "--build-list=BUILD,BUILD,BUILD..."
  echo "    Provide a comma-separated list of builds instead of running all builds"
  echo "    Valid items:"
  echo "      OpenMP, Threads, Serial, OpenMP_Serial, Threads_Serial"
  echo "      Cuda_OpenMP, Cuda_Threads, Cuda_Serial"
  echo ""
  echo "--with-scalars=SCALARS: set KOKKOSKERNELS_SCALARS"
  echo "    Provide a comma-separated list scalar types"
  echo "    Valid items:"
  echo "      float, complex_float, double, complex_double"
  echo "        Example: SCALARS=double,complex_double"
  echo ""
  echo "--with-ordinals=ORDS: set KOKKOSKERNELS_ORDINALS"
  echo "    Provide a comma-separated list ordinal types"
  echo "    Valid items:"
  echo "      int, int64_t"
  echo ""
  echo "--with-offsets=OFFS: set KOKKOSKERNELS_OFFSETS"
  echo "    Provide a comma-separated list offset types"
  echo "    Valid items:"
  echo "      int, size_t"
  echo ""
  echo "--with-layouts=LAYOUTS: set KOKKOSKERNELS_LAYOUTS"
  echo "    Provide a comma-separated list layouts"
  echo "    Valid items:"
  echo "      LayoutLeft,LayoutRight"
  echo ""
  echo "--no-default-eti:  Do not include default ETI types for Kokkos Kernels"
  echo ""
  echo "--disable-test-eti-only:  Do not restrict testing to ETI types for Kokkos Kernels"
  echo ""
  echo "--with-spaces=SPACES:       Set spaces to be instantiated."
  echo "                                Options: hostspace, cudaspace, cudauvmspace"
  echo ""
  echo "--disable-perftests:  Do not build perftests for Kokkos Kernels"
  echo ""
  echo "--enable-perftests:  build perftests for Kokkos Kernels (default)"
  echo ""
  echo "--make-par-level=N:  Set parallelism level for builds (default: N=12)"
  echo ""
  echo "--with-tpls=TPLS: set KOKKOSKERNELS_ENABLE_TPLS"
  echo "    Provide a comma-separated list of TPLs"
  echo "    Valid items:"
  echo "      blas, mkl, cublas, cusparse, cusolver, magma, armpl, rocblas, rocsparse, rocsolver"
  echo ""
  echo "--cmake-flags=[CMAKE Command options]:  Set Kokkos Kernels cmake options not handled by script"
  echo "--kokkos-cmake-flags=[CMAKE Command options]:  Set Kokkos cmake options not handled by script"
  echo ""

  echo "ARGS: list of expressions matching compilers to test"
  echo "  supported compilers sems"
  for COMPILER_DATA in "${COMPILERS[@]}"; do
    ARR=($COMPILER_DATA)
    COMPILER=${ARR[0]}
    echo "    $COMPILER"
  done
  echo ""

  echo "Examples:"
  echo "  Run all tests"
  echo "  % test_all_sandia"
  echo ""
  echo "  Run all gcc tests"
  echo "  % test_all_sandia gcc"
  echo ""
  echo "  Run all gcc/4.8.4 and all intel tests"
  echo "  % test_all_sandia gcc/4.8.4 intel"
  echo ""
  echo "  Run all tests in debug"
  echo "  % test_all_sandia --debug"
  echo ""
  echo "  Run gcc/4.8.4 and only do OpenMP and OpenMP_Serial builds"
  echo "  % test_all_sandia gcc/4.8.4 --build-list=OpenMP,OpenMP_Serial"
  echo ""
  echo "If you want to kill the tests, do:"
  echo "  hit ctrl-z"
  echo "  % kill -9 %1"
  echo
}

MACHINE=""
HOSTNAME=$(hostname)
PROCESSOR=`uname -p`
CUDA_ENABLE_CMD=
HIP_ENABLE_CMD=
#Command(s) for accessing local modules on the current machine,
#e.g. "module use ..."
#This will be added to reproducer instructions/script.
MODULE_ENVIRONMENT=

if [[ "$HOSTNAME" == inouye* ]]; then
  MACHINE=inouye
fi

if [[ "$HOSTNAME" =~ weaver.* ]]; then
  MACHINE=weaver
  source /etc/profile.d/modules.sh
  module load git
fi

if [[ "$HOSTNAME" == *blake* ]]; then # Warning: very generic name
  MACHINE=blake
fi

if [[ "$HOSTNAME" == *solo* ]]; then # Warning: very generic name
  MACHINE=solo
fi

if [[ "$HOSTNAME" == kokkos-dev-2* ]]; then
  MACHINE=kokkos-dev-2
fi

if [[ "$HOSTNAME" == caraway* ]]; then # Warning: very generic name
  MACHINE=caraway
fi

if [[ "$HOSTNAME" == fat* ]]; then # Caraway MI250 queues
  MACHINE=vega90a_caraway
fi

if [[ "$HOSTNAME" == lean* ]]; then # Caraway MI210 queues
  MACHINE=vega90a_caraway
fi

if [[ "$HOSTNAME" == kokkos-dev\.sandia\.gov* ]]; then
  MACHINE=kokkos-dev
fi

if [[ "$HOSTNAME" == sogpu01* ]]; then
  MACHINE=sogpu
fi

if [[ "$HOSTNAME" == sorh7* ]]; then
  MACHINE=sorh7
fi

if [ ! -z "$SEMS_MODULEFILES_ROOT" ]; then
  if [[ "$MACHINE" = "" ]]; then
    MACHINE=sems
  fi
fi

if [[ "$MACHINE" = "" ]]; then
  echo "Unrecognized machine" >&2
  exit 1
fi

echo "Running on machine: $MACHINE"

GCC_BUILD_LIST="OpenMP,Threads,Serial,OpenMP_Serial,Threads_Serial"
ARM_GCC_BUILD_LIST="OpenMP,Serial,OpenMP_Serial"
INTEL_BUILD_LIST="OpenMP,Threads,Serial,OpenMP_Serial,Threads_Serial"
CLANG_BUILD_LIST="Threads,Serial,Threads_Serial"
CUDA_BUILD_LIST="Cuda_OpenMP,Cuda_Threads,Cuda_Serial"
CUDA_IBM_BUILD_LIST="Cuda_OpenMP,Cuda_Serial"

GCC_WARNING_FLAGS="-Wall,-Wunused-parameter,-Wshadow,-pedantic,-Werror,-Wsign-compare,-Wtype-limits,-Wignored-qualifiers,-Wempty-body,-Wclobbered,-Wuninitialized"
CLANG_WARNING_FLAGS="-Wall,-Wunused-parameter,-Wshadow,-pedantic,-Werror,-Wsign-compare,-Wtype-limits,-Wuninitialized"
INTEL_WARNING_FLAGS="-Wall,-Wunused-parameter,-Wshadow,-pedantic,-Werror,-Wsign-compare,-Wtype-limits,-Wuninitialized,-diag-disable=1011,-diag-disable=869"
CUDA_WARNING_FLAGS="-Wall,-Wunused-parameter,-Wshadow,-pedantic,-Werror,-Wsign-compare,-Wtype-limits,-Wuninitialized"
#CUDA_WARNING_FLAGS="-Wall,-Wunused-parameter,-Wshadow,-pedantic,-Wsign-compare,-Wtype-limits,-Wuninitialized"
PGI_WARNING_FLAGS=""

# Default. Machine specific can override.
DEBUG=False
ARGS=""
CUSTOM_BUILD_LIST=""
DRYRUN=False
BUILD_ONLY=False
declare -i NUM_JOBS_TO_RUN_IN_PARALLEL=1
TEST_SCRIPT=False
TEST_SPACK=False
SKIP_HWLOC=False
SPOT_CHECK=False
NO_DEFAULT_ETI=False
ENABLE_PERFTESTS=True
ENABLE_TEST_ETI_ONLY=True

PRINT_HELP=False
OPT_FLAG=""
CXX_FLAGS_EXTRA=""
LD_FLAGS_EXTRA=""
KOKKOS_OPTIONS=""

CXX_STANDARD="17"

GCC_VARIANTS="+blas+lapack +openmp+serial<SPACK_HOST_ARCH>"
CLANG_VARIANTS="+blas+lapack +openmp+serial<SPACK_HOST_ARCH>"
CUDA_VARIANTS="+cusparse+cublas +cuda+cuda_lambda+wrapper<SPACK_CUDA_ARCH>+cuda_uvm"
INTEL_VARIANTS="$GCC_VARIANTS"
PGI_VARIANTS="$GCC_VARIANTS"
SPACK_VARIANTS=("cuda 10.0 $CUDA_VARIANTS std=14"
          "cuda 10.1  $CUDA_VARIANTS"
          "cuda 9.2   $CUDA_VARIANTS"
          "gcc 5.3.0  $GCC_VARIANTS"
          "gcc 6.1.0  $GCC_VARIANTS"
          "gcc 7.2.0  $GCC_VARIANTS std=14"
          "gcc 7.3.0  $GCC_VARIANTS std=14"
          "gcc 8.3.0  $GCC_VARIANTS std=14"
          "gcc 9.1    $GCC_VARIANTS std=17"
          "gcc 9.2.0  $GCC_VARIANTS std=17"
          "intel 17.0.1 $INTEL_VARIANTS"
          "intel 18.0.5 $INTEL_VARIANTS"
          "intel 19.0.5 $INTEL_VARIANTS std=14"
          "clang 3.6.1 $CLANG_VARIANTS"
          "clang 3.7.1 $CLANG_VARIANTS"
          "clang 3.8.1 $CLANG_VARIANTS"
          "clang 3.9.0 $CLANG_VARIANTS"
          "clang 5.0.1 $CLANG_VARIANTS"
          "clang 7.0.1 $CLANG_VARIANTS"
          "clang 8.0   $CLANG_VARIANTS std=14"
          "clang 9.0.0 $CLANG_VARIANTS std=17"
          "pgi 19.4    $PGI_VARIANTS"
      )
SPACK_HOST_ARCH=""
SPACK_CUDA_ARCH=""
SPACK_CUDA_HOST_COMPILER=""

SPOT_CHECK_TPLS=False
KOKKOSKERNELS_ENABLE_TPL_CMD=
KOKKOSKERNELS_TPL_PATH_CMD=
KOKKOSKERNELS_TPL_LIBS_CMD=
KOKKOSKERNELS_EXTRA_LINKER_FLAGS_CMD=

# Enable double and complex_double in all builds by default
KOKKOSKERNELS_SCALARS="double,complex_double"
# Defaults for ordinals, offsets, and layouts match the CMake defaults
KOKKOSKERNELS_ORDINALS="int"
KOKKOSKERNELS_OFFSETS="int,size_t"
KOKKOSKERNELS_LAYOUTS="LayoutLeft"

CTESTTIMEOUT=2500

KOKKOS_DEPRECATED_CODE=""

MAKE_PAR_LEVEL=12

#
# Handle arguments.
#

while [[ $# > 0 ]]
do
  key="$1"

  case $key in
    --kokkoskernels-path*)
      KOKKOSKERNELS_PATH="${key#*=}"
      ;;
    --kokkos-path*)
      KOKKOS_PATH="${key#*=}"
      ;;
    --build-list*)
      CUSTOM_BUILD_LIST="${key#*=}"
      ;;
    --debug*)
      DEBUG=True
      ;;
    --boundscheck*)
      KOKKOS_BOUNDS_CHECK="--boundscheck"
      ;;
    --deprecated-code)
      KOKKOS_DEPRECATED_CODE="--deprecated-code"
      ;;
    --build-only*)
      BUILD_ONLY=True
      ;;
    --test-script*)
      TEST_SCRIPT=True
      ;;
    --spack*)
      TEST_SPACK=True
      ;;
    --kokkoskernels-branch*)
      KOKKOSKERNELS_BRANCH="${key#*=}"
      ;;
    --skip-hwloc*)
      SKIP_HWLOC=True
      ;;
    --num*)
      NUM_JOBS_TO_RUN_IN_PARALLEL="${key#*=}"
      ;;
    --dry-run*)
      DRYRUN=True
      ;;
    --spot-check-tpls*)
      SPOT_CHECK_TPLS=True
      ;;
    --spot-check*)
      SPOT_CHECK=True
      ;;
    --timeout*)
      CTESTTIMEOUT="${key#*=}"
      ;;
    --arch*)
      ARCH_FLAG="--arch=${key#*=}"
      ;;
    --opt-flag*)
      OPT_FLAG="${key#*=}"
      ;;
    --with-cuda-options*)
      KOKKOS_CUDA_OPTIONS="${key#*=}"
      export KOKKOS_CUDA_OPTIONS
      ;;
    --with-options*)
      KOKKOS_OPTIONS="${key#*=}"
      export KOKKOS_OPTIONS
      ;;
    --cxxflags-extra*)
      CXX_FLAGS_EXTRA="${key#*=}"
      ;;
    --cxxstandard*)
      FULL_CXX_STANDARD="${key#*=}"
      if [[ ${FULL_CXX_STANDARD} == *++* ]]; then
         CXX_STANDARD="${FULL_CXX_STANDARD#*++}"
      else
         CXX_STANDARD="${FULL_CXX_STANDARD}"
      fi
      ;;
    --make-par-level*)
      MAKE_PAR_LEVEL="${key#*=}"
      ;;
    --ldflags-extra*)
      LD_FLAGS_EXTRA="${key#*=}"
      ;;
    --with-scalars*)
      # Overwrite the default double and complex_double types
      KOKKOSKERNELS_SCALARS="${key#*=}"
      ;;
    --with-ordinals*)
      KOKKOSKERNELS_ORDINALS="${key#*=}"
      ;;
    --with-offsets*)
      KOKKOSKERNELS_OFFSETS="${key#*=}"
      ;;
    --with-layouts*)
      KOKKOSKERNELS_LAYOUTS="${key#*=}"
      ;;
    --no-default-eti*)
      NO_DEFAULT_ETI=True
      ;;
    --disable-test-eti-only*)
      ENABLE_TEST_ETI_ONLY=False
      ;;
    --disable-perftests*)
      ENABLE_PERFTESTS=False
      ;;
    --enable-perftests*)
      ENABLE_PERFTESTS=True
      ;;
    --with-spaces*)
      KOKKOSKERNELS_SPACES="${key#*=}"
      ;;
    --with-tpls*)
      KOKKOSKERNELS_ENABLE_TPLS="${key#*=}"
      ;;
    --cmake-flags*)
      PASSTHRU_CMAKE_FLAGS="${key#*=}"
      ;;
    --kokkos-cmake-flags*)
      KOKKOS_PASSTHRU_CMAKE_FLAGS="${key#*=}"
      ;;
    --help*)
      PRINT_HELP=True
      ;;
    *)
      # args, just append
      ARGS="$ARGS $1"
      ;;
  esac

  shift
done

if [[ "${SPOT_CHECK}" = "True" && "${SPOT_CHECK_TPLS}" = "True" ]]; then
    echo "Warning: --spot-check and --spot-check-tpls were requested but only one is allowed - defaulting to --spot-check"
    SPOT_CHECK_TPLS=False
fi

SCRIPT_KOKKOSKERNELS_ROOT=$( cd "$( dirname "$0" )" && cd .. && pwd )

# Set kokkos path.
if [ -z "$KOKKOSKERNELS_PATH" ]; then
  KOKKOSKERNELS_PATH=$SCRIPT_KOKKOSKERNELS_ROOT
else
  # Ensure KOKKOSKERNELS_PATH is abs path.
  KOKKOSKERNELS_PATH=$( cd $KOKKOSKERNELS_PATH && pwd )
fi

# Set kokkos path.
if [ -z "$KOKKOS_PATH" ]; then
  KOKKOS_PATH=$KOKKOSKERNELS_PATH/../kokkos
else
  # Ensure KOKKOS_PATH is abs path.
  KOKKOS_PATH=$( cd $KOKKOS_PATH && pwd )
  SPACK_DEV_BUILD_KOKKOS=$KOKKOS_PATH
fi



#
# Machine specific config.
#

if [ "$MACHINE" = "sems" ]; then
  module purge
  MODULE_ENVIRONMENT="sh /projects/sems/modulefiles/utils/sems-v2-modules-init.sh"
  eval "$MODULE_ENVIRONMENT"

  module load sems-cmake sems-git
  BASE_MODULE_LIST="sems-cmake,sems-<COMPILER_NAME>/<COMPILER_VERSION>"
  SKIP_HWLOC=True
  # No sems hwloc module

  if [ -z "$ARCH_FLAG" ]; then
    ARCH_FLAG=""
  fi

  # Format: (compiler module-list build-list exe-name warning-flag)
  COMPILERS=("gcc/8.3.0 $BASE_MODULE_LIST $GCC_BUILD_LIST g++ $GCC_WARNING_FLAGS"
             "gcc/10.1.0 $BASE_MODULE_LIST $GCC_BUILD_LIST g++ $GCC_WARNING_FLAGS"
             "clang/11.0.1 $BASE_MODULE_LIST $CLANG_BUILD_LIST clang++ $CLANG_WARNING_FLAGS"
             "clang/14.0.2 $BASE_MODULE_LIST $CLANG_BUILD_LIST clang++ $CLANG_WARNING_FLAGS"
             "intel/19.0.5 $BASE_MODULE_LIST $INTEL_BUILD_LIST icpc $INTEL_WARNING_FLAGS"
             "intel/19.1.2 $BASE_MODULE_LIST $INTEL_BUILD_LIST icpc $INTEL_WARNING_FLAGS"
             "intel/2021.3 $BASE_MODULE_LIST $INTEL_BUILD_LIST icpc $INTEL_WARNING_FLAGS"
  )
elif [ "$MACHINE" = "sogpu" ]; then
  module purge
  MODULE_ENVIRONMENT="sh /projects/sems/modulefiles/utils/sems-v2-modules-init.sh"
  eval "$MODULE_ENVIRONMENT"

  module load sems-cmake sems-git
  BASE_MODULE_LIST="sems-cmake,sems-<COMPILER_NAME>/<COMPILER_VERSION>"

  CUDA_MODULE_LIST="sems-cmake,sems-gcc/7.2.0,sems-<COMPILER_NAME>/<COMPILER_VERSION>"
  CUDA11_MODULE_LIST="sems-cmake,sems-gcc/8.3.0,sems-<COMPILER_NAME>/<COMPILER_VERSION>"
  CUDA11_MODULE_TPL_LIST="$CUDA11_MODULE_LIST,sems-openblas/0.3.10"
  SKIP_HWLOC=True
  # No sems hwloc module

  if [ -z "$ARCH_FLAG" ]; then
    ARCH_FLAG="--arch=Volta70"
  fi

  if [ "$SPOT_CHECK" = "True" ]; then
    # Format: (compiler module-list build-list exe-name warning-flag)
    COMPILERS=("cuda/11.1.0 $CUDA11_MODULE_LIST "Cuda_OpenMP" $KOKKOS_PATH/bin/nvcc_wrapper $CUDA_WARNING_FLAGS"
              )
  elif [ "$SPOT_CHECK_TPLS" = "True" ]; then
    # Format: (compiler module-list build-list exe-name warning-flag)
    COMPILERS=("cuda/11.1.0 $CUDA11_MODULE_TPL_LIST "Cuda_Serial" $KOKKOS_PATH/bin/nvcc_wrapper $CUDA_WARNING_FLAGS"
              )
  else
    # Format: (compiler module-list build-list exe-name warning-flag)
    COMPILERS=("gcc/8.3.0 $BASE_MODULE_LIST $GCC_BUILD_LIST g++ $GCC_WARNING_FLAGS"
               "gcc/10.1.0 $BASE_MODULE_LIST $GCC_BUILD_LIST g++ $GCC_WARNING_FLAGS"
               "clang/11.0.1 $BASE_MODULE_LIST $CLANG_BUILD_LIST clang++ $CLANG_WARNING_FLAGS"
               "clang/14.0.2 $BASE_MODULE_LIST $CLANG_BUILD_LIST clang++ $CLANG_WARNING_FLAGS"
               "intel/19.0.5 $BASE_MODULE_LIST $INTEL_BUILD_LIST icpc $INTEL_WARNING_FLAGS"
               "intel/19.1.2 $BASE_MODULE_LIST $INTEL_BUILD_LIST icpc $INTEL_WARNING_FLAGS"
               "intel/2021.3 $BASE_MODULE_LIST $INTEL_BUILD_LIST icpc $INTEL_WARNING_FLAGS"
               "cuda/11.1.0 $CUDA11_MODULE_LIST $CUDA_BUILD_LIST $KOKKOS_PATH/bin/nvcc_wrapper $CUDA_WARNING_FLAGS"
               "cuda/11.4.2 $CUDA11_MODULE_LIST $CUDA_BUILD_LIST $KOKKOS_PATH/bin/nvcc_wrapper $CUDA_WARNING_FLAGS"
              )

  fi
elif [ "$MACHINE" = "sorh7" ]; then
  module load sems-cmake/3.23.1 sems-git
  BASE_MODULE_LIST="sems-cmake/3.23.1,sems-<COMPILER_NAME>/<COMPILER_VERSION>"
  INTEL_BASE_MODULE_LIST="sems-cmake/3.23.1,sems-gcc/8.3.0,sems-<COMPILER_NAME>/<COMPILER_VERSION>"
  CLANG_BASE_MODULE_LIST="sems-cmake/3.23.1,sems-gcc/10.1.0,sems-<COMPILER_NAME>/<COMPILER_VERSION>"
  SKIP_HWLOC=True
  # No sems hwloc module

  if [ -z "$ARCH_FLAG" ]; then
    ARCH_FLAG="--arch=SKX"
  fi

  if [ "$SPOT_CHECK" = "True" ]; then
    # Format: (compiler module-list build-list exe-name warning-flag)
    COMPILERS=("intel/19.0.5 $INTEL_BASE_MODULE_LIST "OpenMP,Threads" icpc $INTEL_WARNING_FLAGS"
              )
  elif [ "$SPOT_CHECK_TPLS" = "True" ]; then
    # Format: (compiler module-list build-list exe-name warning-flag)
    COMPILERS=("intel/19.0.5 $INTEL_BASE_MODULE_LIST "OpenMP,Threads" icpc $INTEL_WARNING_FLAGS"
              )
  else
    # Format: (compiler module-list build-list exe-name warning-flag)
    COMPILERS=("gcc/8.3.0 $BASE_MODULE_LIST $GCC_BUILD_LIST g++ $GCC_WARNING_FLAGS"
               "clang/10.0.1 $CLANG_BASE_MODULE_LIST $CLANG_BUILD_LIST clang++ $CLANG_WARNING_FLAGS"
               "intel/19.0.5 $INTEL_BASE_MODULE_LIST $INTEL_BUILD_LIST icpc $INTEL_WARNING_FLAGS"
              )

  fi
elif [ "$MACHINE" = "inouye" ]; then
  MODULE_ENVIRONMENT="module purge"
  eval "$MODULE_ENVIRONMENT"
  SKIP_HWLOC=True
  export omp_proc_bind=close
  export omp_places=cores
  export omp_num_threads=47

  BASE_MODULE_LIST="cmake/3.17.0,<COMPILER_NAME>/<COMPILER_VERSION>"

  ARMPL_MODULE_TPL_LIST="cmake/3.17.0,gcc/10.2.0,<COMPILER_NAME>/<COMPILER_VERSION>"
  ARMPL_MODULE_TPL_LIST="cmake/3.17.0,gcc/10.2.0,<COMPILER_NAME>/<COMPILER_VERSION>"

  ARMCLANG_WARNING_FLAGS="-Wall,-Wshadow,-pedantic,-Wsign-compare,-Wtype-limits,-Wuninitialized"

  GCC_BUILD_LIST="OpenMP,Serial,OpenMP_Serial"

  if [ "$SPOT_CHECK" = "True" ]; then
    # Format: (compiler module-list build-list exe-name warning-flag)
    COMPILERS=("gcc/10.2.0 $BASE_MODULE_LIST "OpenMP_Serial" g++ $GCC_WARNING_FLAGS"
               #"arm/20.3  $BASE_MODULE_LIST "OpenMP_Serial" armclang++ $ARMCLANG_WARNING_FLAGS"
    )
  elif [ "$SPOT_CHECK_TPLS" = "True" ]; then
    # Format: (compiler module-list build-list exe-name warning-flag)
    COMPILERS=("armpl/20.3.0 $ARMPL_MODULE_TPL_LIST "OpenMP,Serial" g++ $GCC_WARNING_FLAGS"
               "armpl/21.0.0 $ARMPL_MODULE_TPL_LIST "OpenMP,Serial" g++ $GCC_WARNING_FLAGS"
               "armpl/21.1.0 $ARMPL_MODULE_TPL_LIST "OpenMP,Serial" g++ $GCC_WARNING_FLAGS"
    )
  else
    # Format: (compiler module-list build-list exe-name warning-flag)
    COMPILERS=("gcc/10.2.0 $BASE_MODULE_LIST $GCC_BUILD_LIST g++ $GCC_WARNING_FLAGS"
               "armpl/20.3.0 $ARMPL_MODULE_TPL_LIST $GCC_BUILD_LIST g++ $GCC_WARNING_FLAGS"
               "armpl/21.0.0 $ARMPL_MODULE_TPL_LIST $GCC_BUILD_LIST g++ $GCC_WARNING_FLAGS"
               "armpl/21.1.0 $ARMPL_MODULE_TPL_LIST $GCC_BUILD_LIST g++ $GCC_WARNING_FLAGS"
               #"arm/20.3 $BASE_MODULE_LIST $ARM_GCC_BUILD_LIST armclang++ $ARMCLANG_WARNING_FLAGS"
    )
  fi

  if [ -z "$ARCH_FLAG" ]; then
    ARCH_FLAG="--arch=A64FX"
  fi

  SPACK_HOST_ARCH="+a64fx"
elif [ "$MACHINE" = "weaver" ]; then
  # Use the legacy env for now until all modules are part of the new system
  MODULE_ENVIRONMENT="source /projects/ppc64le-pwr9-rhel8/legacy-env.sh"
  eval "$MODULE_ENVIRONMENT"
  SKIP_HWLOC=True

  GCC93_MODULE_TPL_LIST="cmake/3.23.1,<COMPILER_NAME>/<COMPILER_VERSION>,openblas/0.3.20/gcc/9.3.0,gcc/9.3.0"
  CLANG13_MODULE_TPL_LIST="cmake/3.23.1,<COMPILER_NAME>/<COMPILER_VERSION>,openblas/0.3.20/gcc/9.3.0,cuda/10.1.243"

  BASE_MODULE_LIST="cmake/3.23.1,<COMPILER_NAME>/<COMPILER_VERSION>"
  # Cuda/11 modules available rhel8 queue (rhel8 OS); gcc/8.3.1 load by default
  RHEL8_CUDA11_MODULE_LIST="cmake/3.23.1,cuda/11.2.2/gcc/8.3.1,openblas/0.3.18/gcc/8.3.1"

  # Don't do Threads on weaver
  GCC_IBM_BUILD_LIST="OpenMP,Serial,OpenMP_Serial"

  if [ "$SPOT_CHECK" = "True" ]; then
    # Format: (compiler module-list build-list exe-name warning-flag)
    COMPILERS=("cuda/11.2.2/gcc/8.3.1 $RHEL8_CUDA11_MODULE_LIST "Cuda_OpenMP" ${KOKKOS_PATH}/bin/nvcc_wrapper $CUDA_WARNING_FLAGS"
               "gcc/9.3.0 $BASE_MODULE_LIST $GCC_IBM_BUILD_LIST g++ $GCC_WARNING_FLAGS"
    )
  elif [ "$SPOT_CHECK_TPLS" = "True" ]; then
    # Format: (compiler module-list build-list exe-name warning-flag)
    COMPILERS=("cuda/11.2.2/gcc/8.3.1 $RHEL8_CUDA11_MODULE_LIST "Cuda_Serial" ${KOKKOS_PATH}/bin/nvcc_wrapper $CUDA_WARNING_FLAGS"
               "gcc/9.3.0 $GCC93_MODULE_TPL_LIST "OpenMP,Serial" g++ $GCC_WARNING_FLAGS"
               "clang/13.0.0 $CLANG13_MODULE_TPL_LIST "Cuda" clang++ $CUDA_WARNING_FLAGS"
    )
  else
    # Format: (compiler module-list build-list exe-name warning-flag)
    COMPILERS=("gcc/8.3.1 $BASE_MODULE_LIST $GCC_IBM_BUILD_LIST g++ $GCC_WARNING_FLAGS"
               "gcc/9.3.0 $BASE_MODULE_LIST $GCC_IBM_BUILD_LIST g++ $GCC_WARNING_FLAGS"
               "cuda/11.2.2/gcc/8.3.1 $RHEL8_CUDA11_MODULE_LIST $CUDA_IBM_BUILD_LIST ${KOKKOS_PATH}/bin/nvcc_wrapper $CUDA_WARNING_FLAGS"
               "clang/13.0.0 $CLANG13_MODULE_TPL_LIST $CUDA_IBM_BUILD_LIST clang++ $CUDA_WARNING_FLAGS"
    )
  fi

  if [ -z "$ARCH_FLAG" ]; then
    ARCH_FLAG="--arch=Power9,Volta70"
  fi

  SPACK_HOST_ARCH="+power9"
  SPACK_CUDA_ARCH="+volta70"
elif [ "$MACHINE" = "caraway" ]; then
  SKIP_HWLOC=True
  # BUILD_ONLY=True
  # report_and_log_test_result: only testing compilation of code for now,
  #   output description and success based only on build succes; build time output (no run-time)

  BASE_MODULE_LIST="cmake,<COMPILER_NAME>/<COMPILER_VERSION>"
  ROCM520_MODULE_LIST="$BASE_MODULE_LIST,openblas/0.3.20"

  HIPCLANG_BUILD_LIST="Hip_Serial"
  HIPCLANG_WARNING_FLAGS=""

  if [ "$SPOT_CHECK_TPLS" = "True" ]; then
    # Format: (compiler module-list build-list exe-name warning-flag)
    COMPILERS=("rocm/5.2.0 $ROCM520_MODULE_LIST $HIPCLANG_BUILD_LIST hipcc $HIPCLANG_WARNING_FLAGS"
    )
  else
    # Format: (compiler module-list build-list exe-name warning-flag)
    COMPILERS=("rocm/5.2.0 $BASE_MODULE_LIST $HIPCLANG_BUILD_LIST hipcc $HIPCLANG_WARNING_FLAGS"
               "gcc/11.3.0 $BASE_MODULE_LIST $GCC_BUILD_LIST g++ $GCC_WARNING_FLAGS"
    )
  fi



  if [ -z "$ARCH_FLAG" ]; then
    ARCH_FLAG="--arch=VEGA908"
  fi
elif [ "$MACHINE" = "vega90a_caraway" ]; then
  SKIP_HWLOC=True
  # BUILD_ONLY=True
  # report_and_log_test_result: only testing compilation of code for now,
  #   output description and success based only on build succes; build time output (no run-time)

  BASE_MODULE_LIST="cmake,<COMPILER_NAME>/<COMPILER_VERSION>"
  ROCM520_MODULE_LIST="$BASE_MODULE_LIST,openblas/0.3.20"
  ROCM_TPL_MODULE_LIST="$BASE_MODULE_LIST,openblas/0.3.23"

  HIPCLANG_BUILD_LIST="Hip_Serial"
  HIPCLANG_WARNING_FLAGS=""

  if [ "$SPOT_CHECK_TPLS" = "True" ]; then
    # Format: (compiler module-list build-list exe-name warning-flag)
    COMPILERS=("rocm/5.6.1 $ROCM_TPL_MODULE_LIST $HIPCLANG_BUILD_LIST hipcc $HIPCLANG_WARNING_FLAGS"
               "rocm/6.0.0 $ROCM_TPL_MODULE_LIST $HIPCLANG_BUILD_LIST hipcc $HIPCLANG_WARNING_FLAGS"
    )
  else
    # Format: (compiler module-list build-list exe-name warning-flag)
    COMPILERS=("rocm/5.2.0 $BASE_MODULE_LIST $HIPCLANG_BUILD_LIST hipcc $HIPCLANG_WARNING_FLAGS"
               "rocm/5.6.1 $BASE_MODULE_LIST $HIPCLANG_BUILD_LIST hipcc $HIPCLANG_WARNING_FLAGS"
               "rocm/6.0.0 $BASE_MODULE_LIST $HIPCLANG_BUILD_LIST hipcc $HIPCLANG_WARNING_FLAGS"
               "gcc/11.3.0 $BASE_MODULE_LIST $GCC_BUILD_LIST g++ $GCC_WARNING_FLAGS"
    )
  fi



  if [ -z "$ARCH_FLAG" ]; then
    ARCH_FLAG="--arch=VEGA90A"
  fi
elif [ "$MACHINE" = "blake" ]; then
  MODULE_ENVIRONMENT="source /projects/x86-64-icelake-rocky8/spack-config/blake-setup-user-module-env.sh"
  eval "$MODULE_ENVIRONMENT"
  SKIP_HWLOC=True

  module load cmake

  BASE_MODULE_LIST="cmake,<COMPILER_NAME>/<COMPILER_VERSION>"
  BASE_MODULE_LIST_TPLS="cmake,<COMPILER_NAME>/<COMPILER_VERSION>,openblas/0.3.23"
  BASE_MODULE_LIST_ONEAPI_202310="cmake,<COMPILER_NAME>-oneapi-compilers/<COMPILER_VERSION>,intel-oneapi-dpl/2022.1.0,intel-oneapi-mkl/2023.1.0,intel-oneapi-tbb/2021.9.0"
  BASE_MODULE_LIST_ONEAPI_202320="cmake,<COMPILER_NAME>-oneapi-compilers/<COMPILER_VERSION>,intel-oneapi-dpl/2022.2.0,intel-oneapi-mkl/2023.2.0,intel-oneapi-tbb/2021.10.0"
  ONEAPI_FLAGS_EXTRA="-fp-model=precise"
  LLVM_EXTRA_FLAGS="-fPIC ${CLANG_WARNING_FLAGS}"
  # Remove -Wuninitialized: compiler issues show up with Threads backend
  GCC11_WARNING_FLAGS="-Wall,-Wunused-parameter,-Wshadow,-pedantic,-Werror,-Wsign-compare,-Wtype-limits,-Wignored-qualifiers,-Wempty-body,-Wclobbered"
  # update KOKKOS_PASSTHRU_CMAKE_FLAGS to disable onedpl on Blake
  KOKKOS_PASSTHRU_CMAKE_FLAGS="${KOKKOS_PASSTHRU_CMAKE_FLAGS} -DKokkos_ENABLE_ONEDPL=OFF"

  if [ "$SPOT_CHECK" = "True" ]; then
      # Format: (compiler module-list build-list exe-name warning-flag)
    COMPILERS=("intel/2023.1.0 $BASE_MODULE_LIST_ONEAPI_202310 "OpenMP,Threads,Serial" icpx $ONEAPI_FLAGS_EXTRA"
               "intel/2023.2.0 $BASE_MODULE_LIST_ONEAPI_202320 "OpenMP,Threads,Serial" icpx $ONEAPI_FLAGS_EXTRA"
               "llvm/15.0.7 $BASE_MODULE_LIST "Threads,Serial" clang++ $LLVM_EXTRA_FLAGS"
               "gcc/11.3.0 $BASE_MODULE_LIST "OpenMP,Threads,Serial" g++ $GCC11_WARNING_FLAGS"
               "gcc/12.2.0 $BASE_MODULE_LIST "OpenMP,Threads,Serial" g++ $GCC11_WARNING_FLAGS"
    )
  elif [ "$SPOT_CHECK_TPLS" = "True" ]; then
      # Format: (compiler module-list build-list exe-name warning-flag)
      # Known issues:
      # gcc/12.2.0+openblas/0.3.23 with OpenMP: internal compiler error: in get_vectype_for_scalar_type, at tree-vect-stmts
    COMPILERS=("intel/2023.1.0 $BASE_MODULE_LIST_ONEAPI_202310 "OpenMP,Threads,Serial" icpx $ONEAPI_FLAGS_EXTRA"
               "intel/2023.2.0 $BASE_MODULE_LIST_ONEAPI_202320 "OpenMP,Threads,Serial" icpx $ONEAPI_FLAGS_EXTRA"
               "llvm/15.0.7 $BASE_MODULE_LIST_TPLS "Threads,Serial" clang++ $LLVM_EXTRA_FLAGS"
               "gcc/11.3.0 $BASE_MODULE_LIST_TPLS "OpenMP,Threads,Serial" g++ $GCC11_WARNING_FLAGS"
               "gcc/12.2.0 $BASE_MODULE_LIST_TPLS "OpenMP,Threads,Serial" g++ $GCC11_WARNING_FLAGS"
    )
  else
      # gcc/12.2.0 with OpenMP: internal compiler error: in get_vectype_for_scalar_type, at tree-vect-stmts
    COMPILERS=("intel/2023.1.0 $BASE_MODULE_LIST_ONEAPI_202310 $INTEL_BUILD_LIST icpx $ONEAPI_FLAGS_EXTRA"
               "intel/2023.2.0 $BASE_MODULE_LIST_ONEAPI_202320 $INTEL_BUILD_LIST icpx $ONEAPI_FLAGS_EXTRA"
               "llvm/15.0.7 $BASE_MODULE_LIST $CLANG_BUILD_LIST clang++ $LLVM_EXTRA_FLAGS"
               "gcc/11.3.0 $BASE_MODULE_LIST $GCC_BUILD_LIST g++ $GCC11_WARNING_FLAGS"
               "gcc/12.2.0 $BASE_MODULE_LIST $GCC_BUILD_LIST g++ $GCC11_WARNING_FLAGS"
    )

  fi

  if [ -z "$ARCH_FLAG" ]; then
    ARCH_FLAG="--arch=SPR"
  fi
  SPACK_HOST_ARCH="+spr"
elif [ "$MACHINE" = "solo" ]; then
  SKIP_HWLOC=True
  export SLURM_TASKS_PER_NODE=32

  module load cmake/3.22.3

  BASE_MODULE_LIST="cmake/3.22.3,<COMPILER_NAME>/<COMPILER_VERSION>"
  BASE_MODULE_LIST_LLVM="cmake/3.22.3,<COMPILER_NAME>/<COMPILER_VERSION>,gnu/10.2.1"
  BASE_MODULE_LIST_INTEL="cmake/3.22.3,gnu/8.2.1,<COMPILER_NAME>/<COMPILER_VERSION>"
  ONEAPI_WARNING_FLAGS=""

  GNU102_MODULE_TPL_LIST="$BASE_MODULE_LIST,openblas/0.3.21"

  if [ "$SPOT_CHECK" = "True" ]; then
    COMPILERS=("gnu/10.2.1 $BASE_MODULE_LIST "Threads_Serial,OpenMP" g++ $GNU_WARNING_FLAGS"
               "llvm/10.0.1 $BASE_MODULE_LIST_LLVM "Threads_Serial" clang++ $CLANG_WARNING_FLAGS"
    )
  elif [ "$SPOT_CHECK_TPLS" = "True" ]; then
    COMPILERS=("intel/19.0.5.281 $BASE_MODULE_LIST_INTEL,mkl/19.0.5.281 "OpenMP,Threads" icpc $INTEL_WARNING_FLAGS"
               "gnu/10.2.1 $GNU102_MODULE_TPL_LIST "OpenMP_Serial" g++ $GNU_WARNING_FLAGS"
    )
  else
      ###"clang/10.0.1 $BASE_MODULE_LIST $CLANG_BUILD_LIST clang++ $CLANG_WARNING_FLAGS"
    COMPILERS=("gnu/10.2.1 $BASE_MODULE_LIST $GNU_BUILD_LIST g++ $GNU_WARNING_FLAGS"
    )

  fi

  if [ -z "$ARCH_FLAG" ]; then
    ARCH_FLAG="--arch=BDW"
  fi
  SPACK_HOST_ARCH="+bdw"
elif [ "$MACHINE" = "kokkos-dev-2" ]; then
  MODULE_ENVIRONMENT="source /projects/sems/modulefiles/utils/sems-archive-modules-init.sh ; module use /home/projects/x86-64/modulefiles/local"
  eval "$MODULE_ENVIRONMENT"
  module load sems-archive-env

  module load sems-archive-git
  module load sems-archive-tex
  module load sems-archive-cmake/3.17.1
  module load sems-archive-gdb

  SKIP_HWLOC=True

  BASE_MODULE_LIST="sems-archive-env,sems-archive-cmake/3.17.1,sems-archive-<COMPILER_NAME>/<COMPILER_VERSION>"
  INTEL_BASE_MODULE_LIST="sems-archive-env,sems-archive-cmake/3.17.1,sems-archive-gcc/8.3.0,sems-archive-<COMPILER_NAME>/<COMPILER_VERSION>"
  CLANG_BASE_MODULE_LIST="sems-archive-env,sems-archive-cmake/3.17.1,sems-archive-gcc/9.2.0,sems-archive-<COMPILER_NAME>/<COMPILER_VERSION>"
  GCC91_MODULE_LIST="sems-archive-env,sems-archive-cmake/3.17.1,<COMPILER_NAME>/<COMPILER_VERSION>"
  NVCC_SEMSMODULE_LIST="sems-archive-env,sems-archive-cmake/3.17.1,sems-archive-gcc/8.3.0,sems-archive-<COMPILER_NAME>/<COMPILER_VERSION>"
  NVCC11_MODULE_LIST="sems-archive-env,sems-archive-cmake/3.17.1,sems-archive-gcc/9.2.0,<COMPILER_NAME>/<COMPILER_VERSION>"
  LOCAL_MODULE_LIST="cmake/3.21.1,<COMPILER_NAME>/<COMPILER_VERSION>"

  BUILD_LIST_CUDA_NVCC="Cuda_Serial,Cuda_Threads"
  BUILD_LIST_CLANG="Serial,Threads,OpenMP"

  CLANG8_CUDA_WARNING_FLAGS="-Wall,-Wshadow,-pedantic,-Werror,-Wsign-compare,-Wtype-limits,-Wuninitialized,-Wno-pass-failed"

  if [ "$SPOT_CHECK" = "True" ]; then
    # Format: (compiler module-list build-list exe-name warning-flag)
    COMPILERS=("gcc/8.3.0 $BASE_MODULE_LIST "Serial" g++ $GCC_WARNING_FLAGS"
               "gcc/9.1 $GCC91_MODULE_LIST "OpenMP,Serial" g++ $GCC_WARNING_FLAGS"
               "intel/19.0.5 $INTEL_BASE_MODULE_LIST "Threads" icpc $INTEL_WARNING_FLAGS"
               "clang/9.0.0 $CLANG_BASE_MODULE_LIST "Serial,Threads" clang++ $CLANG_WARNING_FLAGS"
               "cuda/11.0 $NVCC11_MODULE_LIST "Cuda_OpenMP" $KOKKOS_PATH/bin/nvcc_wrapper $CUDA_WARNING_FLAGS"
    )
  elif [ "$SPOT_CHECK_TPLS" = "True" ]; then
    # Format: (compiler module-list build-list exe-name warning-flag)
    COMPILERS=("gcc/8.3.0 $BASE_MODULE_LIST "Serial" g++ $GCC_WARNING_FLAGS"
               "gcc/9.1 $GCC91_MODULE_LIST "OpenMP,Serial" g++ $GCC_WARNING_FLAGS"
               "intel/19.0.5 $INTEL_BASE_MODULE_LIST "Threads" icpc $INTEL_WARNING_FLAGS"
               "clang/9.0.0 $CLANG_BASE_MODULE_LIST "Serial,Threads" clang++ $CLANG_WARNING_FLAGS"
               "cuda/11.0 $NVCC11_MODULE_LIST "Cuda_OpenMP" $KOKKOS_PATH/bin/nvcc_wrapper $CUDA_WARNING_FLAGS"
    )
  else
    # Format: (compiler module-list build-list exe-name warning-flag)
    COMPILERS=("cuda/11.0 $NVCC11_MODULE_LIST $CUDA_BUILD_LIST $KOKKOS_PATH/bin/nvcc_wrapper $CUDA_WARNING_FLAGS"
               "cuda/11.1 $NVCC_SEMSMODULE_LIST $CUDA_BUILD_LIST $KOKKOS_PATH/bin/nvcc_wrapper $CUDA_WARNING_FLAGS"
               "cuda/11.2 $NVCC11_MODULE_LIST $CUDA_BUILD_LIST $KOKKOS_PATH/bin/nvcc_wrapper $CUDA_WARNING_FLAGS"
               "cuda/11.7 $NVCC11_MODULE_LIST $CUDA_BUILD_LIST $KOKKOS_PATH/bin/nvcc_wrapper $CUDA_WARNING_FLAGS"
               "cuda/12.0 $NVCC11_MODULE_LIST $CUDA_BUILD_LIST $KOKKOS_PATH/bin/nvcc_wrapper $CUDA_WARNING_FLAGS"
               "gcc/8.3.0 $BASE_MODULE_LIST $GCC_BUILD_LIST g++ $GCC_WARNING_FLAGS"
               "gcc/9.1 $GCC91_MODULE_LIST $GCC_BUILD_LIST g++ $GCC_WARNING_FLAGS"
               "gcc/9.2.0 $BASE_MODULE_LIST $GCC_BUILD_LIST g++ $GCC_WARNING_FLAGS"
               "gcc/10.3 $LOCAL_MODULE_LIST $GCC_BUILD_LIST g++ $GCC_WARNING_FLAGS"
               "gcc/11.1 $LOCAL_MODULE_LIST $GCC_BUILD_LIST g++ $GCC_WARNING_FLAGS"
               "intel/19.0.5 $INTEL_BASE_MODULE_LIST $INTEL_BUILD_LIST icpc $INTEL_WARNING_FLAGS"
               "clang/9.0.0 $CLANG_BASE_MODULE_LIST $CLANG_BUILD_LIST clang++ $CLANG_WARNING_FLAGS"
               "clang/10.0.0 $CLANG_BASE_MODULE_LIST $BUILD_LIST_CLANG clang++ $CLANG_WARNING_FLAGS"
               "clang/13.0.0 $LOCAL_MODULE_LIST $BUILD_LIST_CLANG clang++ $CLANG_WARNING_FLAGS"
    )
  fi

  if [ -z "$ARCH_FLAG" ]; then
    ARCH_FLAG="--arch=Volta70"
  fi
  SPACK_CUDA_ARCH="+volta70"
else
  echo "Unhandled machine $MACHINE" >&2
  exit 1
fi

export OMP_NUM_THREADS=${omp_num_threads:=8}
export OMP_PROC_BIND=${omp_proc_bind:=spread}
export OMP_PLACES=${omp_places:=cores}
export KOKKOS_NUM_THREADS=8

declare -i NUM_RESULTS_TO_KEEP=7

RESULT_ROOT_PREFIX=TestAll

if [ "$PRINT_HELP" = "True" ]; then
  print_help
  exit 0
fi

UNCOMMITTED=`cd ${KOKKOSKERNELS_PATH}; git status --porcelain 2>/dev/null`
if ! [ -z "$UNCOMMITTED" ]; then
  echo "WARNING!! THE FOLLOWING CHANGES ARE UNCOMMITTED!! :"
  echo "$UNCOMMITTED"
  echo ""
fi

GITSTATUS=`cd ${KOKKOSKERNELS_PATH}; git log -n 1 --format=oneline`
echo "KokkosKernels Repository Status: " ${GITSTATUS}
echo ""
KOKKOSGITSTATUS=`cd ${KOKKOS_PATH}; git log -n 1 --format=oneline`
echo "Kokkos Repository Status: " ${KOKKOSGITSTATUS}
echo ""
echo ""

# Set build type.
if [ "$DEBUG" = "True" ]; then
  BUILD_TYPE=debug
else
  BUILD_TYPE=release
fi

# If no args provided, do all compilers.
if [ -z "$ARGS" ]; then
  ARGS='?'
fi

# Process args to figure out which compilers to test.
COMPILERS_TO_TEST=""

for ARG in $ARGS; do
  for COMPILER_DATA in "${COMPILERS[@]}"; do
    ARR=($COMPILER_DATA)
    COMPILER=${ARR[0]}

    if [[ "$COMPILER" = $ARG* ]]; then
      if [[ "$COMPILERS_TO_TEST" != *${COMPILER}* ]]; then
        COMPILERS_TO_TEST="$COMPILERS_TO_TEST $COMPILER"
      else
        echo "Tried to add $COMPILER twice"
      fi
     fi
  done
done

if [ "$COMPILERS_TO_TEST" == "" ]; then
   echo "-----------------------------------------------"
   echo "   !!!! Invalid Compiler provided  '$ARGS' !!!!"
   echo "-----------------------------------------------"
   print_help
   exit 1
fi


#
# Functions.
#

# get_compiler_name <COMPILER>
get_compiler_name() {
  echo $1 | cut -d/ -f1
}

# get_compiler_version <COMPILER>
get_compiler_version() {
  echo $1 | cut -d/ -f2
}

# Do not call directly.
get_compiler_data() {
  local compiler=$1
  local item=$2
  local compiler_name=$(get_compiler_name $compiler)
  local compiler_vers=$(get_compiler_version $compiler)

  local compiler_data
  for compiler_data in "${COMPILERS[@]}" ; do
    local arr=($compiler_data)
    if [ "$compiler" = "${arr[0]}" ]; then
      echo "${arr[$item]}" | tr , ' ' | sed -e "s/<COMPILER_NAME>/$compiler_name/g" -e "s/<COMPILER_VERSION>/$compiler_vers/g"
      return 0
    fi
  done

  # Not found.
  echo "Unrecognized compiler $compiler" >&2
  exit 1
}

get_variants_data() {
  local compiler=$1
  local item=$2

  local variants_data
  for variants_data in "${SPACK_VARIANTS[@]}" ; do
    local arr=($variants_data)
    matcher="${arr[0]}/${arr[1]}"
    if [ "$compiler" = "$matcher" ]; then
      if [ "$item" = 4 ]; then
         echo "%${arr[0]}@${arr[1]}"
      else
         echo "${arr[$item]}"| tr , ' ' | sed -e "s/<SPACK_HOST_ARCH>/$SPACK_HOST_ARCH/g" -e "s/<SPACK_CUDA_ARCH>/$SPACK_CUDA_ARCH/g"
      fi
      return 0
    fi
  done

  # Not found.
  echo "Unrecognized compiler $compiler when looking for Spack variants" >&2
  exit 1
}

#
# For all getters, usage: <GETTER> <COMPILER>
#

get_compiler_modules() {
  get_compiler_data $1 1
}

get_compiler_build_list() {
  get_compiler_data $1 2
}

get_compiler_exe_name() {
  get_compiler_data $1 3
}

get_compiler_warning_flags() {
  get_compiler_data $1 4
}

get_kernels_variants() {
  get_variants_data $1 2
}

get_kokkos_variants() {
  get_variants_data $1 3
}

get_compiler_variant() {
  get_variants_data $1 4
}

run_cmd() {
  echo "RUNNING: $*"
  if [ "$DRYRUN" != "True" ]; then
    eval "$* 2>&1"
  fi
}

# report_and_log_test_results <SUCCESS> <DESC> <COMMENT>
report_and_log_test_result() {
  # Use sane var names.
  local success=$1; local desc=$2; local comment=$3;

  if [ "$success" = "0" ]; then
    echo "  PASSED $desc"
    echo $comment > $PASSED_DIR/$desc
  else
    # For failures, comment should be the name of the phase that failed.
    echo "  FAILED $desc" >&2
    echo $comment > $FAILED_DIR/$desc
    cat ${desc}.${comment}.log
  fi
}

setup_env() {
  local compiler=$1
  local compiler_modules=$(get_compiler_modules $compiler)

  # Test UVM for cuda/9.2* builds
  if [[ "$compiler" == cuda/9.2* ]]; then
    if [[ "$KOKKOS_CUDA_OPTIONS" = "" ]]; then
      export KOKKOS_CUDA_OPTIONS="force_uvm"
    else
      export KOKKOS_CUDA_OPTIONS="${KOKKOS_CUDA_OPTIONS},force_uvm"
    fi
    echo "cuda/9.2 + UVM Check KOKKOS_CUDA_OPTIONS: $KOKKOS_CUDA_OPTIONS"
  fi

  # Default tpls passed in via command-line - users responsibility to ensure this works if set manually
  KOKKOSKERNELS_ENABLE_TPL_CMD="--with-tpls=$KOKKOSKERNELS_ENABLE_TPLS"
  echo "SETUP_ENV: compiler=$compiler modules=$compiler_modules"
  local NEW_TPL_LIST=
  # Reset KOKKOSKERNELS_ENABLE_TPL_CMD is --spot-check-tpls used
  if [[ "${SPOT_CHECK_TPLS}" = "True" ]]; then
    # device tpls
    if [[ "$compiler" == cuda* ]]; then
      NEW_TPL_LIST="cublas,cusparse,cusolver,"
      export KOKKOS_CUDA_OPTIONS="${KOKKOS_CUDA_OPTIONS},enable_lambda"
    fi
    if [[ "$compiler" == rocm* ]]; then
      NEW_TPL_LIST="rocblas,rocsparse,rocsolver,"
    fi
    # host tpls - use mkl with intel, else use host blas
    if [[ "$compiler" == intel* ]]; then
      NEW_TPL_LIST="mkl,"
    else
      if [[ "$compiler" == armpl* ]]; then
        NEW_TPL_LIST="armpl,${NEW_TPL_LIST}"
      else
        NEW_TPL_LIST="blas,${NEW_TPL_LIST}"
      fi
    fi

    # Overwrite new tpl list with trailing comma removed
    NEW_TPL_LIST=$(echo ${NEW_TPL_LIST} | sed 's/.\w*$//')
    #echo "TESTING NEW_TPL_LIST=$NEW_TPL_LIST"

    KOKKOSKERNELS_ENABLE_TPL_CMD="--with-tpls=${KOKKOSKERNELS_ENABLE_TPLS},${NEW_TPL_LIST}"
    #echo "TPL USAGE: KOKKOSKERNELS_ENABLE_TPL_CMD=$KOKKOSKERNELS_ENABLE_TPL_CMD"
  fi

  module purge

  local mod
  for mod in $compiler_modules; do
    #echo "Loading module $mod"
    module load $mod 2>&1
    # It is ridiculously hard to check for the success of a loaded
    # module. Module does not return error codes and piping to grep
    # causes module to run in a subshell.
    module list 2>&1 | grep "$mod" >& /dev/null || return 1

    if [[ "${SPOT_CHECK_TPLS}" = "True" ]]; then
      # Some machines will require explicitly setting include dirs and libs
      if ([[ "$MACHINE" = weaver* ]] || [[ "$MACHINE" = sogpu* ]]) && [[ "$mod" = openblas* ]]; then
        BLAS_LIBRARY_DIRS="${OPENBLAS_ROOT}/lib"
        LAPACK_LIBRARY_DIRS="${OPENBLAS_ROOT}/lib"
        BLAS_LIBRARIES="blas"
        LAPACK_LIBRARIES="lapack"
        KOKKOSKERNELS_TPL_PATH_CMD="--user-blas-path=${BLAS_LIBRARY_DIRS} --user-lapack-path=${LAPACK_LIBRARY_DIRS}"
        KOKKOSKERNELS_TPL_LIBS_CMD="--user-blas-lib=${BLAS_LIBRARIES} --user-lapack-lib=${LAPACK_LIBRARIES}"
        KOKKOSKERNELS_EXTRA_LINKER_FLAGS_CMD="--extra-linker-flags=-lgfortran,-lm"
        echo "TPL PATHS: KOKKOSKERNELS_TPL_PATH_CMD=$KOKKOSKERNELS_TPL_PATH_CMD"
        echo "TPL LIBS:  KOKKOSKERNELS_TPL_LIBS_CMD=$KOKKOSKERNELS_TPL_LIBS_CMD"
      elif [[ "$MACHINE" = blake* ]] && [[ "$mod" = openblas* ]]; then
        BLAS_LIBRARY_DIRS="${OPENBLAS_ROOT}/lib"
        LAPACK_LIBRARY_DIRS="${OPENBLAS_ROOT}/lib"
        BLAS_LIBRARIES="openblas"
        LAPACK_LIBRARIES="openblas"
        KOKKOSKERNELS_TPL_PATH_CMD="--user-blas-path=${BLAS_LIBRARY_DIRS} --user-lapack-path=${LAPACK_LIBRARY_DIRS}"
        KOKKOSKERNELS_TPL_LIBS_CMD="--user-blas-lib=${BLAS_LIBRARIES} --user-lapack-lib=${LAPACK_LIBRARIES}"
        KOKKOSKERNELS_EXTRA_LINKER_FLAGS_CMD="--extra-linker-flags=-lgfortran,-lm"
        echo "TPL PATHS: KOKKOSKERNELS_TPL_PATH_CMD=$KOKKOSKERNELS_TPL_PATH_CMD"
        echo "TPL LIBS:  KOKKOSKERNELS_TPL_LIBS_CMD=$KOKKOSKERNELS_TPL_LIBS_CMD"
      elif ([[ "$MACHINE" = weaver* ]]) && [[ "$mod" = netlib* ]]; then
        BLAS_LIBRARY_DIRS="${BLAS_ROOT}/lib"
        LAPACK_LIBRARY_DIRS="${BLAS_ROOT}/lib"
        BLAS_LIBRARIES="blas"
        LAPACK_LIBRARIES="lapack"
        KOKKOSKERNELS_TPL_PATH_CMD="--user-blas-path=${BLAS_LIBRARY_DIRS} --user-lapack-path=${LAPACK_LIBRARY_DIRS}"
        KOKKOSKERNELS_TPL_LIBS_CMD="--user-blas-lib=${BLAS_LIBRARIES} --user-lapack-lib=${LAPACK_LIBRARIES}"
        KOKKOSKERNELS_EXTRA_LINKER_FLAGS_CMD="--extra-linker-flags=-lgfortran,-lm"
        echo "TPL PATHS: KOKKOSKERNELS_TPL_PATH_CMD=$KOKKOSKERNELS_TPL_PATH_CMD"
        echo "TPL LIBS:  KOKKOSKERNELS_TPL_LIBS_CMD=$KOKKOSKERNELS_TPL_LIBS_CMD"
      fi
    fi

  done

  if [ -e ${CM_ALL_SCRIPT_PATH}/update_lib.sh ]; then
     echo "calling ${CM_ALL_SCRIPT_PATH}/update_lib.sh $MACHINE $compiler"
     source ${CM_ALL_SCRIPT_PATH}/update_lib.sh $MACHINE $compiler
  fi

  return 0
}

# single_build_and_test <COMPILER> <BUILD> <BUILD_TYPE>
single_build_and_test() {
  # Use sane var names.
  local compiler=$1; local build=$2; local build_type=$3;

  # Set up env.
  local compiler_modules_list=$(get_compiler_modules $compiler)
  local BUILD_AND_TEST_DIR=$ROOT_DIR/$compiler/"${build}-$build_type"
  mkdir -p $BUILD_AND_TEST_DIR
  cd $BUILD_AND_TEST_DIR

  local kokkos_variants=$(get_kokkos_variants $compiler)
  local kernels_variants=$(get_kernels_variants $compiler)
  wrapper_spec=""
  if [[ $kokkos_variants == *"wrapper"* ]]; then
    #Don't build nvcc_wrapper to depend on MPI with Spack
    wrapper_spec="^kokkos-nvcc-wrapper ~mpi"
  fi
  local compiler_variant=$(get_compiler_variant $compiler)
  if [[ $compiler_variant == *"cuda"* ]]; then
    #even though using nvcc, I need to tell spack
    #to use a particular underlying host compiler
    compiler_variant=$SPACK_CUDA_HOST_COMPILER
  fi
  if [ ! -z "$KOKKOSKERNELS_SCALARS" ]; then
    kernels_variants="$kernels_variants scalars=$KOKKOSKERNELS_SCALARS"
  fi
  if [ ! -z "$KOKKOSKERNELS_LAYOUTS" ]; then
    kernels_variants="$kernels_variants layouts=$KOKKOSKERNELS_LAYOUTS"
  fi
  if [ ! -z "$KOKKOSKERNELS_OFFSETS" ]; then
    kernels_variants="$kernels_variants offsets=$KOKKOSKERNELS_OFFSETS"
  fi
  if [ ! -z "$KOKKOSKERNELS_ORDINALS" ]; then
    kernels_variants="$kernels_variants ordinals=$KOKKOSKERNELS_ORDINALS"
  fi
  if [ ! -z "$KOKKOSKERNELS_SPACES" ]; then
      kernels_variants="$kernels_variants spaces=$KOKKOSKERNELS_SPACES"
      KOKKOSKERNELS_SPACES="--with-spaces=$KOKKOSKERNELS_SPACES"
  fi


  echo "  #   Load modules:" &> reload_modules.sh
  if [[ ! -z "$MODULE_ENVIRONMENT" ]]
  then
    echo "        $MODULE_ENVIRONMENT" &>> reload_modules.sh
  fi
  echo "        module purge" &>> reload_modules.sh
  echo "        module load $compiler_modules_list" &>> reload_modules.sh
  echo "        export OMP_NUM_THREADS=$omp_num_threads" &>> reload_modules.sh
  echo "        export OMP_PROC_BIND=$omp_proc_bind" &>> reload_modules.sh
  echo "        export OMP_PLACES=$omp_places" &>> reload_modules.sh
  echo "        export KOKKOS_NUM_THREADS=8" &>> reload_modules.sh
  echo "" &>> reload_modules.sh
  chmod +x reload_modules.sh

  local desc=$(echo "${compiler}-${build}-${build_type}" | sed 's:/:-:g')
  setup_env $compiler >& ${desc}.configure.log || { report_and_log_test_result 1 ${desc} configure && return 0; }

  # Set up flags.
  local compiler_warning_flags=$(get_compiler_warning_flags $compiler)
  local compiler_exe=$(get_compiler_exe_name $compiler)

  if [[ "$build_type" = hwloc* ]]; then
    local extra_args="$extra_args --with-hwloc=$(dirname $(dirname $(which hwloc-info)))"
  fi

  if [[ "$OPT_FLAG" = "" ]]; then
    OPT_FLAG="-O3"
  fi

  if [[ "$build_type" = *debug* ]]; then
    local extra_args="$extra_args --debug"
    local cxxflags="-g $compiler_warning_flags"
  else
    local cxxflags="$OPT_FLAG $compiler_warning_flags"
  fi

  local cxxflags="${cxxflags} ${CXX_FLAGS_EXTRA}"
  local ldflags="${LD_FLAGS_EXTRA}"

  local cxx_standard="${CXX_STANDARD}"

  if [ "${NO_DEFAULT_ETI}" = "True" ]; then
    local extra_args="$extra_args --no-default-eti"
  fi

  if [ "${ENABLE_TEST_ETI_ONLY}" = "False" ]; then
    local extra_args="$extra_args --disable-test-eti-only"
  fi

  if [ "${ENABLE_PERFTESTS}" = "False" ]; then
    local extra_args="$extra_args --disable-perftests"
  fi


  echo "  Starting job $desc"

  local comment="no_comment"

  # Keep variable wrapped in single quotes so complex<scalar> passed correctly to generate_makefile
  # This is likely unnecessary with current argument naming scheme
  local kk_scalars=\'${KOKKOSKERNELS_SCALARS}\'

  if [ "$TEST_SCRIPT" = "True" ]; then
    local rand=$[ 1 + $[ RANDOM % 10 ]]
    sleep $rand

    if [ $rand -gt 5 ]; then
      run_cmd ls fake_problem >& ${desc}.configure.log || { report_and_log_test_result 1 $desc configure && return 0; }
    fi
  elif [ "$TEST_SPACK" = "True" ]; then
    spack compiler find
    if [ ! -z "$SPACK_DEV_BUILD_KOKKOS" ]; then
      run_cmd spack dev-build -d $SPACK_DEV_BUILD_KOKKOS kokkos@develop $kokkos_variants $wrapper_spec $compiler_variant >& ${desc}.spack.log || { report_and_log_test_result 1 ${desc} spack && return 0; }
    fi

    if [ -z "$KOKKOSKERNELS_BRANCH" ]; then
      run_cmd spack dev-build -d $KOKKOSKERNELS_PATH kokkos-kernels@develop $kernels_variants ^kokkos@develop $kokkos_variants $wrapper_spec $compiler_variant >& ${desc}.spack.log || { report_and_log_test_result 1 ${desc} spack && return 0; }
    else
      run_cmd spack install kokkos-kernels@$KOKKOSKERNELS_BRANCH $kernels_variants ^kokkos@develop $kokkos_variants $wrapper_spec $compiler_variant  >& ${desc}.spack.log|| { report_and_log_test_result 1 ${desc} spack && return 0; }
    fi
  else
    LOCAL_KOKKOS_DEVICES=${build//_/,}
    if [[ "$LOCAL_KOKKOS_DEVICES" = *Cuda* ]]; then
       CUDA_ENABLE_CMD="--with-cuda=$CUDA_ROOT"
    fi
    if [[ "$LOCAL_KOKKOS_DEVICES" = *Hip* ]]; then
       echo "Hip IS THE KOKKOS DEVICE"
       HIP_ENABLE_CMD="--with-hip"
    fi
    local arch_code=$(echo $ARCH_FLAG | cut -d "=" -f 2)
    local tpl_list_print=$(echo $KOKKOSKERNELS_ENABLE_TPL_CMD | cut -d "=" -f2-)
    echo "kokkos devices: ${LOCAL_KOKKOS_DEVICES}"
    echo "kokkos arch: ${arch_code}"
    echo "kokkos options: ${KOKKOS_OPTIONS}"
    echo "kokkos cuda options: ${KOKKOS_CUDA_OPTIONS}"
    echo "kokkos cxxflags: ${cxxflags}"
    echo "extra_args: ${extra_args}"
    echo "kokkoskernels scalars: ${kk_scalars}"
    echo "kokkoskernels ordinals: ${KOKKOSKERNELS_ORDINALS}"
    echo "kokkoskernels offsets: ${KOKKOSKERNELS_OFFSETS}"
    echo "kokkoskernels layouts: ${KOKKOSKERNELS_LAYOUTS}"
    echo "kokkoskernels tpls list: ${tpl_list_print}"

    # KOKKOS_OPTIONS and KOKKOS_CUDA_OPTIONS are exported and detected by kokkos' generate_makefile.sh during install of kokkos; we pass them to the reproducer script instructions
    echo "  #   Use generate_makefile line below to call cmake which generates makefile for this build:" &> call_generate_makefile.sh
    echo "        ${KOKKOSKERNELS_PATH}/cm_generate_makefile.bash --with-devices=$LOCAL_KOKKOS_DEVICES $ARCH_FLAG --compiler=$(which $compiler_exe) --cxxflags=\"$cxxflags\" --cxxstandard=\"$cxx_standard\" --ldflags=\"$ldflags\" $CUDA_ENABLE_CMD $HIP_ENABLE_CMD --kokkos-path=${KOKKOS_PATH} --kokkoskernels-path=${KOKKOSKERNELS_PATH} --with-scalars=$kk_scalars --with-ordinals=${KOKKOSKERNELS_ORDINALS} --with-offsets=${KOKKOSKERNELS_OFFSETS} --with-layouts=${KOKKOSKERNELS_LAYOUTS} ${KOKKOSKERNELS_ENABLE_TPL_CMD} ${KOKKOSKERNELS_TPL_PATH_CMD} ${KOKKOSKERNELS_TPL_LIBS_CMD} ${KOKKOSKERNELS_EXTRA_LINKER_FLAGS_CMD} --with-options=${KOKKOS_OPTIONS} --with-cuda-options=${KOKKOS_CUDA_OPTIONS} ${KOKKOS_BOUNDS_CHECK} ${KOKKOSKERNELS_SPACES} --no-examples ${KOKKOS_DEPRECATED_CODE} --cmake-flags=${PASSTHRU_CMAKE_FLAGS} --kokkos-cmake-flags=${KOKKOS_PASSTHRU_CMAKE_FLAGS} $extra_args" &>> call_generate_makefile.sh
    chmod +x call_generate_makefile.sh

    # script command with generic path for faster copy/paste of reproducer into issues
    echo "  #     \$KOKKOSKERNELS_PATH/cm_generate_makefile.bash --with-devices=$LOCAL_KOKKOS_DEVICES $ARCH_FLAG --compiler=$(which $compiler_exe) --cxxflags=\"$cxxflags\" --cxxstandard=\"$cxx_standard\" --ldflags=\"$ldflags\" $CUDA_ENABLE_CMD $HIP_ENABLE_CMD --kokkos-path=\$KOKKOS_PATH --kokkoskernels-path=\$KOKKOSKERNELS_PATH --with-scalars=$kk_scalars --with-ordinals=${KOKKOSKERNELS_ORDINALS} --with-offsets=${KOKKOSKERNELS_OFFSETS} --with-layouts=${KOKKOSKERNELS_LAYOUTS} ${KOKKOSKERNELS_ENABLE_TPL_CMD} ${KOKKOSKERNELS_TPL_PATH_CMD} ${KOKKOSKERNELS_TPL_LIBS_CMD} ${KOKKOSKERNELS_EXTRA_LINKER_FLAGS_CMD} --with-options=${KOKKOS_OPTIONS} --with-cuda-options=${KOKKOS_CUDA_OPTIONS} ${KOKKOS_BOUNDS_CHECK} ${KOKKOSKERNELS_SPACES} --no-examples ${KOKKOS_DEPRECATED_CODE} --cmake-flags=${PASSTHRU_CMAKE_FLAGS} --kokkos-cmake-flags=${KOKKOS_PASSTHRU_CMAKE_FLAGS} $extra_args" &> call_generate_makefile_genericpath.sh

    run_cmd ${KOKKOSKERNELS_PATH}/cm_generate_makefile.bash --with-devices=$LOCAL_KOKKOS_DEVICES $ARCH_FLAG --compiler=$(which $compiler_exe) --cxxflags=\"$cxxflags\" --cxxstandard=\"$cxx_standard\" --ldflags=\"$ldflags\" $CUDA_ENABLE_CMD $HIP_ENABLE_CMD --kokkos-path=${KOKKOS_PATH} --kokkoskernels-path=${KOKKOSKERNELS_PATH} --with-scalars=$kk_scalars --with-ordinals=${KOKKOSKERNELS_ORDINALS} --with-offsets=${KOKKOSKERNELS_OFFSETS} --with-layouts=${KOKKOSKERNELS_LAYOUTS} ${KOKKOSKERNELS_ENABLE_TPL_CMD} ${KOKKOSKERNELS_TPL_PATH_CMD} ${KOKKOSKERNELS_TPL_LIBS_CMD} ${KOKKOSKERNELS_EXTRA_LINKER_FLAGS_CMD} ${KOKKOS_BOUNDS_CHECK} ${KOKKOSKERNELS_SPACES} --no-examples ${KOKKOS_DEPRECATED_CODE} --cmake-flags=${PASSTHRU_CMAKE_FLAGS} --kokkos-cmake-flags=${KOKKOS_PASSTHRU_CMAKE_FLAGS} $extra_args &>> ${desc}.configure.log || { report_and_log_test_result 1 ${desc} configure && return 0; }

    local -i build_start_time=$(date +%s)
    run_cmd make -j $MAKE_PAR_LEVEL all >& ${desc}.build.log || { report_and_log_test_result 1 ${desc} build && return 0; }
    local -i build_end_time=$(date +%s)
    comment="build_time=$(($build_end_time-$build_start_time))"

    if [[ "$BUILD_ONLY" == False ]]; then
      run_cmd ctest --timeout ${CTESTTIMEOUT} -V  --output-on-failure >& ${desc}.test.log || { report_and_log_test_result 1 ${desc} test && return 0; }
      local -i run_end_time=$(date +%s)
      comment="$comment run_time=$(($run_end_time-$build_end_time))"
    fi
  fi

  report_and_log_test_result 0 $desc "$comment"

  return 0
}

# wait_for_jobs <NUM-JOBS>
wait_for_jobs() {
  local -i max_jobs=$1
  local -i num_active_jobs=$(jobs | wc -l)
  while [ $num_active_jobs -ge $max_jobs ]
  do
    sleep 1
    num_active_jobs=$(jobs | wc -l)
    jobs >& /dev/null
  done
}

# run_in_background <COMPILER> <BUILD> <BUILD_TYPE>
run_in_background() {
  local compiler=$1

  local -i num_jobs=$NUM_JOBS_TO_RUN_IN_PARALLEL
  # Don't override command line input.
  # if [[ "$BUILD_ONLY" == True ]]; then
  #   num_jobs=8
  # else
    if [[ "$compiler" == cuda* ]]; then
      num_jobs=1
    fi
    if [[ "$compiler" == rocm* ]]; then
      num_jobs=1
    fi
    if [[ "$compiler" == clang ]]; then
      num_jobs=1
    fi
  # fi
  wait_for_jobs $num_jobs

  single_build_and_test $* &
}

# build_and_test_all <COMPILER>
build_and_test_all() {
  # Get compiler data.
  local compiler=$1
  if [ -z "$CUSTOM_BUILD_LIST" ]; then
    local compiler_build_list=$(get_compiler_build_list $compiler)
  else
    local compiler_build_list=$(echo "$CUSTOM_BUILD_LIST" | tr , ' ')
  fi

  # Do builds.
  local build
  for build in $compiler_build_list
  do
    run_in_background $compiler $build $BUILD_TYPE

    # If not cuda, do a hwloc test too.
    if [[ "$compiler" != cuda* && "$SKIP_HWLOC" == False ]]; then
      run_in_background $compiler $build "hwloc-$BUILD_TYPE"
    fi
  done

  return 0
}

get_test_root_dir() {
  local existing_results=$(find . -maxdepth 1 -name "$RESULT_ROOT_PREFIX*" | sort)
  local -i num_existing_results=$(echo $existing_results | tr ' ' '\n' | wc -l)
  local -i num_to_delete=${num_existing_results}-${NUM_RESULTS_TO_KEEP}

  if [ $num_to_delete -gt 0 ]; then
    /bin/rm -rf $(echo $existing_results | tr ' ' '\n' | head -n $num_to_delete)
  fi

  echo $(pwd)/${RESULT_ROOT_PREFIX}_$(date +"%Y-%m-%d_%H.%M.%S")
}

wait_summarize_and_exit() {
  wait_for_jobs 1

  echo "#######################################################"
  echo "PASSED TESTS"
  echo "#######################################################"

  local passed_test
  for passed_test in $(\ls -1 $PASSED_DIR | sort)
  do
    echo $passed_test $(cat $PASSED_DIR/$passed_test)
  done

  local -i rv=0
  if [ "$(ls -A $FAILED_DIR)" ]; then
    echo "#######################################################"
    echo "FAILED TESTS"
    echo "#######################################################"

    local failed_test
    for failed_test in $(\ls -1 $FAILED_DIR | sort)
    do
      echo $failed_test "("$(cat $FAILED_DIR/$failed_test)" failed)"
      rv=$rv+1

      local str=$failed_test
      # Note: all relevant info in str to assemble the build directory path 
      # is separated by dashes; however the compiler name may include dashes as well
      # the final two pieces of str always the version and build-type (as set in BUILD_AND_TEST_DIR)
      # leaving the compiler name as the remaining fields preceding version
      local getdashes="${str//[^-]}"
      local numdashes=${#getdashes}
      local lbuild=$(echo "$str" | cut -d- -f${numdashes}-)
      local vers=$(echo "$str" | cut -d- -f$((numdashes-1)))
      local comp=$(echo "$str" | cut -d- -f-$((numdashes-2)))
      # Generate reproducer instructions
      #local filename=reproducer_instructions-$comp-$vers-$lbuild
      local faildir=$ROOT_DIR/$comp/$vers/$lbuild
      # Output reproducer instructions
      if [ $TEST_SPACK = "False" ]; then
        echo "#######################################################"
        echo "  # Reproducer instructions:"
        cat $faildir/reload_modules.sh
        cat $faildir/call_generate_makefile_genericpath.sh
        echo ""
        echo "  #  To reload modules, reconfigure, rebuild, and retest directly from this failing build do the following:"
        echo "      # Move to the build directory"
        echo "        cd $faildir"
        echo "      # To reload modules"
        echo "        source ./reload_modules.sh"
        echo "      # To reconfigure"
        echo "        ./call_generate_makefile.sh"
        echo "      # To rebuild"
        echo "        make -j"
        echo "      # To retest"
        echo "        ctest -V"
        echo "#######################################################"
      fi
    done
  fi

  exit $rv
}

#
# Main.
#

CM_ALL_SCRIPT=$0
CM_ALL_SCRIPT_PATH=$(cd `dirname $CM_ALL_SCRIPT` && pwd)

ROOT_DIR=$(get_test_root_dir)
mkdir -p $ROOT_DIR
cd $ROOT_DIR

PASSED_DIR=$ROOT_DIR/results/passed
FAILED_DIR=$ROOT_DIR/results/failed
mkdir -p $PASSED_DIR
mkdir -p $FAILED_DIR

echo "Going to test compilers: " $COMPILERS_TO_TEST
for COMPILER in $COMPILERS_TO_TEST; do
  echo "Testing compiler $COMPILER"
  build_and_test_all $COMPILER
done

wait_summarize_and_exit
