SlurmCI bac79e6b787f70f7de32d4ea15a23033e124176c

_summary.md

Commit: bac79e6b787f70f7de32d4ea15a23033e124176c

command	ntasks	jobid	status	elapsed
scripts/test-cpu-init.sh		10192324	✅	00:05:56
scripts/test-cpu-tests.sh		10192325	✅	00:00:01
scripts/test-cpu.sh test/Numerics/DGMethods/advection_diffusion/advection_diffusion_model_1dimex_bgmres.jl	--ntasks=1	10192326	✅	00:06:55
scripts/test-cpu.sh test/Numerics/DGMethods/advection_diffusion/pseudo1D_advection_diffusion.jl	--ntasks=3	10192327	✅	00:07:00
scripts/test-cpu.sh test/Numerics/DGMethods/advection_diffusion/pseudo1D_advection_diffusion_1dimex.jl	--ntasks=3	10192328	✅	00:06:17
scripts/test-cpu.sh test/Numerics/DGMethods/advection_diffusion/pseudo1D_advection_diffusion_mrigark_implicit.jl	--ntasks=3	10192329	✅	00:05:37
scripts/test-cpu.sh test/Numerics/DGMethods/advection_diffusion/pseudo1D_heat_eqn.jl	--ntasks=3	10192330	✅	00:04:43
scripts/test-cpu.sh test/Numerics/DGMethods/advection_diffusion/periodic_3D_hyperdiffusion.jl	--ntasks=3	10192331	✅	00:03:16
scripts/test-cpu.sh test/Numerics/DGMethods/compressible_Navier_Stokes/ref_state.jl	--ntasks=1	10192332	✅	00:01:34
scripts/test-cpu.sh test/Numerics/Mesh/mpi_connect_1d.jl	--ntasks=5	10192333	✅	00:00:49
scripts/test-cpu.sh test/Numerics/Mesh/mpi_connect_sphere.jl	--ntasks=5	10192334	✅	00:01:11
scripts/test-cpu.sh test/Numerics/Mesh/mpi_getpartition.jl	--ntasks=5	10192335	✅	00:00:40
scripts/test-cpu.sh test/Numerics/Mesh/mpi_sortcolumns.jl	--ntasks=4	10192336	✅	00:00:37
scripts/test-cpu.sh test/Numerics/ODESolvers/ode_tests_convergence.jl	--ntasks=1	10192337	✅	00:13:34
scripts/test-cpu.sh test/Arrays/varsindex.jl	--ntasks=1	10192338	✅	00:00:43
scripts/test-cpu.sh test/Diagnostics/diagnostic_fields_test.jl	--ntasks=3	10192339	✅	00:02:27
scripts/test-cpu.sh test/Numerics/DGMethods/vars_test.jl	--ntasks=1	10192340	✅	00:01:38
scripts/test-cpu.sh test/Numerics/DGMethods/remainder_model.jl	--ntasks=3	10192341	✅	00:03:45
scripts/test-cpu.sh test/Numerics/DGMethods/Euler/isentropicvortex.jl	--ntasks=3	10192342	✅	00:03:58
scripts/test-cpu.sh test/Numerics/DGMethods/Euler/isentropicvortex_imex.jl	--ntasks=3	10192343	✅	00:02:56
scripts/test-cpu.sh test/Numerics/DGMethods/Euler/isentropicvortex_multirate.jl	--ntasks=3	10192344	✅	00:02:49
scripts/test-cpu.sh test/Numerics/DGMethods/Euler/isentropicvortex_mrigark.jl	--ntasks=3	10192345	✅	00:02:37
scripts/test-cpu.sh test/Numerics/DGMethods/Euler/isentropicvortex_mrigark_implicit.jl	--ntasks=3	10192346	✅	00:03:29
scripts/test-cpu.sh test/Numerics/DGMethods/Euler/acousticwave_1d_imex.jl	--ntasks=3	10192347	✅	00:06:27
scripts/test-cpu.sh test/Numerics/DGMethods/Euler/acousticwave_mrigark.jl	--ntasks=3	10192348	✅	00:27:50
scripts/test-cpu.sh test/Numerics/DGMethods/compressible_Navier_Stokes/mms_bc_atmos.jl	--ntasks=3	10192349	✅	00:03:44
scripts/test-cpu.sh test/Numerics/DGMethods/compressible_Navier_Stokes/mms_bc_dgmodel.jl	--ntasks=3	10192350	✅	00:03:14
scripts/test-cpu.sh test/Numerics/DGMethods/compressible_Navier_Stokes/density_current_model.jl	--ntasks=3	10192351	✅	00:05:28
scripts/test-cpu.sh test/Numerics/DGMethods/advection_diffusion/direction_splitting_advection_diffusion.jl --fix-rng-seed	--ntasks=3	10192352	✅	00:13:01
scripts/test-cpu.sh test/Numerics/DGMethods/conservation/sphere.jl --fix-rng-seed	--ntasks=3	10192353	✅	00:02:23
scripts/test-cpu.sh test/Numerics/DGMethods/advection_diffusion/advection_sphere.jl	--ntasks=2	10192354	✅	00:02:06
scripts/test-cpu.sh test/Driver/gcm_driver_test.jl	--ntasks=1	10192355	✅	00:03:45
scripts/test-cpu.sh test/Numerics/SystemSolvers/poisson.jl	--ntasks=2	10192356	✅	00:02:22
scripts/test-cpu.sh test/Numerics/SystemSolvers/columnwiselu.jl	--ntasks=1	10192357	✅	00:01:06
scripts/test-cpu.sh test/Numerics/SystemSolvers/bandedsystem.jl	--ntasks=3	10192358	✅	00:03:36
scripts/test-cpu.sh test/Numerics/Mesh/interpolation.jl	--ntasks=3	10192359	✅	00:03:05
scripts/test-cpu.sh test/Ocean/ShallowWater/GyreDriver.jl	--ntasks=1	10192360	✅	00:12:00
scripts/test-cpu.sh test/Ocean/ShallowWater/2D_hydrostatic_spindown.jl	--ntasks=1	10192361	✅	00:01:50
scripts/test-cpu.sh test/Ocean/HydrostaticBoussinesq/3D_hydrostatic_spindown.jl	--ntasks=1	10192362	✅	00:06:04
scripts/test-cpu.sh test/Ocean/SplitExplicit/hydrostatic_spindown.jl	--ntasks=1	10192363	✅	00:02:42
scripts/test-cpu.sh tutorials/Numerics/DGMethods/nonnegative.jl	--ntasks=3	10192364	✅	00:02:12
scripts/test-cpu.sh test/Atmos/Parameterizations/Microphysics/KM_saturation_adjustment.jl	--ntasks=3	10192365	✅	00:02:44
scripts/test-cpu.sh test/Atmos/Parameterizations/Microphysics/KM_warm_rain.jl	--ntasks=3	10192366	✅	00:12:48
scripts/test-cpu.sh test/Atmos/Parameterizations/Microphysics/KM_ice.jl	--ntasks=3	10192367	✅	00:03:35
scripts/test-gpu-init.sh		10192368	✅	00:05:58
scripts/test-gpu-tests.sh		10192369	✅	00:17:37
scripts/test-gpu.sh test/Numerics/DGMethods/advection_diffusion/pseudo1D_advection_diffusion.jl --integration-testing	--ntasks=3	10192370	✅	00:11:05
scripts/test-gpu.sh test/Numerics/DGMethods/advection_diffusion/pseudo1D_advection_diffusion_1dimex.jl	--ntasks=3	10192371	✅	00:08:44
scripts/test-gpu.sh test/Numerics/DGMethods/advection_diffusion/pseudo1D_advection_diffusion_mrigark_implicit.jl	--ntasks=3	10192372	✅	00:08:35
scripts/test-gpu.sh test/Numerics/DGMethods/advection_diffusion/pseudo1D_heat_eqn.jl --integration-testing	--ntasks=3	10192373	✅	00:07:14
scripts/test-gpu.sh test/Numerics/DGMethods/advection_diffusion/periodic_3D_hyperdiffusion.jl --integration-testing	--ntasks=3	10192374	✅	00:04:14
scripts/test-gpu.sh tutorials/Atmos/dry_rayleigh_benard.jl --fix-rng-seed	--ntasks=1	10192375	✅	00:11:34
scripts/test-gpu.sh tutorials/Atmos/risingbubble.jl	--ntasks=3	10192376	✅	00:08:48
scripts/test-gpu.sh experiments/AtmosGCM/heldsuarez.jl --diagnostics=default --fix-rng-seed	--ntasks=1	10192377	✅	00:11:51
scripts/test-gpu.sh experiments/AtmosLES/surfacebubble.jl	--ntasks=3	10192378	✅	00:16:30
scripts/test-gpu.sh experiments/AtmosLES/dycoms.jl --fix-rng-seed	--ntasks=3	10192379	✅	00:06:23
scripts/test-gpu.sh experiments/AtmosLES/bomex.jl --diagnostics=default --fix-rng-seed	--ntasks=1	10192380	✅	00:17:01
scripts/test-gpu.sh experiments/OceanBoxGCM/homogeneous_box.jl	--ntasks=3	10192381	❌	00:03:26
scripts/test-gpu.sh experiments/OceanBoxGCM/ocean_gyre.jl	--ntasks=3	10192382	❌	00:02:48
scripts/test-gpu.sh test/Arrays/varsindex.jl	--ntasks=1	10192383	✅	00:00:43
scripts/test-gpu.sh test/Diagnostics/diagnostic_fields_test.jl	--ntasks=3	10192384	✅	00:03:49
scripts/test-gpu.sh test/Numerics/DGMethods/vars_test.jl	--ntasks=1	10192385	✅	00:01:54
scripts/test-gpu.sh test/Numerics/DGMethods/remainder_model.jl	--ntasks=3	10192386	✅	00:06:48
scripts/test-gpu.sh test/Numerics/DGMethods/Euler/isentropicvortex.jl	--ntasks=3	10192387	✅	00:06:01
scripts/test-gpu.sh test/Numerics/DGMethods/Euler/isentropicvortex_imex.jl	--ntasks=3	10192388	✅	00:04:43
scripts/test-gpu.sh test/Numerics/DGMethods/Euler/isentropicvortex_multirate.jl	--ntasks=3	10192389	✅	00:05:05
scripts/test-gpu.sh test/Numerics/DGMethods/Euler/isentropicvortex_mrigark.jl	--ntasks=3	10192390	✅	00:03:20
scripts/test-gpu.sh test/Numerics/DGMethods/Euler/isentropicvortex_mrigark_implicit.jl	--ntasks=3	10192391	✅	00:07:55
scripts/test-gpu.sh test/Numerics/DGMethods/Euler/acousticwave_1d_imex.jl	--ntasks=3	10192392	✅	00:07:29
scripts/test-gpu.sh test/Numerics/DGMethods/Euler/acousticwave_mrigark.jl	--ntasks=3	10192393	✅	00:05:49
scripts/test-gpu.sh test/Numerics/DGMethods/compressible_Navier_Stokes/mms_bc_atmos.jl	--ntasks=3	10192394	✅	00:04:54
scripts/test-gpu.sh test/Numerics/DGMethods/compressible_Navier_Stokes/mms_bc_dgmodel.jl	--ntasks=3	10192395	✅	00:04:06
scripts/test-gpu.sh test/Numerics/DGMethods/compressible_Navier_Stokes/density_current_model.jl	--ntasks=3	10192396	✅	00:05:27
scripts/test-gpu.sh test/Numerics/DGMethods/advection_diffusion/direction_splitting_advection_diffusion.jl --fix-rng-seed	--ntasks=3	10192397	✅	00:20:26
scripts/test-gpu.sh test/Numerics/DGMethods/conservation/sphere.jl --fix-rng-seed	--ntasks=3	10192398	✅	00:02:56
scripts/test-gpu.sh test/Numerics/DGMethods/advection_diffusion/advection_sphere.jl	--ntasks=2	10192399	✅	00:02:56
scripts/test-gpu.sh test/Driver/gcm_driver_test.jl	--ntasks=1	10192400	✅	00:03:01
scripts/test-gpu.sh test/Numerics/SystemSolvers/poisson.jl	--ntasks=2	10192401	✅	00:03:03
scripts/test-gpu.sh test/Numerics/SystemSolvers/columnwiselu.jl	--ntasks=1	10192402	✅	00:01:11
scripts/test-gpu.sh test/Numerics/SystemSolvers/bandedsystem.jl	--ntasks=3	10192403	✅	00:04:24
scripts/test-gpu.sh test/Numerics/Mesh/interpolation.jl	--ntasks=3	10192404	✅	00:03:32
scripts/test-gpu.sh test/Ocean/ShallowWater/GyreDriver.jl	--ntasks=1	10192405	✅	00:19:21
scripts/test-gpu.sh test/Ocean/ShallowWater/2D_hydrostatic_spindown.jl	--ntasks=1	10192406	✅	00:02:53
scripts/test-gpu.sh test/Ocean/HydrostaticBoussinesq/3D_hydrostatic_spindown.jl	--ntasks=1	10192407	✅	00:03:55
scripts/test-gpu.sh test/Ocean/SplitExplicit/hydrostatic_spindown.jl	--ntasks=1	10192408	✅	00:03:33
scripts/test-gpu.sh tutorials/Numerics/DGMethods/nonnegative.jl	--ntasks=3	10192409	✅	00:02:40
scripts/test-gpu.sh test/Atmos/Parameterizations/Microphysics/KM_saturation_adjustment.jl	--ntasks=3	10192410	✅	00:03:07
scripts/test-gpu.sh test/Atmos/Parameterizations/Microphysics/KM_warm_rain.jl	--ntasks=3	10192411	✅	00:03:45
scripts/test-gpu.sh test/Atmos/Parameterizations/Microphysics/KM_ice.jl	--ntasks=3	10192412	✅	00:04:30

out_10192324

+ hostname
hpc-23-28.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
++ pwd
+ export CURL_HOME=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
+ CURL_HOME=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
+ printf -- '--silent\n--show-error\n'
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ julia --color=no --project -e 'using Pkg; Pkg.instantiate(); Pkg.build(;verbose=true)'
[?25l    Fetching: [>                                        ]  0.0 %
    Fetching: [===================>                     ]  45.2 %
    Resolving Deltas: [==================================>      ]  84.7 %
[?25h[ Info: using system MPI
┌ Info: Using implementation
│   libmpi = "libmpi"
└   mpiexec_cmd = `mpiexec`
┌ Info: MPI implementation detected
│   impl = OpenMPI::MPIImpl = 2
│   version = v"4.0.3"
└   abi = "OpenMPI"
africa
antarctica
asia
australasia
europe
northamerica
southamerica
etcetera
backward
pacificnew
[ Info: Extracting 2020a tzdata archive
[ Info: Converting tz source files into TimeZone data
[ Info: Successfully built TimeZones
    Cloning default registries into `/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu`
    Cloning registry from "https://github.com/JuliaRegistries/General.git"
      Added registry `General` to `/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu/registries/General`
  Installed XML2_jll ───────────────────── v2.9.9+5
  Installed OpenMPI_jll ────────────────── v4.0.2+2
  Installed OpenSpecFun_jll ────────────── v0.5.3+3
  Installed CondaBinDeps ───────────────── v0.2.0
  Installed GaussQuadrature ────────────── v0.5.4
  Installed LightXML ───────────────────── v0.9.0
  Installed DataAPI ────────────────────── v1.3.0
  Installed HTTP ───────────────────────── v0.8.15
  Installed Adapt ──────────────────────── v2.0.2
  Installed CLIMAParameters ────────────── v0.1.6
  Installed ArgParse ───────────────────── v1.1.0
  Installed StatsBase ──────────────────── v0.33.0
  Installed URIParser ──────────────────── v0.4.1
  Installed DiffResults ────────────────── v1.0.2
  Installed OpenBLAS_jll ───────────────── v0.3.9+4
  Installed ForwardDiff ────────────────── v0.10.10
  Installed CodecZlib ──────────────────── v0.7.0
  Installed PDMats ─────────────────────── v0.9.12
  Installed Combinatorics ──────────────── v1.0.2
  Installed MicrosoftMPI_jll ───────────── v10.1.2+3
  Installed PackageCompiler ────────────── v1.1.1
  Installed NNPACK_jll ─────────────────── v2018.6.22+0
  Installed VersionParsing ─────────────── v1.2.0
  Installed Rmath_jll ──────────────────── v0.2.2+1
  Installed Distributions ──────────────── v0.23.4
  Installed Mocking ────────────────────── v0.7.1
  Installed DataStructures ─────────────── v0.17.18
  Installed GenericSchur ───────────────── v0.4.0
  Installed KernelAbstractions ─────────── v0.3.0
  Installed Arpack ─────────────────────── v0.4.0
  Installed StaticNumbers ──────────────── v0.3.3
  Installed BinDeps ────────────────────── v1.0.1
  Installed Coverage ───────────────────── v1.1.1
  Installed BinaryProvider ─────────────── v0.5.10
  Installed MatrixFactorizations ───────── v0.4.1
  Installed GPUCompiler ────────────────── v0.4.0
  Installed Intervals ──────────────────── v1.2.0
  Installed IniFile ────────────────────── v0.5.0
  Installed Rmath ──────────────────────── v0.6.1
  Installed Literate ───────────────────── v2.5.0
  Installed EzXML ──────────────────────── v1.1.0
  Installed WriteVTK ───────────────────── v1.6.0
  Installed Arpack_jll ─────────────────── v3.5.0+3
  Installed MbedTLS ────────────────────── v1.0.2
  Installed TimerOutputs ───────────────── v0.5.6
  Installed SortingAlgorithms ──────────── v0.3.1
  Installed ArrayLayouts ───────────────── v0.3.5
  Installed DoubleFloats ───────────────── v1.1.12
  Installed MPICH_jll ──────────────────── v3.3.2+10
  Installed QuadGK ─────────────────────── v2.3.1
  Installed LazyArrays ─────────────────── v0.16.13
  Installed MacroTools ─────────────────── v0.5.5
  Installed Polynomials ────────────────── v1.1.1
  Installed Libiconv_jll ───────────────── v1.16.0+4
  Installed OrderedCollections ─────────── v1.2.0
  Installed MPI ────────────────────────── v0.15.0
  Installed Reexport ───────────────────── v0.2.0
  Installed Cassette ───────────────────── v0.3.3
  Installed FillArrays ─────────────────── v0.8.11
  Installed ExprTools ──────────────────── v0.1.1
  Installed CEnum ──────────────────────── v0.4.1
  Installed StaticArrays ───────────────── v0.12.3
  Installed StatsFuns ──────────────────── v0.9.5
  Installed TranscodingStreams ─────────── v0.9.5
  Installed GenericSVD ─────────────────── v0.3.0
  Installed NNlib ──────────────────────── v0.7.0
  Installed CompilerSupportLibraries_jll ─ v0.3.3+0
  Installed LLVM ───────────────────────── v1.6.0
  Installed NaNMath ────────────────────── v0.3.3
  Installed DiffRules ──────────────────── v1.0.1
  Installed Parsers ────────────────────── v1.0.5
  Installed Missings ───────────────────── v0.4.3
  Installed Dierckx ────────────────────── v0.4.2
  Installed Zlib_jll ───────────────────── v1.2.11+12
  Installed TimeZones ──────────────────── v1.2.0
  Installed MbedTLS_jll ────────────────── v2.16.0+2
  Installed AbstractFFTs ───────────────── v0.5.0
  Installed FileIO ─────────────────────── v1.3.0
  Installed SpecialFunctions ───────────── v0.10.3
  Installed TextWrap ───────────────────── v1.0.1
  Installed DocStringExtensions ────────── v0.8.2
  Installed CUDA ───────────────────────── v1.0.2
  Installed RootSolvers ────────────────── v0.2.0
  Installed CFTime ─────────────────────── v0.1.0
  Installed Conda ──────────────────────── v1.4.1
  Installed Formatting ─────────────────── v0.4.1
  Installed CommonSubexpressions ───────── v0.2.0
  Installed Quadmath ───────────────────── v0.5.4
  Installed GPUArrays ──────────────────── v4.0.0
  Installed NCDatasets ─────────────────── v0.10.2
  Installed JSON ───────────────────────── v0.21.0
  Installed Requires ───────────────────── v1.0.1
  Installed CoverageTools ──────────────── v1.1.0
  Installed RecipesBase ────────────────── v1.0.1
  Installed JLD2 ───────────────────────── v0.1.13
   Building MPI ───────→ `/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu/packages/MPI/k7f4E/deps/build.log`
   Building Dierckx ───→ `/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu/packages/Dierckx/ZTanR/deps/build.log`
   Building TimeZones ─→ `/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu/packages/TimeZones/OjMoF/deps/build.log`
   Building Conda ─────→ `/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu/packages/Conda/3rPhK/deps/build.log`
   Building NCDatasets → `/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu/packages/NCDatasets/KjkFI/deps/build.log`
   Building MPI ───────→ `/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu/packages/MPI/k7f4E/deps/build.log`
   Building TimeZones ─→ `/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu/packages/TimeZones/OjMoF/deps/build.log`
   Building Dierckx ───→ `/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu/packages/Dierckx/ZTanR/deps/build.log`
   Building Conda ─────→ `/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu/packages/Conda/3rPhK/deps/build.log`
   Building NCDatasets → `/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu/packages/NCDatasets/KjkFI/deps/build.log`
+ julia --color=no --project -e 'using Pkg; Pkg.precompile()'
Precompiling project...
+ cat Manifest.toml
# This file is machine-generated - editing it directly is not advised

[[AbstractFFTs]]
deps = ["LinearAlgebra"]
git-tree-sha1 = "051c95d6836228d120f5f4b984dd5aba1624f716"
uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c"
version = "0.5.0"

[[Adapt]]
deps = ["LinearAlgebra"]
git-tree-sha1 = "0fac443759fa829ed8066db6cf1077d888bb6573"
uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
version = "2.0.2"

[[ArgParse]]
deps = ["Logging", "TextWrap"]
git-tree-sha1 = "a8fc2e149cd6db276c76faebe197ccd3a92fb9ff"
uuid = "c7e460c6-2fb9-53a9-8c5b-16f535851c63"
version = "1.1.0"

[[Arpack]]
deps = ["Arpack_jll", "Libdl", "LinearAlgebra"]
git-tree-sha1 = "2ff92b71ba1747c5fdd541f8fc87736d82f40ec9"
uuid = "7d9fca2a-8960-54d3-9f78-7d1dccf2cb97"
version = "0.4.0"

[[Arpack_jll]]
deps = ["Libdl", "OpenBLAS_jll", "Pkg"]
git-tree-sha1 = "e214a9b9bd1b4e1b4f15b22c0994862b66af7ff7"
uuid = "68821587-b530-5797-8361-c406ea357684"
version = "3.5.0+3"

[[ArrayLayouts]]
deps = ["FillArrays", "LinearAlgebra"]
git-tree-sha1 = "a3254b3780a3544838ca0b7e23b1e9b06eb71bd8"
uuid = "4c555306-a7a7-4459-81d9-ec55ddd5c99a"
version = "0.3.5"

[[Base64]]
uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

[[BinDeps]]
deps = ["Libdl", "Pkg", "SHA", "URIParser", "Unicode"]
git-tree-sha1 = "46cf2c1668ad07aba5a9d331bdeea994a1f13856"
uuid = "9e28174c-4ba2-5203-b857-d8d62c4213ee"
version = "1.0.1"

[[BinaryProvider]]
deps = ["Libdl", "Logging", "SHA"]
git-tree-sha1 = "ecdec412a9abc8db54c0efc5548c64dfce072058"
uuid = "b99e7846-7c00-51b0-8f62-c81ae34c0232"
version = "0.5.10"

[[CEnum]]
git-tree-sha1 = "215a9aa4a1f23fbd05b92769fdd62559488d70e9"
uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82"
version = "0.4.1"

[[CFTime]]
deps = ["Dates", "Printf"]
git-tree-sha1 = "143ef231a14c2ab1ac7e838c559ee1e45c0d1b57"
uuid = "179af706-886a-5703-950a-314cd64e0468"
version = "0.1.0"

[[CLIMAParameters]]
deps = ["Test"]
git-tree-sha1 = "314f0bdd5817abb2df61cc8d3e857c496d421109"
uuid = "6eacf6c3-8458-43b9-ae03-caf5306d3d53"
version = "0.1.6"

[[CUDA]]
deps = ["AbstractFFTs", "Adapt", "BinaryProvider", "CEnum", "DataStructures", "ExprTools", "GPUArrays", "GPUCompiler", "LLVM", "Libdl", "LinearAlgebra", "Logging", "MacroTools", "NNlib", "Pkg", "Printf", "Random", "Reexport", "Requires", "SparseArrays", "Statistics", "TimerOutputs"]
git-tree-sha1 = "d658b3881a25b317ea7fb128efbd82b5e63396ad"
uuid = "052768ef-5323-5732-b1bb-66c8b64840ba"
version = "1.0.2"

[[Cassette]]
git-tree-sha1 = "ff6f5109371926beb67ec3101be17d2c211e497d"
uuid = "7057c7e9-c182-5462-911a-8362d720325c"
version = "0.3.3"

[[CodecZlib]]
deps = ["TranscodingStreams", "Zlib_jll"]
git-tree-sha1 = "ded953804d019afa9a3f98981d99b33e3db7b6da"
uuid = "944b1d66-785c-5afd-91f1-9de20f533193"
version = "0.7.0"

[[Combinatorics]]
git-tree-sha1 = "08c8b6831dc00bfea825826be0bc8336fc369860"
uuid = "861a8166-3701-5b0c-9a16-15d98fcdc6aa"
version = "1.0.2"

[[CommonSubexpressions]]
deps = ["Test"]
git-tree-sha1 = "efdaf19ab11c7889334ca247ff4c9f7c322817b0"
uuid = "bbf7d656-a473-5ed7-a52c-81e309532950"
version = "0.2.0"

[[CompilerSupportLibraries_jll]]
deps = ["Libdl", "Pkg"]
git-tree-sha1 = "7c4f882c41faa72118841185afc58a2eb00ef612"
uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"
version = "0.3.3+0"

[[Conda]]
deps = ["JSON", "VersionParsing"]
git-tree-sha1 = "7a58bb32ce5d85f8bf7559aa7c2842f9aecf52fc"
uuid = "8f4d0f93-b110-5947-807f-2305c1781a2d"
version = "1.4.1"

[[CondaBinDeps]]
deps = ["BinDeps", "Conda"]
git-tree-sha1 = "25f750df2893991f2c9b18425bfac6f2ce855154"
uuid = "a9693cdc-2bc8-5703-a9cd-1da358117377"
version = "0.2.0"

[[Coverage]]
deps = ["CoverageTools", "HTTP", "JSON", "LibGit2", "MbedTLS"]
git-tree-sha1 = "aad69c0c2d8080427f507c36b289c77fe465c9fe"
uuid = "a2441757-f6aa-5fb2-8edb-039e3f45d037"
version = "1.1.1"

[[CoverageTools]]
deps = ["LibGit2"]
git-tree-sha1 = "328154f6062b0053588295df6bffe22e1436597a"
uuid = "c36e975a-824b-4404-a568-ef97ca766997"
version = "1.1.0"

[[DataAPI]]
git-tree-sha1 = "176e23402d80e7743fc26c19c681bfb11246af32"
uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
version = "1.3.0"

[[DataStructures]]
deps = ["InteractiveUtils", "OrderedCollections"]
git-tree-sha1 = "be680f1ad03c0a03796aa3fda5a2180df7f83b46"
uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
version = "0.17.18"

[[Dates]]
deps = ["Printf"]
uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"

[[DelimitedFiles]]
deps = ["Mmap"]
uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab"

[[Dierckx]]
deps = ["BinaryProvider", "Libdl", "Random", "Test"]
git-tree-sha1 = "956bfe5c4f9b794545f6527424fc0294cc5decbe"
uuid = "39dd38d3-220a-591b-8e3c-4c3a8c710a94"
version = "0.4.2"

[[DiffResults]]
deps = ["StaticArrays"]
git-tree-sha1 = "da24935df8e0c6cf28de340b958f6aac88eaa0cc"
uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5"
version = "1.0.2"

[[DiffRules]]
deps = ["NaNMath", "Random", "SpecialFunctions"]
git-tree-sha1 = "eb0c34204c8410888844ada5359ac8b96292cfd1"
uuid = "b552c78f-8df3-52c6-915a-8e097449b14b"
version = "1.0.1"

[[Distributed]]
deps = ["Random", "Serialization", "Sockets"]
uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"

[[Distributions]]
deps = ["FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SpecialFunctions", "Statistics", "StatsBase", "StatsFuns"]
git-tree-sha1 = "78c4c32a2357a00a0a7d614880f02c2c6e1ec73c"
uuid = "31c24e10-a181-5473-b8eb-7969acd0382f"
version = "0.23.4"

[[DocStringExtensions]]
deps = ["LibGit2", "Markdown", "Pkg", "Test"]
git-tree-sha1 = "c5714d9bcdba66389612dc4c47ed827c64112997"
uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
version = "0.8.2"

[[DoubleFloats]]
deps = ["GenericSVD", "GenericSchur", "LinearAlgebra", "Polynomials", "Printf", "Quadmath", "Random", "Requires", "SpecialFunctions"]
git-tree-sha1 = "a73770c1418f8a7c4c127328e0cd34fdcf0a1d29"
uuid = "497a8b3b-efae-58df-a0af-a86822472b78"
version = "1.1.12"

[[ExprTools]]
git-tree-sha1 = "6f0517056812fd6aa3af23d4b70d5325a2ae4e95"
uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04"
version = "0.1.1"

[[EzXML]]
deps = ["Printf", "XML2_jll"]
git-tree-sha1 = "0fa3b52a04a4e210aeb1626def9c90df3ae65268"
uuid = "8f5d6c58-4d21-5cfd-889c-e3ad7ee6a615"
version = "1.1.0"

[[FileIO]]
deps = ["Pkg"]
git-tree-sha1 = "202335fd24c2776493e198d6c66a6d910400a895"
uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
version = "1.3.0"

[[FillArrays]]
deps = ["LinearAlgebra", "Random", "SparseArrays"]
git-tree-sha1 = "bf726ba7ce99e00d10bf63c031285fb9ab3676ae"
uuid = "1a297f60-69ca-5386-bcde-b61e274b549b"
version = "0.8.11"

[[Formatting]]
deps = ["Printf"]
git-tree-sha1 = "a0c901c29c0e7c763342751c0a94211d56c0de5c"
uuid = "59287772-0a20-5a39-b81b-1366585eb4c0"
version = "0.4.1"

[[ForwardDiff]]
deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "NaNMath", "Random", "SpecialFunctions", "StaticArrays"]
git-tree-sha1 = "869540e4367122fbffaace383a5bdc34d6e5e5ac"
uuid = "f6369f11-7733-5829-9624-2563aa707210"
version = "0.10.10"

[[GPUArrays]]
deps = ["AbstractFFTs", "Adapt", "LinearAlgebra", "Printf", "Random", "Serialization"]
git-tree-sha1 = "ae20accf251c6da038fe33cbc2c53d5af7f51344"
uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7"
version = "4.0.0"

[[GPUCompiler]]
deps = ["DataStructures", "InteractiveUtils", "LLVM", "Libdl", "TimerOutputs", "UUIDs"]
git-tree-sha1 = "65f7395a1245635f0c2279649fdbef09a1b0aa7b"
uuid = "61eb1bfa-7361-4325-ad38-22787b887f55"
version = "0.4.0"

[[GaussQuadrature]]
deps = ["SpecialFunctions"]
git-tree-sha1 = "ff30c61ad1ff425d0d8bac6b77833216741c2e68"
uuid = "d54b0c1a-921d-58e0-8e36-89d8069c0969"
version = "0.5.4"

[[GenericSVD]]
deps = ["LinearAlgebra"]
git-tree-sha1 = "62909c3eda8a25b5673a367d1ad2392ebb265211"
uuid = "01680d73-4ee2-5a08-a1aa-533608c188bb"
version = "0.3.0"

[[GenericSchur]]
deps = ["LinearAlgebra", "Printf"]
git-tree-sha1 = "43b4dc5648028be2c6e96201aa3653903bd1af21"
uuid = "c145ed77-6b09-5dd9-b285-bf645a82121e"
version = "0.4.0"

[[HTTP]]
deps = ["Base64", "Dates", "IniFile", "MbedTLS", "Sockets"]
git-tree-sha1 = "ec87d5e2acbe1693789efbbe14f5ea7525758f71"
uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3"
version = "0.8.15"

[[IniFile]]
deps = ["Test"]
git-tree-sha1 = "098e4d2c533924c921f9f9847274f2ad89e018b8"
uuid = "83e8ac13-25f8-5344-8a64-a9f2b223428f"
version = "0.5.0"

[[InteractiveUtils]]
deps = ["Markdown"]
uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"

[[Intervals]]
deps = ["Dates", "Printf", "RecipesBase", "TimeZones"]
git-tree-sha1 = "11555ad4a66a93e2268364f2ea11139634f9dda3"
uuid = "d8418881-c3e1-53bb-8760-2df7ec849ed5"
version = "1.2.0"

[[JLD2]]
deps = ["CodecZlib", "DataStructures", "FileIO", "Mmap", "Pkg", "Printf", "UUIDs"]
git-tree-sha1 = "e03e697cf84c275ece9cbefd1eabaf49bf5e7254"
uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
version = "0.1.13"

[[JSON]]
deps = ["Dates", "Mmap", "Parsers", "Unicode"]
git-tree-sha1 = "b34d7cef7b337321e97d22242c3c2b91f476748e"
uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
version = "0.21.0"

[[KernelAbstractions]]
deps = ["Adapt", "CUDA", "Cassette", "LLVM", "MacroTools", "SpecialFunctions", "StaticArrays"]
git-tree-sha1 = "6e812284039b7df4c32845d6d23fff9afc29d0ee"
uuid = "63c18a36-062a-441e-b654-da1e3ab1ce7c"
version = "0.3.0"

[[LLVM]]
deps = ["CEnum", "Libdl", "Printf", "Unicode"]
git-tree-sha1 = "0efb86dc7db298459186546bab008f3b53157582"
uuid = "929cbde3-209d-540e-8aea-75f648917ca0"
version = "1.6.0"

[[LazyArrays]]
deps = ["ArrayLayouts", "FillArrays", "LinearAlgebra", "MacroTools", "MatrixFactorizations", "StaticArrays"]
git-tree-sha1 = "eb02bd7606db9516205b400da8ba01e829ed5f7b"
uuid = "5078a376-72f3-5289-bfd5-ec5146d43c02"
version = "0.16.13"

[[LibGit2]]
deps = ["Printf"]
uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"

[[Libdl]]
uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"

[[Libiconv_jll]]
deps = ["Libdl", "Pkg"]
git-tree-sha1 = "a06937d9aa48855f52484c9a77a2670158fa90e2"
uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531"
version = "1.16.0+4"

[[LightXML]]
deps = ["Libdl", "XML2_jll"]
git-tree-sha1 = "e129d9391168c677cd4800f5c0abb1ed8cb3794f"
uuid = "9c8b4983-aa76-5018-a973-4c85ecc9e179"
version = "0.9.0"

[[LinearAlgebra]]
deps = ["Libdl"]
uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"

[[Literate]]
deps = ["Base64", "JSON", "REPL"]
git-tree-sha1 = "422133037d6dc5df9f9b97c2cb81fcd9e35ddffe"
uuid = "98b081ad-f1c9-55d3-8b20-4c87d4299306"
version = "2.5.0"

[[Logging]]
uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"

[[MPI]]
deps = ["Distributed", "DocStringExtensions", "Libdl", "MPICH_jll", "MicrosoftMPI_jll", "OpenMPI_jll", "Pkg", "Random", "Requires", "Serialization", "Sockets"]
git-tree-sha1 = "925f4e00b51d7a68ecb88c1b60d3b7aefb8c8fcb"
uuid = "da04e1cc-30fd-572f-bb4f-1f8673147195"
version = "0.15.0"

[[MPICH_jll]]
deps = ["CompilerSupportLibraries_jll", "Libdl", "Pkg"]
git-tree-sha1 = "4d37f1e07b4e2a74462eebf9ee48c626d15ffdac"
uuid = "7cb0a576-ebde-5e09-9194-50597f1243b4"
version = "3.3.2+10"

[[MacroTools]]
deps = ["Markdown", "Random"]
git-tree-sha1 = "f7d2e3f654af75f01ec49be82c231c382214223a"
uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
version = "0.5.5"

[[Markdown]]
deps = ["Base64"]
uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"

[[MatrixFactorizations]]
deps = ["ArrayLayouts", "LinearAlgebra", "Random"]
git-tree-sha1 = "69454f168628adeff74264837fc4c9d258dcc444"
uuid = "a3b82374-2e81-5b9e-98ce-41277c0e4c87"
version = "0.4.1"

[[MbedTLS]]
deps = ["Dates", "MbedTLS_jll", "Random", "Sockets"]
git-tree-sha1 = "426a6978b03a97ceb7ead77775a1da066343ec6e"
uuid = "739be429-bea8-5141-9913-cc70e7f3736d"
version = "1.0.2"

[[MbedTLS_jll]]
deps = ["Libdl", "Pkg"]
git-tree-sha1 = "c83f5a1d038f034ad0549f9ee4d5fac3fb429e33"
uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
version = "2.16.0+2"

[[MicrosoftMPI_jll]]
deps = ["Libdl", "Pkg"]
git-tree-sha1 = "720de13004e416f2864c92593d3839e062703985"
uuid = "9237b28f-5490-5468-be7b-bb81f5f5e6cf"
version = "10.1.2+3"

[[Missings]]
deps = ["DataAPI"]
git-tree-sha1 = "de0a5ce9e5289f27df672ffabef4d1e5861247d5"
uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28"
version = "0.4.3"

[[Mmap]]
uuid = "a63ad114-7e13-5084-954f-fe012c677804"

[[Mocking]]
deps = ["ExprTools"]
git-tree-sha1 = "916b850daad0d46b8c71f65f719c49957e9513ed"
uuid = "78c3b35d-d492-501b-9361-3d52fe80e533"
version = "0.7.1"

[[NCDatasets]]
deps = ["BinDeps", "CFTime", "CondaBinDeps", "DataStructures", "Dates", "Libdl", "Printf"]
git-tree-sha1 = "271e4670a6b30ec1e5daf6ba8b30240b7d85f535"
uuid = "85f8d34a-cbdd-5861-8df4-14fed0d494ab"
version = "0.10.2"

[[NNPACK_jll]]
deps = ["Libdl", "Pkg"]
git-tree-sha1 = "c3d1a616362645754b18e12dbba96ec311b0867f"
uuid = "a6bfbf70-4841-5cb9-aa18-3a8ad3c413ee"
version = "2018.6.22+0"

[[NNlib]]
deps = ["Libdl", "LinearAlgebra", "NNPACK_jll", "Pkg", "Requires", "Statistics"]
git-tree-sha1 = "04bae5dcd446ee45d6a3b9e84adca85ee33fb20c"
uuid = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
version = "0.7.0"

[[NaNMath]]
git-tree-sha1 = "928b8ca9b2791081dc71a51c55347c27c618760f"
uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3"
version = "0.3.3"

[[OpenBLAS_jll]]
deps = ["CompilerSupportLibraries_jll", "Libdl", "Pkg"]
git-tree-sha1 = "1887096f6897306a4662f7c5af936da7d5d1a062"
uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"
version = "0.3.9+4"

[[OpenMPI_jll]]
deps = ["Libdl", "Pkg"]
git-tree-sha1 = "41b983e26a7ab8c9bf05f7d70c274b817d541b46"
uuid = "fe0851c0-eecd-5654-98d4-656369965a5c"
version = "4.0.2+2"

[[OpenSpecFun_jll]]
deps = ["CompilerSupportLibraries_jll", "Libdl", "Pkg"]
git-tree-sha1 = "d51c416559217d974a1113522d5919235ae67a87"
uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e"
version = "0.5.3+3"

[[OrderedCollections]]
git-tree-sha1 = "12ce190210d278e12644bcadf5b21cbdcf225cd3"
uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
version = "1.2.0"

[[PDMats]]
deps = ["Arpack", "LinearAlgebra", "SparseArrays", "SuiteSparse", "Test"]
git-tree-sha1 = "2fc6f50ddd959e462f0a2dbc802ddf2a539c6e35"
uuid = "90014a1f-27ba-587c-ab20-58faa44d9150"
version = "0.9.12"

[[PackageCompiler]]
deps = ["Libdl", "Pkg", "UUIDs"]
git-tree-sha1 = "8989a56e4506b7b6a050f2b912e7856742d6998c"
uuid = "9b87118b-4619-50d2-8e1e-99f35a4d4d9d"
version = "1.1.1"

[[Parsers]]
deps = ["Dates", "Test"]
git-tree-sha1 = "eb3e09940c0d7ae01b01d9291ebad7b081c844d3"
uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
version = "1.0.5"

[[Pkg]]
deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"]
uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"

[[Polynomials]]
deps = ["Intervals", "LinearAlgebra", "RecipesBase"]
git-tree-sha1 = "05f4b8b55595b86b252b28fb1e7c402fb4df98d2"
uuid = "f27b6e38-b328-58d1-80ce-0feddd5e7a45"
version = "1.1.1"

[[Printf]]
deps = ["Unicode"]
uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"

[[QuadGK]]
deps = ["DataStructures", "LinearAlgebra"]
git-tree-sha1 = "dc84e810393cfc6294248c9032a9cdacc14a3db4"
uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc"
version = "2.3.1"

[[Quadmath]]
deps = ["Printf", "Random", "Requires"]
git-tree-sha1 = "cd993c45147a8432bf24358f14bf2cfd4aeb14df"
uuid = "be4d8f0f-7fa4-5f49-b795-2f01399ab2dd"
version = "0.5.4"

[[REPL]]
deps = ["InteractiveUtils", "Markdown", "Sockets"]
uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"

[[Random]]
deps = ["Serialization"]
uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"

[[RecipesBase]]
git-tree-sha1 = "54f8ceb165a0f6d083f0d12cb4996f5367c6edbc"
uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01"
version = "1.0.1"

[[Reexport]]
deps = ["Pkg"]
git-tree-sha1 = "7b1d07f411bc8ddb7977ec7f377b97b158514fe0"
uuid = "189a3867-3050-52da-a836-e630ba90ab69"
version = "0.2.0"

[[Requires]]
deps = ["UUIDs"]
git-tree-sha1 = "d37400976e98018ee840e0ca4f9d20baa231dc6b"
uuid = "ae029012-a4dd-5104-9daa-d747884805df"
version = "1.0.1"

[[Rmath]]
deps = ["Random", "Rmath_jll"]
git-tree-sha1 = "86c5647b565873641538d8f812c04e4c9dbeb370"
uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa"
version = "0.6.1"

[[Rmath_jll]]
deps = ["Libdl", "Pkg"]
git-tree-sha1 = "d76185aa1f421306dec73c057aa384bad74188f0"
uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f"
version = "0.2.2+1"

[[RootSolvers]]
deps = ["DocStringExtensions", "ForwardDiff", "Test"]
git-tree-sha1 = "0e5b394adc5c6fb39b3964bce2a259a44cc312d3"
uuid = "7181ea78-2dcb-4de3-ab41-2b8ab5a31e74"
version = "0.2.0"

[[SHA]]
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"

[[Serialization]]
uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"

[[Sockets]]
uuid = "6462fe0b-24de-5631-8697-dd941f90decc"

[[SortingAlgorithms]]
deps = ["DataStructures", "Random", "Test"]
git-tree-sha1 = "03f5898c9959f8115e30bc7226ada7d0df554ddd"
uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c"
version = "0.3.1"

[[SparseArrays]]
deps = ["LinearAlgebra", "Random"]
uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"

[[SpecialFunctions]]
deps = ["OpenSpecFun_jll"]
git-tree-sha1 = "d8d8b8a9f4119829410ecd706da4cc8594a1e020"
uuid = "276daf66-3868-5448-9aa4-cd146d93841b"
version = "0.10.3"

[[StaticArrays]]
deps = ["LinearAlgebra", "Random", "Statistics"]
git-tree-sha1 = "5c06c0aeb81bef54aed4b3f446847905eb6cbda0"
uuid = "90137ffa-7385-5640-81b9-e52037218182"
version = "0.12.3"

[[StaticNumbers]]
deps = ["Requires"]
git-tree-sha1 = "a0df7d5ade3fd0f0e6c93ad63facc05b12c40e6a"
uuid = "c5e4b96a-f99f-5557-8ed2-dc63ef9b5131"
version = "0.3.3"

[[Statistics]]
deps = ["LinearAlgebra", "SparseArrays"]
uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"

[[StatsBase]]
deps = ["DataAPI", "DataStructures", "LinearAlgebra", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics"]
git-tree-sha1 = "a6102b1f364befdb05746f386b67c6b7e3262c45"
uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
version = "0.33.0"

[[StatsFuns]]
deps = ["Rmath", "SpecialFunctions"]
git-tree-sha1 = "04a5a8e6ab87966b43f247920eab053fd5fdc925"
uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c"
version = "0.9.5"

[[SuiteSparse]]
deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"]
uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9"

[[Test]]
deps = ["Distributed", "InteractiveUtils", "Logging", "Random"]
uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

[[TextWrap]]
git-tree-sha1 = "9250ef9b01b66667380cf3275b3f7488d0e25faf"
uuid = "b718987f-49a8-5099-9789-dcd902bef87d"
version = "1.0.1"

[[TimeZones]]
deps = ["Dates", "EzXML", "Mocking", "Printf", "RecipesBase", "Serialization", "Unicode"]
git-tree-sha1 = "db7bc2051d4c2e5f336409224df81485c00de6cb"
uuid = "f269a46b-ccf7-5d73-abea-4c690281aa53"
version = "1.2.0"

[[TimerOutputs]]
deps = ["Printf"]
git-tree-sha1 = "f458ca23ff80e46a630922c555d838303e4b9603"
uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f"
version = "0.5.6"

[[TranscodingStreams]]
deps = ["Random", "Test"]
git-tree-sha1 = "7c53c35547de1c5b9d46a4797cf6d8253807108c"
uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"
version = "0.9.5"

[[URIParser]]
deps = ["Unicode"]
git-tree-sha1 = "53a9f49546b8d2dd2e688d216421d050c9a31d0d"
uuid = "30578b45-9adc-5946-b283-645ec420af67"
version = "0.4.1"

[[UUIDs]]
deps = ["Random", "SHA"]
uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"

[[Unicode]]
uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"

[[VersionParsing]]
git-tree-sha1 = "80229be1f670524750d905f8fc8148e5a8c4537f"
uuid = "81def892-9a0e-5fdd-b105-ffc91e053289"
version = "1.2.0"

[[WriteVTK]]
deps = ["Base64", "CodecZlib", "LightXML", "Random", "TranscodingStreams"]
git-tree-sha1 = "46c5cc2f4e094eb57685f5fb0629b7e14b6ad18b"
uuid = "64499a7a-5c06-52f2-abe2-ccb03c286192"
version = "1.6.0"

[[XML2_jll]]
deps = ["Libdl", "Libiconv_jll", "Pkg", "Zlib_jll"]
git-tree-sha1 = "6b2ffe6728bdba991da4fc1aa5980a53db46a23f"
uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a"
version = "2.9.9+5"

[[Zlib_jll]]
deps = ["Libdl", "Pkg"]
git-tree-sha1 = "4838361eb3ba796a57457bdc183e1a128639d0ec"
uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
version = "1.2.11+12"

out_10192325

+ hostname
hpc-23-28.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_NUM_THREADS=4
+ JULIA_NUM_THREADS=4
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_

out_10192326

+ hostname
hpc-23-28.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192326_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192326_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/advection_diffusion/advection_diffusion_model_1dimex_bgmres.jl
[ Info: (Array, Float64, 2, "Batched GMRES", 1, true)
┌ Info: Starting
└ norm(Q₀) = 4.2914804456671035e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 2.7809865655158178e-01
│ norm(Q) / norm(Q₀)      = 6.4802498828199084e-01
│ norm(Q) - norm(Q₀)      = -1.5104938801512857e-01
│ norm(Q - Qe)            = 7.2801198255145042e-02
└ norm(Q - Qe) / norm(Qe) = 2.8371215264817484e-01
[ Info: 
[ Info: (Array, Float64, 2, "Batched GMRES", 1, false)
┌ Info: Starting
└ norm(Q₀) = 4.2914804456671035e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 2.7809865655158178e-01
│ norm(Q) / norm(Q₀)      = 6.4802498828199084e-01
│ norm(Q) - norm(Q₀)      = -1.5104938801512857e-01
│ norm(Q - Qe)            = 7.2801198255145042e-02
└ norm(Q - Qe) / norm(Qe) = 2.8371215264817484e-01
[ Info: 
[ Info: (Array, Float64, 3, "Batched GMRES", 1, true)
┌ Info: Starting
└ norm(Q₀) = 5.6541735527969472e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Update
│ simtime = 4.9218750000000000e-01
│ runtime = 00:01:01
└ norm(Q) = 3.6921895132000682e-01
┌ Info: Finished
│ norm(Q)                 = 3.6810030090193091e-01
│ norm(Q) / norm(Q₀)      = 6.5102405765355920e-01
│ norm(Q) - norm(Q₀)      = -1.9731705437776381e-01
│ norm(Q - Qe)            = 1.0462203776128312e-01
└ norm(Q - Qe) / norm(Qe) = 2.8922403247396861e-01
[ Info: 
[ Info: (Array, Float64, 3, "Batched GMRES", 1, false)
┌ Info: Starting
└ norm(Q₀) = 5.6541735527969472e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 3.6810030090193091e-01
│ norm(Q) / norm(Q₀)      = 6.5102405765355920e-01
│ norm(Q) - norm(Q₀)      = -1.9731705437776381e-01
│ norm(Q - Qe)            = 1.0462203776128312e-01
└ norm(Q - Qe) / norm(Qe) = 2.8922403247396861e-01
[ Info: 
[ Info: (Array, Float32, 2, "Batched GMRES", 1, true)
┌ Info: Starting
└ norm(Q₀) = 4.2914792895317078e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 2.7809858322143555e-01
│ norm(Q) / norm(Q₀)      = 6.4802497625350952e-01
│ norm(Q) - norm(Q₀)      = -1.5104934573173523e-01
│ norm(Q - Qe)            = 7.2801239788532257e-02
└ norm(Q - Qe) / norm(Qe) = 2.8371238708496094e-01
[ Info: 
[ Info: (Array, Float32, 2, "Batched GMRES", 1, false)
┌ Info: Starting
└ norm(Q₀) = 4.2914792895317078e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 2.7809858322143555e-01
│ norm(Q) / norm(Q₀)      = 6.4802497625350952e-01
│ norm(Q) - norm(Q₀)      = -1.5104934573173523e-01
│ norm(Q - Qe)            = 7.2801239788532257e-02
└ norm(Q - Qe) / norm(Qe) = 2.8371238708496094e-01
[ Info: 
[ Info: (Array, Float32, 3, "Batched GMRES", 1, true)
┌ Info: Starting
└ norm(Q₀) = 5.6541705131530762e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 3.6809968948364258e-01
│ norm(Q) / norm(Q₀)      = 6.5102332830429077e-01
│ norm(Q) - norm(Q₀)      = -1.9731736183166504e-01
│ norm(Q - Qe)            = 1.0462126135826111e-01
└ norm(Q - Qe) / norm(Qe) = 2.8922191262245178e-01
[ Info: 
[ Info: (Array, Float32, 3, "Batched GMRES", 1, false)
┌ Info: Starting
└ norm(Q₀) = 5.6541705131530762e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 3.6809968948364258e-01
│ norm(Q) / norm(Q₀)      = 6.5102332830429077e-01
│ norm(Q) - norm(Q₀)      = -1.9731736183166504e-01
│ norm(Q - Qe)            = 1.0462126135826111e-01
└ norm(Q - Qe) / norm(Qe) = 2.8922191262245178e-01
[ Info: 
Test Summary:                                                                                                                                                          | Pass  Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/advection_diffusion_model_1dimex_bgmres.jl |    8      8

out_10192327

+ hostname
hpc-23-28.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192327_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192327_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/advection_diffusion/pseudo1D_advection_diffusion.jl
┌ Info: time step
└   dt = 0.00390625
[ Info: (Array, Float64, 2, EveryDirection, true)
┌ Info: Starting
└ norm(Q₀) = 3.8737679305268846e-01
┌ Info: Finished
│ norm(Q)                 = 2.1231902141153636e-01
│ norm(Q) / norm(Q₀)      = 5.4809432371612954e-01
│ norm(Q) - norm(Q₀)      = -1.7505777164115210e-01
│ norm(Q - Qe)            = 1.2357162985295374e-02
└ norm(Q - Qe) / norm(Qe) = 5.9111207392709135e-02
[ Info: 
┌ Info: time step
└   dt = 0.00390625
[ Info: (Array, Float64, 2, EveryDirection, false)
┌ Info: Starting
└ norm(Q₀) = 3.8737679305268846e-01
┌ Info: Finished
│ norm(Q)                 = 2.1231902141153636e-01
│ norm(Q) / norm(Q₀)      = 5.4809432371612954e-01
│ norm(Q) - norm(Q₀)      = -1.7505777164115210e-01
│ norm(Q - Qe)            = 1.2357162985295374e-02
└ norm(Q - Qe) / norm(Qe) = 5.9111207392709135e-02
[ Info: 
┌ Info: time step
└   dt = 0.00390625
[ Info: (Array, Float64, 2, HorizontalDirection, true)
┌ Info: Starting
└ norm(Q₀) = 3.3630637465395113e-01
┌ Info: Finished
│ norm(Q)                 = 1.7559633237315539e-01
│ norm(Q) / norm(Q₀)      = 5.2213203675915620e-01
│ norm(Q) - norm(Q₀)      = -1.6071004228079575e-01
│ norm(Q - Qe)            = 4.6783743619257093e-02
└ norm(Q - Qe) / norm(Qe) = 2.1319129181721064e-01
[ Info: 
┌ Info: time step
└   dt = 0.00390625
[ Info: (Array, Float64, 2, HorizontalDirection, false)
┌ Info: Starting
└ norm(Q₀) = 3.3630637465395113e-01
┌ Info: Finished
│ norm(Q)                 = 1.7559633237315539e-01
│ norm(Q) / norm(Q₀)      = 5.2213203675915620e-01
│ norm(Q) - norm(Q₀)      = -1.6071004228079575e-01
│ norm(Q - Qe)            = 4.6783743619257093e-02
└ norm(Q - Qe) / norm(Qe) = 2.1319129181721064e-01
[ Info: 
┌ Info: time step
└   dt = 0.00390625
[ Info: (Array, Float64, 2, VerticalDirection, true)
┌ Info: Starting
└ norm(Q₀) = 3.3630637465395119e-01
┌ Info: Finished
│ norm(Q)                 = 1.7559633237315533e-01
│ norm(Q) / norm(Q₀)      = 5.2213203675915598e-01
│ norm(Q) - norm(Q₀)      = -1.6071004228079586e-01
│ norm(Q - Qe)            = 4.6783743619257134e-02
└ norm(Q - Qe) / norm(Qe) = 2.1319129181721083e-01
[ Info: 
┌ Info: time step
└   dt = 0.00390625
[ Info: (Array, Float64, 2, VerticalDirection, false)
┌ Info: Starting
└ norm(Q₀) = 3.3630637465395119e-01
┌ Info: Finished
│ norm(Q)                 = 1.7559633237315533e-01
│ norm(Q) / norm(Q₀)      = 5.2213203675915598e-01
│ norm(Q) - norm(Q₀)      = -1.6071004228079586e-01
│ norm(Q - Qe)            = 4.6783743619257134e-02
└ norm(Q - Qe) / norm(Qe) = 2.1319129181721083e-01
[ Info: 
┌ Info: time step
└   dt = 0.00390625
[ Info: (Array, Float64, 3, EveryDirection, true)
┌ Info: Starting
└ norm(Q₀) = 5.4103588436435512e-01
┌ Info: Finished
│ norm(Q)                 = 3.0178604704383444e-01
│ norm(Q) / norm(Q₀)      = 5.5779303326320528e-01
│ norm(Q) - norm(Q₀)      = -2.3924983732052069e-01
│ norm(Q - Qe)            = 9.6252415559793074e-03
└ norm(Q - Qe) / norm(Qe) = 3.1725960166418578e-02
[ Info: 
┌ Info: time step
└   dt = 0.00390625
[ Info: (Array, Float64, 3, EveryDirection, false)
┌ Info: Starting
└ norm(Q₀) = 5.4103588436435512e-01
┌ Info: Finished
│ norm(Q)                 = 3.0178604704383444e-01
│ norm(Q) / norm(Q₀)      = 5.5779303326320528e-01
│ norm(Q) - norm(Q₀)      = -2.3924983732052069e-01
│ norm(Q - Qe)            = 9.6252415559793074e-03
└ norm(Q - Qe) / norm(Qe) = 3.1725960166418578e-02
[ Info: 
┌ Info: time step
└   dt = 0.00390625
[ Info: (Array, Float64, 3, HorizontalDirection, true)
┌ Info: Starting
└ norm(Q₀) = 5.4783351448370787e-01
┌ Info: Finished
│ norm(Q)                 = 3.0026443962997829e-01
│ norm(Q) / norm(Q₀)      = 5.4809432371612943e-01
│ norm(Q) - norm(Q₀)      = -2.4756907485372959e-01
│ norm(Q - Qe)            = 1.7475667486259405e-02
└ norm(Q - Qe) / norm(Qe) = 5.9111207392708767e-02
[ Info: 
┌ Info: time step
└   dt = 0.00390625
[ Info: (Array, Float64, 3, HorizontalDirection, false)
┌ Info: Starting
└ norm(Q₀) = 5.4783351448370787e-01
┌ Info: Finished
│ norm(Q)                 = 3.0026443962997829e-01
│ norm(Q) / norm(Q₀)      = 5.4809432371612943e-01
│ norm(Q) - norm(Q₀)      = -2.4756907485372959e-01
│ norm(Q - Qe)            = 1.7475667486259405e-02
└ norm(Q - Qe) / norm(Qe) = 5.9111207392708767e-02
[ Info: 
┌ Info: time step
└   dt = 0.00390625
[ Info: (Array, Float64, 3, VerticalDirection, true)
┌ Info: Starting
└ norm(Q₀) = 4.7560903614814531e-01
┌ Info: Finished
│ norm(Q)                 = 2.4833071474509019e-01
│ norm(Q) / norm(Q₀)      = 5.2213203675915609e-01
│ norm(Q) - norm(Q₀)      = -2.2727832140305512e-01
│ norm(Q - Qe)            = 6.6162204724939042e-02
└ norm(Q - Qe) / norm(Qe) = 2.1319129181721019e-01
[ Info: 
┌ Info: time step
└   dt = 0.00390625
[ Info: (Array, Float64, 3, VerticalDirection, false)
┌ Info: Starting
└ norm(Q₀) = 4.7560903614814531e-01
┌ Info: Finished
│ norm(Q)                 = 2.4833071474509019e-01
│ norm(Q) / norm(Q₀)      = 5.2213203675915609e-01
│ norm(Q) - norm(Q₀)      = -2.2727832140305512e-01
│ norm(Q - Qe)            = 6.6162204724939042e-02
└ norm(Q - Qe) / norm(Qe) = 2.1319129181721019e-01
[ Info: 
┌ Info: time step
└   dt = 0.00390625f0
[ Info: (Array, Float32, 2, EveryDirection, true)
┌ Info: Starting
└ norm(Q₀) = 3.8737669587135315e-01
┌ Info: Finished
│ norm(Q)                 = 2.1231888234615326e-01
│ norm(Q) / norm(Q₀)      = 5.4809409379959106e-01
│ norm(Q) - norm(Q₀)      = -1.7505781352519989e-01
│ norm(Q - Qe)            = 1.2357012368738651e-02
└ norm(Q - Qe) / norm(Qe) = 5.9110499918460846e-02
[ Info: 
┌ Info: time step
└   dt = 0.00390625f0
[ Info: (Array, Float32, 2, EveryDirection, false)
┌ Info: Starting
└ norm(Q₀) = 3.8737669587135315e-01
┌ Info: Finished
│ norm(Q)                 = 2.1231888234615326e-01
│ norm(Q) / norm(Q₀)      = 5.4809409379959106e-01
│ norm(Q) - norm(Q₀)      = -1.7505781352519989e-01
│ norm(Q - Qe)            = 1.2357012368738651e-02
└ norm(Q - Qe) / norm(Qe) = 5.9110499918460846e-02
[ Info: 
┌ Info: time step
└   dt = 0.00390625f0
[ Info: (Array, Float32, 2, HorizontalDirection, true)
┌ Info: Starting
└ norm(Q₀) = 3.3630603551864624e-01
┌ Info: Finished
│ norm(Q)                 = 1.7559613287448883e-01
│ norm(Q) / norm(Q₀)      = 5.2213197946548462e-01
│ norm(Q) - norm(Q₀)      = -1.6070990264415741e-01
│ norm(Q - Qe)            = 4.6783894300460815e-02
└ norm(Q - Qe) / norm(Qe) = 2.1319198608398438e-01
[ Info: 
┌ Info: time step
└   dt = 0.00390625f0
[ Info: (Array, Float32, 2, HorizontalDirection, false)
┌ Info: Starting
└ norm(Q₀) = 3.3630603551864624e-01
┌ Info: Finished
│ norm(Q)                 = 1.7559613287448883e-01
│ norm(Q) / norm(Q₀)      = 5.2213197946548462e-01
│ norm(Q) - norm(Q₀)      = -1.6070990264415741e-01
│ norm(Q - Qe)            = 4.6783894300460815e-02
└ norm(Q - Qe) / norm(Qe) = 2.1319198608398438e-01
[ Info: 
┌ Info: time step
└   dt = 0.00390625f0
[ Info: (Array, Float32, 2, VerticalDirection, true)
┌ Info: Starting
└ norm(Q₀) = 3.3630603551864624e-01
┌ Info: Finished
│ norm(Q)                 = 1.7559617757797241e-01
│ norm(Q) / norm(Q₀)      = 5.2213209867477417e-01
│ norm(Q) - norm(Q₀)      = -1.6070985794067383e-01
│ norm(Q - Qe)            = 4.6783849596977234e-02
└ norm(Q - Qe) / norm(Qe) = 2.1319174766540527e-01
[ Info: 
┌ Info: time step
└   dt = 0.00390625f0
[ Info: (Array, Float32, 2, VerticalDirection, false)
┌ Info: Starting
└ norm(Q₀) = 3.3630603551864624e-01
┌ Info: Finished
│ norm(Q)                 = 1.7559617757797241e-01
│ norm(Q) / norm(Q₀)      = 5.2213209867477417e-01
│ norm(Q) - norm(Q₀)      = -1.6070985794067383e-01
│ norm(Q - Qe)            = 4.6783849596977234e-02
└ norm(Q - Qe) / norm(Qe) = 2.1319174766540527e-01
[ Info: 
┌ Info: time step
└   dt = 0.00390625f0
[ Info: (Array, Float32, 3, EveryDirection, true)
┌ Info: Starting
└ norm(Q₀) = 5.4103583097457886e-01
┌ Info: Finished
│ norm(Q)                 = 3.0178582668304443e-01
│ norm(Q) / norm(Q₀)      = 5.5779266357421875e-01
│ norm(Q) - norm(Q₀)      = -2.3925000429153442e-01
│ norm(Q - Qe)            = 9.6252299845218658e-03
└ norm(Q - Qe) / norm(Qe) = 3.1725935637950897e-02
[ Info: 
┌ Info: time step
└   dt = 0.00390625f0
[ Info: (Array, Float32, 3, EveryDirection, false)
┌ Info: Starting
└ norm(Q₀) = 5.4103583097457886e-01
┌ Info: Finished
│ norm(Q)                 = 3.0178582668304443e-01
│ norm(Q) / norm(Q₀)      = 5.5779266357421875e-01
│ norm(Q) - norm(Q₀)      = -2.3925000429153442e-01
│ norm(Q - Qe)            = 9.6252299845218658e-03
└ norm(Q - Qe) / norm(Qe) = 3.1725935637950897e-02
[ Info: 
┌ Info: time step
└   dt = 0.00390625f0
[ Info: (Array, Float32, 3, HorizontalDirection, true)
┌ Info: Starting
└ norm(Q₀) = 5.4783332347869873e-01
┌ Info: Finished
│ norm(Q)                 = 3.0026426911354065e-01
│ norm(Q) / norm(Q₀)      = 5.4809421300888062e-01
│ norm(Q) - norm(Q₀)      = -2.4756905436515808e-01
│ norm(Q - Qe)            = 1.7475524917244911e-02
└ norm(Q - Qe) / norm(Qe) = 5.9110768139362335e-02
[ Info: 
┌ Info: time step
└   dt = 0.00390625f0
[ Info: (Array, Float32, 3, HorizontalDirection, false)
┌ Info: Starting
└ norm(Q₀) = 5.4783332347869873e-01
┌ Info: Finished
│ norm(Q)                 = 3.0026426911354065e-01
│ norm(Q) / norm(Q₀)      = 5.4809421300888062e-01
│ norm(Q) - norm(Q₀)      = -2.4756905436515808e-01
│ norm(Q - Qe)            = 1.7475524917244911e-02
└ norm(Q - Qe) / norm(Qe) = 5.9110768139362335e-02
[ Info: 
┌ Info: time step
└   dt = 0.00390625f0
[ Info: (Array, Float32, 3, VerticalDirection, true)
┌ Info: Starting
└ norm(Q₀) = 4.7560846805572510e-01
┌ Info: Finished
│ norm(Q)                 = 2.4833039939403534e-01
│ norm(Q) / norm(Q₀)      = 5.2213197946548462e-01
│ norm(Q) - norm(Q₀)      = -2.2727806866168976e-01
│ norm(Q - Qe)            = 6.6162295639514923e-02
└ norm(Q - Qe) / norm(Qe) = 2.1319165825843811e-01
[ Info: 
┌ Info: time step
└   dt = 0.00390625f0
[ Info: (Array, Float32, 3, VerticalDirection, false)
┌ Info: Starting
└ norm(Q₀) = 4.7560846805572510e-01
┌ Info: Finished
│ norm(Q)                 = 2.4833039939403534e-01
│ norm(Q) / norm(Q₀)      = 5.2213197946548462e-01
│ norm(Q) - norm(Q₀)      = -2.2727806866168976e-01
│ norm(Q - Qe)            = 6.6162295639514923e-02
└ norm(Q - Qe) / norm(Qe) = 2.1319165825843811e-01
[ Info: 
Test Summary:                                                                                                                                               | Test Summary:                                                                                                                                               | Pass  Pass  Total
Test Summary:                                                                                                                                               | Total
Pass  Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/pseudo1D_advection_diffusion.jl |   24     24
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/pseudo1D_advection_diffusion.jl |   24     24
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/pseudo1D_advection_diffusion.jl |   24     24

out_10192328

+ hostname
hpc-23-28.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192328_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192328_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/advection_diffusion/pseudo1D_advection_diffusion_1dimex.jl
[ Info: (Array, Float64, 2, SingleColumnLU, 1, true)
┌ Info: Starting
└ norm(Q₀) = 4.2914804456671030e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 2.7809865655333527e-01
│ norm(Q) / norm(Q₀)      = 6.4802498828607691e-01
│ norm(Q) - norm(Q₀)      = -1.5104938801337503e-01
│ norm(Q - Qe)            = 7.2801198255507404e-02
└ norm(Q - Qe) / norm(Qe) = 2.8371215264958705e-01
[ Info: 
[ Info: (Array, Float64, 2, ManyColumnLU, 1, true)
┌ Info: Starting
└ norm(Q₀) = 4.2914804456671030e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 2.7809865655333532e-01
│ norm(Q) / norm(Q₀)      = 6.4802498828607702e-01
│ norm(Q) - norm(Q₀)      = -1.5104938801337497e-01
│ norm(Q - Qe)            = 7.2801198255507404e-02
└ norm(Q - Qe) / norm(Qe) = 2.8371215264958705e-01
[ Info: 
[ Info: (Array, Float64, 2, SingleColumnLU, 1, false)
┌ Info: Starting
└ norm(Q₀) = 4.2914804456671030e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 2.7809865655333527e-01
│ norm(Q) / norm(Q₀)      = 6.4802498828607691e-01
│ norm(Q) - norm(Q₀)      = -1.5104938801337503e-01
│ norm(Q - Qe)            = 7.2801198255507404e-02
└ norm(Q - Qe) / norm(Qe) = 2.8371215264958705e-01
[ Info: 
[ Info: (Array, Float64, 2, ManyColumnLU, 1, false)
┌ Info: Starting
└ norm(Q₀) = 4.2914804456671030e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 2.7809865655333532e-01
│ norm(Q) / norm(Q₀)      = 6.4802498828607702e-01
│ norm(Q) - norm(Q₀)      = -1.5104938801337497e-01
│ norm(Q - Qe)            = 7.2801198255507404e-02
└ norm(Q - Qe) / norm(Qe) = 2.8371215264958705e-01
[ Info: 
[ Info: (Array, Float64, 3, SingleColumnLU, 1, true)
┌ Info: Starting
└ norm(Q₀) = 5.6541735527969450e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 3.6810030091336315e-01
│ norm(Q) / norm(Q₀)      = 6.5102405767377858e-01
│ norm(Q) - norm(Q₀)      = -1.9731705436633135e-01
│ norm(Q - Qe)            = 1.0462203776357507e-01
└ norm(Q - Qe) / norm(Qe) = 2.8922403248030482e-01
[ Info: 
[ Info: (Array, Float64, 3, ManyColumnLU, 1, true)
┌ Info: Starting
└ norm(Q₀) = 5.6541735527969450e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 3.6810030091336315e-01
│ norm(Q) / norm(Q₀)      = 6.5102405767377858e-01
│ norm(Q) - norm(Q₀)      = -1.9731705436633135e-01
│ norm(Q - Qe)            = 1.0462203776357505e-01
└ norm(Q - Qe) / norm(Qe) = 2.8922403248030476e-01
[ Info: 
[ Info: (Array, Float64, 3, SingleColumnLU, 1, false)
┌ Info: Starting
└ norm(Q₀) = 5.6541735527969450e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 3.6810030091336315e-01
│ norm(Q) / norm(Q₀)      = 6.5102405767377858e-01
│ norm(Q) - norm(Q₀)      = -1.9731705436633135e-01
│ norm(Q - Qe)            = 1.0462203776357507e-01
└ norm(Q - Qe) / norm(Qe) = 2.8922403248030482e-01
[ Info: 
[ Info: (Array, Float64, 3, ManyColumnLU, 1, false)
┌ Info: Starting
└ norm(Q₀) = 5.6541735527969450e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 3.6810030091336315e-01
│ norm(Q) / norm(Q₀)      = 6.5102405767377858e-01
│ norm(Q) - norm(Q₀)      = -1.9731705436633135e-01
│ norm(Q - Qe)            = 1.0462203776357505e-01
└ norm(Q - Qe) / norm(Qe) = 2.8922403248030476e-01
[ Info: 
[ Info: (Array, Float32, 2, SingleColumnLU, 1, true)
┌ Info: Starting
└ norm(Q₀) = 4.2914795875549316e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 2.7809855341911316e-01
│ norm(Q) / norm(Q₀)      = 6.4802485704421997e-01
│ norm(Q) - norm(Q₀)      = -1.5104940533638000e-01
│ norm(Q - Qe)            = 7.2801202535629272e-02
└ norm(Q - Qe) / norm(Qe) = 2.8371223807334900e-01
[ Info: 
[ Info: (Array, Float32, 2, ManyColumnLU, 1, true)
┌ Info: Starting
└ norm(Q₀) = 4.2914795875549316e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 2.7809852361679077e-01
│ norm(Q) / norm(Q₀)      = 6.4802479743957520e-01
│ norm(Q) - norm(Q₀)      = -1.5104943513870239e-01
│ norm(Q - Qe)            = 7.2801187634468079e-02
└ norm(Q - Qe) / norm(Qe) = 2.8371217846870422e-01
[ Info: 
[ Info: (Array, Float32, 2, SingleColumnLU, 1, false)
┌ Info: Starting
└ norm(Q₀) = 4.2914795875549316e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 2.7809855341911316e-01
│ norm(Q) / norm(Q₀)      = 6.4802485704421997e-01
│ norm(Q) - norm(Q₀)      = -1.5104940533638000e-01
│ norm(Q - Qe)            = 7.2801202535629272e-02
└ norm(Q - Qe) / norm(Qe) = 2.8371223807334900e-01
[ Info: 
[ Info: (Array, Float32, 2, ManyColumnLU, 1, false)
┌ Info: Starting
└ norm(Q₀) = 4.2914795875549316e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 2.7809852361679077e-01
│ norm(Q) / norm(Q₀)      = 6.4802479743957520e-01
│ norm(Q) - norm(Q₀)      = -1.5104943513870239e-01
│ norm(Q - Qe)            = 7.2801187634468079e-02
└ norm(Q - Qe) / norm(Qe) = 2.8371217846870422e-01
[ Info: 
[ Info: (Array, Float32, 3, SingleColumnLU, 1, true)
┌ Info: Starting
└ norm(Q₀) = 5.6541711091995239e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 3.6809992790222168e-01
│ norm(Q) / norm(Q₀)      = 6.5102368593215942e-01
│ norm(Q) - norm(Q₀)      = -1.9731718301773071e-01
│ norm(Q - Qe)            = 1.0462181270122528e-01
└ norm(Q - Qe) / norm(Qe) = 2.8922349214553833e-01
[ Info: 
[ Info: (Array, Float32, 3, ManyColumnLU, 1, true)
┌ Info: Starting
└ norm(Q₀) = 5.6541711091995239e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 3.6809992790222168e-01
│ norm(Q) / norm(Q₀)      = 6.5102368593215942e-01
│ norm(Q) - norm(Q₀)      = -1.9731718301773071e-01
│ norm(Q - Qe)            = 1.0462182015180588e-01
└ norm(Q - Qe) / norm(Qe) = 2.8922352194786072e-01
[ Info: 
[ Info: (Array, Float32, 3, SingleColumnLU, 1, false)
┌ Info: Starting
└ norm(Q₀) = 5.6541711091995239e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 3.6809992790222168e-01
│ norm(Q) / norm(Q₀)      = 6.5102368593215942e-01
│ norm(Q) - norm(Q₀)      = -1.9731718301773071e-01
│ norm(Q - Qe)            = 1.0462181270122528e-01
└ norm(Q - Qe) / norm(Qe) = 2.8922349214553833e-01
[ Info: 
[ Info: (Array, Float32, 3, ManyColumnLU, 1, false)
┌ Info: Starting
└ norm(Q₀) = 5.6541711091995239e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 3.6809992790222168e-01
│ norm(Q) / norm(Q₀)      = 6.5102368593215942e-01
│ norm(Q) - norm(Q₀)      = -1.9731718301773071e-01
│ norm(Q - Qe)            = 1.0462182015180588e-01
└ norm(Q - Qe) / norm(Qe) = 2.8922352194786072e-01
[ Info: 
Test Summary:                                                                                                                                                      | Pass  Total
Test Summary:                                                                                                                                                      | Test Summary:                                                                                                                                                      | Pass  Pass  Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/pseudo1D_advection_diffusion_1dimex.jl |   16     16
Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/pseudo1D_advection_diffusion_1dimex.jl |   16     16
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/pseudo1D_advection_diffusion_1dimex.jl |   16     16

out_10192329

+ hostname
hpc-23-28.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192329_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192329_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/advection_diffusion/pseudo1D_advection_diffusion_mrigark_implicit.jl
[ Info: (Array, Float64, 2, SingleColumnLU, 1, true)
┌ Info: Starting
└ norm(Q₀) = 4.2914804456671030e-01
┌ Info: time step
│   dt = 0.125
│   numberofsteps = 4
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 2.7858038837683252e-01
│ norm(Q) / norm(Q₀)      = 6.4914751891296962e-01
│ norm(Q) - norm(Q₀)      = -1.5056765618987777e-01
│ norm(Q - Qe)            = 7.3188989633310012e-02
└ norm(Q - Qe) / norm(Qe) = 2.8522340698621473e-01
[ Info: 
[ Info: (Array, Float64, 2, SingleColumnLU, 1, false)
┌ Info: Starting
└ norm(Q₀) = 4.2914804456671030e-01
┌ Info: time step
│   dt = 0.125
│   numberofsteps = 4
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 2.7858038837683252e-01
│ norm(Q) / norm(Q₀)      = 6.4914751891296962e-01
│ norm(Q) - norm(Q₀)      = -1.5056765618987777e-01
│ norm(Q - Qe)            = 7.3188989633310012e-02
└ norm(Q - Qe) / norm(Qe) = 2.8522340698621473e-01
[ Info: 
[ Info: (Array, Float64, 3, SingleColumnLU, 1, true)
┌ Info: Starting
└ norm(Q₀) = 5.6541735527969450e-01
┌ Info: time step
│   dt = 0.125
│   numberofsteps = 4
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 3.6868537487180708e-01
│ norm(Q) / norm(Q₀)      = 6.5205882244175162e-01
│ norm(Q) - norm(Q₀)      = -1.9673198040788742e-01
│ norm(Q - Qe)            = 1.0501469629884561e-01
└ norm(Q - Qe) / norm(Qe) = 2.9030952352393552e-01
[ Info: 
[ Info: (Array, Float64, 3, SingleColumnLU, 1, false)
┌ Info: Starting
└ norm(Q₀) = 5.6541735527969450e-01
┌ Info: time step
│   dt = 0.125
│   numberofsteps = 4
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 3.6868537487180708e-01
│ norm(Q) / norm(Q₀)      = 6.5205882244175162e-01
│ norm(Q) - norm(Q₀)      = -1.9673198040788742e-01
│ norm(Q - Qe)            = 1.0501469629884561e-01
└ norm(Q - Qe) / norm(Qe) = 2.9030952352393552e-01
[ Info: 
[ Info: (Array, Float32, 2, SingleColumnLU, 1, true)
┌ Info: Starting
└ norm(Q₀) = 4.2914795875549316e-01
┌ Info: time step
│   dt = 0.125
│   numberofsteps = 4
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 2.7857267856597900e-01
│ norm(Q) / norm(Q₀)      = 6.4912968873977661e-01
│ norm(Q) - norm(Q₀)      = -1.5057528018951416e-01
│ norm(Q - Qe)            = 7.3185592889785767e-02
└ norm(Q - Qe) / norm(Qe) = 2.8521025180816650e-01
[ Info: 
[ Info: (Array, Float32, 2, SingleColumnLU, 1, false)
┌ Info: Starting
└ norm(Q₀) = 4.2914795875549316e-01
┌ Info: time step
│   dt = 0.125
│   numberofsteps = 4
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 2.7857267856597900e-01
│ norm(Q) / norm(Q₀)      = 6.4912968873977661e-01
│ norm(Q) - norm(Q₀)      = -1.5057528018951416e-01
│ norm(Q - Qe)            = 7.3185592889785767e-02
└ norm(Q - Qe) / norm(Qe) = 2.8521025180816650e-01
[ Info: 
[ Info: (Array, Float32, 3, SingleColumnLU, 1, true)
┌ Info: Starting
└ norm(Q₀) = 5.6541711091995239e-01
┌ Info: time step
│   dt = 0.125
│   numberofsteps = 4
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 3.6867141723632813e-01
│ norm(Q) / norm(Q₀)      = 6.5203440189361572e-01
│ norm(Q) - norm(Q₀)      = -1.9674569368362427e-01
│ norm(Q - Qe)            = 1.0501006990671158e-01
└ norm(Q - Qe) / norm(Qe) = 2.9029682278633118e-01
[ Info: 
[ Info: (Array, Float32, 3, SingleColumnLU, 1, false)
┌ Info: Starting
└ norm(Q₀) = 5.6541711091995239e-01
┌ Info: time step
│   dt = 0.125
│   numberofsteps = 4
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 3.6867141723632813e-01
│ norm(Q) / norm(Q₀)      = 6.5203440189361572e-01
│ norm(Q) - norm(Q₀)      = -1.9674569368362427e-01
│ norm(Q - Qe)            = 1.0501006990671158e-01
└ norm(Q - Qe) / norm(Qe) = 2.9029682278633118e-01
[ Info: 
Test Summary:                                                                                                                                                                | Pass  Total
Test Summary:                                                                                                                                                                | Pass  Total
Test Summary:                                                                                                                                                                | Pass  /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/pseudo1D_advection_diffusion_mrigark_implicit.jl |    8      8
Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/pseudo1D_advection_diffusion_mrigark_implicit.jl |    8      8
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/pseudo1D_advection_diffusion_mrigark_implicit.jl |    8      8

out_10192330

+ hostname
hpc-23-28.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192330_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192330_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/advection_diffusion/pseudo1D_heat_eqn.jl
[ Info: (Array, Float64, 2, EveryDirection)
┌ Info: time step
│   dt = 0.00024390243902439024
│   numberofsteps = 41
│   dt * numberofsteps = 0.01
└   timeend = 0.01
┌ Info: Starting
└ norm(Q₀) = 1.0525946028760074e+00
┌ Info: Finished
│ norm(Q)                 = 7.6769756052324201e-01
│ norm(Q) / norm(Q₀)      = 7.2933830215893158e-01
│ norm(Q) - norm(Q₀)      = -2.8489704235276536e-01
│ norm(Q - Qe)            = 5.1574832681277024e-03
└ norm(Q - Qe) / norm(Qe) = 6.7183283629129193e-03
[ Info: 
[ Info: (Array, Float64, 2, HorizontalDirection)
┌ Info: time step
│   dt = 0.00024390243902439024
│   numberofsteps = 41
│   dt * numberofsteps = 0.01
└   timeend = 0.01
┌ Info: Starting
└ norm(Q₀) = 9.0395387652143555e-01
┌ Info: Finished
│ norm(Q)                 = 5.7963038857095939e-01
│ norm(Q) / norm(Q₀)      = 6.4121677402554345e-01
│ norm(Q) - norm(Q₀)      = -3.2432348795047616e-01
│ norm(Q - Qe)            = 2.0515449798977917e-02
└ norm(Q - Qe) / norm(Qe) = 3.5415953970101416e-02
[ Info: 
[ Info: (Array, Float64, 2, VerticalDirection)
┌ Info: time step
│   dt = 0.00024390243902439024
│   numberofsteps = 41
│   dt * numberofsteps = 0.01
└   timeend = 0.01
┌ Info: Starting
└ norm(Q₀) = 9.0395387652143577e-01
┌ Info: Finished
│ norm(Q)                 = 5.7963038857095917e-01
│ norm(Q) / norm(Q₀)      = 6.4121677402554311e-01
│ norm(Q) - norm(Q₀)      = -3.2432348795047661e-01
│ norm(Q - Qe)            = 2.0515449798977878e-02
└ norm(Q - Qe) / norm(Qe) = 3.5415953970101353e-02
[ Info: 
[ Info: (Array, Float64, 3, EveryDirection)
┌ Info: time step
│   dt = 0.00024390243902439024
│   numberofsteps = 41
│   dt * numberofsteps = 0.01
└   timeend = 0.01
┌ Info: Starting
└ norm(Q₀) = 1.1510565538414834e+00
┌ Info: Finished
│ norm(Q)                 = 9.1444897086429000e-01
│ norm(Q) / norm(Q₀)      = 7.9444312949911100e-01
│ norm(Q) - norm(Q₀)      = -2.3660758297719342e-01
│ norm(Q - Qe)            = 1.2605810186712843e-03
└ norm(Q - Qe) / norm(Qe) = 1.3785136194861493e-03
[ Info: 
[ Info: (Array, Float64, 3, HorizontalDirection)
┌ Info: time step
│   dt = 0.00024390243902439024
│   numberofsteps = 41
│   dt * numberofsteps = 0.01
└   timeend = 0.01
┌ Info: Starting
└ norm(Q₀) = 1.0525946028760080e+00
┌ Info: Finished
│ norm(Q)                 = 7.6769756052324201e-01
│ norm(Q) / norm(Q₀)      = 7.2933830215893103e-01
│ norm(Q) - norm(Q₀)      = -2.8489704235276603e-01
│ norm(Q - Qe)            = 5.1574832681275419e-03
└ norm(Q - Qe) / norm(Qe) = 6.7183283629127077e-03
[ Info: 
[ Info: (Array, Float64, 3, VerticalDirection)
┌ Info: time step
│   dt = 0.00024390243902439024
│   numberofsteps = 41
│   dt * numberofsteps = 0.01
└   timeend = 0.01
┌ Info: Starting
└ norm(Q₀) = 9.0395387652143522e-01
┌ Info: Finished
│ norm(Q)                 = 5.7963038857095983e-01
│ norm(Q) / norm(Q₀)      = 6.4121677402554422e-01
│ norm(Q) - norm(Q₀)      = -3.2432348795047539e-01
│ norm(Q - Qe)            = 2.0515449798978087e-02
└ norm(Q - Qe) / norm(Qe) = 3.5415953970101714e-02
[ Info: 
[ Info: (Array, Float32, 2, EveryDirection)
┌ Info: time step
│   dt = 0.00024390243902439024
│   numberofsteps = 41
│   dt * numberofsteps = 0.01
└   timeend = 0.01
┌ Info: Starting
└ norm(Q₀) = 1.0525944232940674e+00
┌ Info: Finished
│ norm(Q)                 = 7.6769733428955078e-01
│ norm(Q) / norm(Q₀)      = 7.2933822870254517e-01
│ norm(Q) - norm(Q₀)      = -2.8489708900451660e-01
│ norm(Q - Qe)            = 5.1570823416113853e-03
└ norm(Q - Qe) / norm(Qe) = 6.7178071476519108e-03
[ Info: 
[ Info: (Array, Float32, 2, HorizontalDirection)
┌ Info: time step
│   dt = 0.00024390243902439024
│   numberofsteps = 41
│   dt * numberofsteps = 0.01
└   timeend = 0.01
┌ Info: Starting
└ norm(Q₀) = 9.0395402908325195e-01
┌ Info: Finished
│ norm(Q)                 = 5.7963037490844727e-01
│ norm(Q) / norm(Q₀)      = 6.4121663570404053e-01
│ norm(Q) - norm(Q₀)      = -3.2432365417480469e-01
│ norm(Q - Qe)            = 2.0514531061053276e-02
└ norm(Q - Qe) / norm(Qe) = 3.5414375364780426e-02
[ Info: 
[ Info: (Array, Float32, 2, VerticalDirection)
┌ Info: time step
│   dt = 0.00024390243902439024
│   numberofsteps = 41
│   dt * numberofsteps = 0.01
└   timeend = 0.01
┌ Info: Starting
└ norm(Q₀) = 9.0395408868789673e-01
┌ Info: Finished
│ norm(Q)                 = 5.7963043451309204e-01
│ norm(Q) / norm(Q₀)      = 6.4121669530868530e-01
│ norm(Q) - norm(Q₀)      = -3.2432365417480469e-01
│ norm(Q - Qe)            = 2.0514758303761482e-02
└ norm(Q - Qe) / norm(Qe) = 3.5414762794971466e-02
[ Info: 
[ Info: (Array, Float32, 3, EveryDirection)
┌ Info: time step
│   dt = 0.00024390243902439024
│   numberofsteps = 41
│   dt * numberofsteps = 0.01
└   timeend = 0.01
┌ Info: Starting
└ norm(Q₀) = 1.1510562896728516e+00
┌ Info: Finished
│ norm(Q)                 = 9.1444873809814453e-01
│ norm(Q) / norm(Q₀)      = 7.9444313049316406e-01
│ norm(Q) - norm(Q₀)      = -2.3660755157470703e-01
│ norm(Q - Qe)            = 1.2602073838934302e-03
└ norm(Q - Qe) / norm(Qe) = 1.3781054876744747e-03
[ Info: 
[ Info: (Array, Float32, 3, HorizontalDirection)
┌ Info: time step
│   dt = 0.00024390243902439024
│   numberofsteps = 41
│   dt * numberofsteps = 0.01
└   timeend = 0.01
┌ Info: Starting
└ norm(Q₀) = 1.0525941848754883e+00
┌ Info: Finished
│ norm(Q)                 = 7.6769763231277466e-01
│ norm(Q) / norm(Q₀)      = 7.2933864593505859e-01
│ norm(Q) - norm(Q₀)      = -2.8489655256271362e-01
│ norm(Q - Qe)            = 5.1570436917245388e-03
└ norm(Q - Qe) / norm(Qe) = 6.7177568562328815e-03
[ Info: 
[ Info: (Array, Float32, 3, VerticalDirection)
┌ Info: time step
│   dt = 0.00024390243902439024
│   numberofsteps = 41
│   dt * numberofsteps = 0.01
└   timeend = 0.01
┌ Info: Starting
└ norm(Q₀) = 9.0395385026931763e-01
┌ Info: Finished
│ norm(Q)                 = 5.7963049411773682e-01
│ norm(Q) / norm(Q₀)      = 6.4121693372726440e-01
│ norm(Q) - norm(Q₀)      = -3.2432335615158081e-01
│ norm(Q - Qe)            = 2.0514657720923424e-02
└ norm(Q - Qe) / norm(Qe) = 3.5414576530456543e-02
[ Info: 
Test Summary:                                                                                                                                    | Test Summary:                                                                                                                                    | Pass  Pass  Total
Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/pseudo1D_heat_eqn.jl |   12  Test Summary:                                                                                                                                    |    12
Pass  /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/pseudo1D_heat_eqn.jl |   12  Total
   12
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/pseudo1D_heat_eqn.jl |   12     12

out_10192331

+ hostname
hpc-23-28.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192331_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192331_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/advection_diffusion/periodic_3D_hyperdiffusion.jl
[ Info: (Array, Float64, 2, EveryDirection)
┌ Info: time step
└   dt = 0.021649052503199043
┌ Info: Starting
└ norm(Q₀) = 4.4428829381583705e+00
┌ Info: Finished
│ norm(Q)                 = 4.2303924119636944e+00
│ norm(Q) / norm(Q₀)      = 9.5217282805953107e-01
│ norm(Q) - norm(Q₀)      = -2.1249052619467612e-01
│ norm(Q - Qe)            = 7.6772960298539606e-03
└ norm(Q - Qe) / norm(Qe) = 1.8147794217036933e-03
[ Info: 
[ Info: (Array, Float64, 2, HorizontalDirection)
┌ Info: time step
└   dt = 0.021649052503199043
┌ Info: Starting
└ norm(Q₀) = 4.4428829381583705e+00
┌ Info: Finished
│ norm(Q)                 = 4.4348927223358272e+00
│ norm(Q) / norm(Q₀)      = 9.9820156958133688e-01
│ norm(Q) - norm(Q₀)      = -7.9902158225433340e-03
│ norm(Q - Qe)            = 8.4650675812543560e-04
└ norm(Q - Qe) / norm(Qe) = 1.9087422609556854e-04
[ Info: 
[ Info: (Array, Float64, 2, VerticalDirection)
┌ Info: time step
└   dt = 0.021649052503199043
┌ Info: Starting
└ norm(Q₀) = 4.4428829381583705e+00
┌ Info: Finished
│ norm(Q)                 = 4.3444227732067962e+00
│ norm(Q) / norm(Q₀)      = 9.7783867675064440e-01
│ norm(Q) - norm(Q₀)      = -9.8460164951574392e-02
│ norm(Q - Qe)            = 6.1690465335714068e-03
└ norm(Q - Qe) / norm(Qe) = 1.4199772029164016e-03
[ Info: 
[ Info: (Array, Float64, 3, EveryDirection)
┌ Info: time step
└   dt = 0.021649052503198762
┌ Info: Starting
└ norm(Q₀) = 1.1136655993663432e+01
┌ Info: Finished
│ norm(Q)                 = 6.0910843493796429e+00
│ norm(Q) / norm(Q₀)      = 5.4694015446336552e-01
│ norm(Q) - norm(Q₀)      = -5.0455716442837888e+00
│ norm(Q - Qe)            = 1.7363355506159686e-01
└ norm(Q - Qe) / norm(Qe) = 2.8625704237051630e-02
[ Info: 
[ Info: (Array, Float64, 3, HorizontalDirection)
┌ Info: time step
└   dt = 0.021649052503198762
┌ Info: Starting
└ norm(Q₀) = 1.1136655993663432e+01
┌ Info: Finished
│ norm(Q)                 = 1.0604021232612634e+01
│ norm(Q) / norm(Q₀)      = 9.5217282805953085e-01
│ norm(Q) - norm(Q₀)      = -5.3263476105079732e-01
│ norm(Q - Qe)            = 1.9244127301143988e-02
└ norm(Q - Qe) / norm(Qe) = 1.8147794217036671e-03
[ Info: 
[ Info: (Array, Float64, 3, VerticalDirection)
┌ Info: time step
└   dt = 0.021649052503198762
┌ Info: Starting
└ norm(Q₀) = 1.1136655993663432e+01
┌ Info: Finished
│ norm(Q)                 = 9.4792047633662069e+00
│ norm(Q) / norm(Q₀)      = 8.5117155174405257e-01
│ norm(Q) - norm(Q₀)      = -1.6574512302972249e+00
│ norm(Q - Qe)            = 1.4412891107361817e-01
└ norm(Q - Qe) / norm(Qe) = 1.5217804720985051e-02
[ Info: 

out_10192332

+ hostname
hpc-23-28.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192332_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192332_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/compressible_Navier_Stokes/ref_state.jl
[ Info: (Array, Float64, 3)

out_10192333

+ hostname
hpc-23-28.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192333_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192333_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/Mesh/mpi_connect_1d.jl

out_10192334

+ hostname
hpc-91-24.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192334_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192334_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/Mesh/mpi_connect_sphere.jl

out_10192335

+ hostname
hpc-90-09.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192335_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192335_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/Mesh/mpi_getpartition.jl

out_10192336

+ hostname
hpc-90-10.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192336_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192336_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/Mesh/mpi_sortcolumns.jl

out_10192337

+ hostname
hpc-23-28.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192337_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192337_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/ODESolvers/ode_tests_convergence.jl
Test Summary: | Pass  Total
ODE Solvers   |  180    180

out_10192338

+ hostname
hpc-23-28.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192338_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192338_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Arrays/varsindex.jl
Test Summary:           | Pass  Total
MPIStateArray varsindex |    9      9

out_10192339

+ hostname
hpc-93-23.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192339_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192339_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Diagnostics/diagnostic_fields_test.jl
┌ Info: Model composition
│     param_set = EarthParameterSet()
│     orientation = FlatOrientation()
│     ref_state = HydrostaticState{DryAdiabaticProfile{Float32},Float32}(DryAdiabaticProfile{Float32}(290.0f0, 220.0f0), 0.0f0)
│     turbulence = SmagorinskyLilly{Float32}(0.23f0)
│     hyperdiffusion = NoHyperDiffusion()
│     moisture = EquilMoist{Float32}(3, 0.1f0)
│     precipitation = NoPrecipitation()
│     radiation = NoRadiation()
│     source = (Gravity(),)
│     tracers = NoTracers()
│     boundarycondition = AtmosBC{Impenetrable{FreeSlip},Insulating,Impermeable,ImpermeableTracer}(Impenetrable{FreeSlip}(FreeSlip()), Insulating(), Impermeable(), ImpermeableTracer())
│     init_state_conservative = init_risingbubble!
└     data_config = nothing
┌ Info: Establishing Atmos LES configuration for DryRisingBubble
│     precision        = Float32
│     polynomial order = 4
│     domain           = 2500.00 m x2500.00 m x2500.00 m
│     resolution       = 50x50x50
│     MPI ranks        = 3
│     min(Δ_horz)      = 35.97 m
└     min(Δ_vert)      = 35.97 m
[ Info: Initializing DryRisingBubble
┌ Info: Model composition
│     param_set = EarthParameterSet()
│     orientation = FlatOrientation()
│     ref_state = HydrostaticState{DryAdiabaticProfile{Float64},Float64}(DryAdiabaticProfile{Float64}(290.0, 220.0), 0.0)
│     turbulence = SmagorinskyLilly{Float64}(0.23)
│     hyperdiffusion = NoHyperDiffusion()
│     moisture = EquilMoist{Float64}(3, 0.1)
│     precipitation = NoPrecipitation()
│     radiation = NoRadiation()
│     source = (Gravity(),)
│     tracers = NoTracers()
│     boundarycondition = AtmosBC{Impenetrable{FreeSlip},Insulating,Impermeable,ImpermeableTracer}(Impenetrable{FreeSlip}(FreeSlip()), Insulating(), Impermeable(), ImpermeableTracer())
│     init_state_conservative = init_risingbubble!
└     data_config = nothing
┌ Info: Establishing Atmos LES configuration for DryRisingBubble
│     precision        = Float64
│     polynomial order = 4
│     domain           = 2500.00 m x2500.00 m x2500.00 m
│     resolution       = 50x50x50
│     MPI ranks        = 3
│     min(Δ_horz)      = 35.97 m
└     min(Δ_vert)      = 35.97 m
[ Info: Initializing DryRisingBubble
Test Summary:            | Test Summary:            | Test Summary:            | Pass  Pass  TotalTotal

Pass  Total
Diagnostics Fields tests | Diagnostics Fields tests |    2     2      2
    2
Diagnostics Fields tests |    2      2

out_10192340

+ hostname
hpc-23-28.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192340_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192340_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/vars_test.jl
[ Info: (Array, Float64, 2)
[ Info: (Array, Float64, 3)

out_10192341

+ hostname
hpc-90-18.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192341_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192341_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/remainder_model.jl
┌ Warning: `a::StaticArray - b::Number` is deprecated, use `a .- b` instead.
│   caller = macro expansion at remainder_model.jl:215 [inlined]
└ @ Core /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/remainder_model.jl:215
Test Summary:   | Pass  Test Summary:   | Pass  Test Summary:   | Total
Pass  Total
Total
remainder model |   24     24
remainder model |   24  remainder model |    24
  24     24

out_10192342

+ hostname
hpc-90-18.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192342_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192342_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/Euler/isentropicvortex.jl
┌ Info: Configuration
│ ArrayType     = Array
│ FT        = Float64
│ NumericalFlux = RusanovNumericalFlux
└ dims          = 2
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 0)
│ dt        = 3.5087719298245615e-06
└ norm(Q₀)  = 3.5435113910177101e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 3.5434920766920136e+03
│ norm(Q) / norm(Q₀)      = 9.9999454938235965e-01
│ norm(Q) - norm(Q₀)      = -1.9314325696541346e-02
│ norm(Q - Qe)            = 1.1990999506536774e+01
└ norm(Q - Qe) / norm(Qe) = 3.3837129375777785e-03
┌ Info: Configuration
│ ArrayType     = Array
│ FT        = Float64
│ NumericalFlux = CentralNumericalFluxFirstOrder
└ dims          = 2
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 0)
│ dt        = 3.5087719298245615e-06
└ norm(Q₀)  = 3.5435113910177101e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 3.5435098602154876e+03
│ norm(Q) / norm(Q₀)      = 9.9999956799850387e-01
│ norm(Q) - norm(Q₀)      = -1.5308022225326567e-03
│ norm(Q - Qe)            = 2.0840574601661327e+01
└ norm(Q - Qe) / norm(Qe) = 5.8809544498566457e-03
┌ Info: Configuration
│ ArrayType     = Array
│ FT        = Float64
│ NumericalFlux = RusanovNumericalFlux
└ dims          = 3
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 1)
│ dt        = 3.5087719298245615e-06
└ norm(Q₀)  = 1.1205566910367509e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 1.1205505833106824e+03
│ norm(Q) / norm(Q₀)      = 9.9999454938235854e-01
│ norm(Q) - norm(Q₀)      = -6.1077260684214707e-03
│ norm(Q - Qe)            = 3.7918869862611033e+00
└ norm(Q - Qe) / norm(Qe) = 3.3837129375777477e-03
┌ Info: Configuration
│ ArrayType     = Array
│ FT        = Float64
│ NumericalFlux = CentralNumericalFluxFirstOrder
└ dims          = 3
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 1)
│ dt        = 3.5087719298245615e-06
└ norm(Q₀)  = 1.1205566910367509e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 1.1205562069545820e+03
│ norm(Q) / norm(Q₀)      = 9.9999956799850220e-01
│ norm(Q) - norm(Q₀)      = -4.8408216889583855e-04
│ norm(Q - Qe)            = 6.5903683487904621e+00
└ norm(Q - Qe) / norm(Qe) = 5.8809544498564991e-03
┌ Info: Configuration
│ ArrayType     = Array
│ FT        = Float32
│ NumericalFlux = RusanovNumericalFlux
└ dims          = 2
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 0)
│ dt        = 3.5087718970316928e-06
└ norm(Q₀)  = 3.5435087890625000e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 3.5434868164062500e+03
│ norm(Q) / norm(Q₀)      = 9.9999380111694336e-01
│ norm(Q) - norm(Q₀)      = -2.1972656250000000e-02
│ norm(Q - Qe)            = 1.1990808486938477e+01
└ norm(Q - Qe) / norm(Qe) = 3.3836620859801769e-03
┌ Info: Configuration
│ ArrayType     = Array
│ FT        = Float32
│ NumericalFlux = CentralNumericalFluxFirstOrder
└ dims          = 2
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 0)
│ dt        = 3.5087718970316928e-06
└ norm(Q₀)  = 3.5435087890625000e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 3.5435048828125000e+03
│ norm(Q) / norm(Q₀)      = 9.9999892711639404e-01
│ norm(Q) - norm(Q₀)      = -3.9062500000000000e-03
│ norm(Q - Qe)            = 2.0840440750122070e+01
└ norm(Q - Qe) / norm(Qe) = 5.8809220790863037e-03
┌ Info: Configuration
│ ArrayType     = Array
│ FT        = Float32
│ NumericalFlux = RusanovNumericalFlux
└ dims          = 3
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 1)
│ dt        = 3.5087718970316928e-06
└ norm(Q₀)  = 1.1205548095703125e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 1.1205491943359375e+03
│ norm(Q) / norm(Q₀)      = 9.9999499320983887e-01
│ norm(Q) - norm(Q₀)      = -5.6152343750000000e-03
│ norm(Q - Qe)            = 3.7918071746826172e+00
└ norm(Q - Qe) / norm(Qe) = 3.3836476504802704e-03
┌ Info: Configuration
│ ArrayType     = Array
│ FT        = Float32
│ NumericalFlux = CentralNumericalFluxFirstOrder
└ dims          = 3
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 1)
│ dt        = 3.5087718970316928e-06
└ norm(Q₀)  = 1.1205548095703125e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 1.1205549316406250e+03
│ norm(Q) / norm(Q₀)      = 1.0000001192092896e+00
│ norm(Q) - norm(Q₀)      = 1.2207031250000000e-04
│ norm(Q - Qe)            = 6.5903987884521484e+00
└ norm(Q - Qe) / norm(Qe) = 5.8809919282793999e-03
Test Summary:                                                                                                                     | Pass  Total
Test Summary:                                                                                                                     | Test Summary:                                                                                                                     | Pass  Pass  TotalTotal

/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/Euler/isentropicvortex.jl |    8      8
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/Euler/isentropicvortex.jl/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/Euler/isentropicvortex.jl |  |    8     8      8
    8

out_10192343

+ hostname
hpc-90-18.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192343_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192343_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/Euler/isentropicvortex_imex.jl
┌ Info: Configuration
│ ArrayType = Array
│ FT    = Float64
│ dims      = 2
└ splitting = (Full, Linear)
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 0)
│ dt        = 3.5842293906810039e-06
└ norm(Q₀)  = 3.5435113910177101e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 3.5433940589518711e+03
│ norm(Q) / norm(Q₀)      = 9.9996688819284274e-01
│ norm(Q) - norm(Q₀)      = -1.1733206583903666e-01
│ norm(Q - Qe)            = 2.3225467541888396e+01
└ norm(Q - Qe) / norm(Qe) = 6.5540110824175756e-03
┌ Info: Configuration
│ ArrayType = Array
│ FT    = Float64
│ dims      = 2
└ splitting = (Nonlinear, Linear)
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 0)
│ dt        = 3.5842293906810039e-06
└ norm(Q₀)  = 3.5435113910177101e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 3.5433940589518702e+03
│ norm(Q) / norm(Q₀)      = 9.9996688819284241e-01
│ norm(Q) - norm(Q₀)      = -1.1733206583994615e-01
│ norm(Q - Qe)            = 2.3225467541886673e+01
└ norm(Q - Qe) / norm(Qe) = 6.5540110824170890e-03
Test Summary:                                                                                                                          | Test Summary:                                                                                                                          | Test Summary:                                                                                                                          | Pass  Pass  Pass  Total
Total
Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/Euler/isentropicvortex_imex.jl | /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/Euler/isentropicvortex_imex.jl |    2     2      2
    2
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/Euler/isentropicvortex_imex.jl |    2      2

out_10192344

+ hostname
hpc-90-18.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192344_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192344_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/Euler/isentropicvortex_multirate.jl
┌ Info: Configuration
│ ArrayType  = Array
│ FastMethod = SSPRK33ShuOsher
│ FT     = Float64
└ dims       = 2
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 0)
│ slow_dt   = 2.7777777777777782e-05
│ fast_dt   = 9.2592592592592608e-06
└ norm(Q₀)  = 3.5435113910177101e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 3.5433927502798692e+03
│ norm(Q) / norm(Q₀)      = 9.9996651887781662e-01
│ norm(Q) - norm(Q₀)      = -1.1864073784090579e-01
│ norm(Q - Qe)            = 2.3222373077799016e+01
└ norm(Q - Qe) / norm(Qe) = 6.5531378534115564e-03
┌ Info: Configuration
│ ArrayType  = Array
│ FastMethod = ARK2GiraldoKellyConstantinescu
│ FT     = Float64
└ dims       = 2
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 0)
│ slow_dt   = 2.7777777777777782e-05
│ fast_dt   = 9.2592592592592608e-06
└ norm(Q₀)  = 3.5435113910177101e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 3.5433931034199986e+03
│ norm(Q) / norm(Q₀)      = 9.9996661853606250e-01
│ norm(Q) - norm(Q₀)      = -1.1828759771151454e-01
│ norm(Q - Qe)            = 2.3245216640119065e+01
└ norm(Q - Qe) / norm(Qe) = 6.5595840943898044e-03
Test Summary:                                                                                                                               | Pass  Total
Test Summary:                                                                                                                               | Pass  Test Summary:                                                                                                                               | Pass  Total
Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/Euler/isentropicvortex_multirate.jl |    2      2
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/Euler/isentropicvortex_multirate.jl |    2  /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/Euler/isentropicvortex_multirate.jl |    2      2
    2

out_10192345

+ hostname
hpc-90-18.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192345_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192345_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/Euler/isentropicvortex_mrigark.jl
┌ Info: Configuration
│ ArrayType  = Array
│ mrigark_method = MRIGARKERK33aSandu
│ FT     = Float64
└ dims       = 2
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 0)
│ slow_dt   = 7.1684587813620078e-06
│ fast_dt   = 2.3894862604540028e-06
└ norm(Q₀)  = 3.5435113910177101e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 3.5433917654007246e+03
│ norm(Q) / norm(Q₀)      = 9.9996624093906150e-01
│ norm(Q) - norm(Q₀)      = -1.1962561698555874e-01
│ norm(Q - Qe)            = 2.3357934866480768e+01
└ norm(Q - Qe) / norm(Qe) = 6.5913921302639010e-03
┌ Info: Configuration
│ ArrayType  = Array
│ mrigark_method = MRIGARKERK45aSandu
│ FT     = Float64
└ dims       = 2
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 0)
│ slow_dt   = 7.1684587813620078e-06
│ fast_dt   = 2.3894862604540028e-06
└ norm(Q₀)  = 3.5435113910177101e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 3.5433940921715830e+03
│ norm(Q) / norm(Q₀)      = 9.9996689756764312e-01
│ norm(Q) - norm(Q₀)      = -1.1729884612714159e-01
│ norm(Q - Qe)            = 2.3207510164245548e+01
└ norm(Q - Qe) / norm(Qe) = 6.5489436773429471e-03
Test Summary:                                                                                                                             | Pass  Test Summary:                                                                                                                             | Pass  Total
Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/Euler/isentropicvortex_mrigark.jl |    2      2
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/Euler/isentropicvortex_mrigark.jl |    2      2
Test Summary:                                                                                                                             | Pass  Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/Euler/isentropicvortex_mrigark.jl |    2      2

out_10192346

+ hostname
hpc-90-18.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192346_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192346_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/Euler/isentropicvortex_mrigark_implicit.jl
┌ Info: Configuration
│ ArrayType      = Array
│ mrigark_method = MRIGARKIRK21aSandu
│ FT             = Float64
└ dims           = 2
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 0)
│ dt        = 7.1994240460763143e-07
└ norm(Q₀)  = 3.5435113910177101e+03
┌ Info: Update
│ simtime = 7.1994240460763143e-07
│ runtime = 00:01:08
└ norm(Q) = 3.5435114340963883e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 3.5433941432216739e+03
│ norm(Q) / norm(Q₀)      = 9.9996691197428245e-01
│ norm(Q) - norm(Q₀)      = -1.1724779603628122e-01
│ norm(Q - Qe)            = 2.3236071337680226e+01
└ norm(Q - Qe) / norm(Qe) = 6.5570033750381651e-03
┌ Info: Configuration
│ ArrayType      = Array
│ mrigark_method = MRIGARKESDIRK34aSandu
│ FT             = Float64
└ dims           = 2
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 0)
│ dt        = 7.1994240460763143e-07
└ norm(Q₀)  = 3.5435113910177101e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 3.5433941177190127e+03
│ norm(Q) / norm(Q₀)      = 9.9996690477727979e-01
│ norm(Q) - norm(Q₀)      = -1.1727329869745517e-01
│ norm(Q - Qe)            = 2.3235626679100864e+01
└ norm(Q - Qe) / norm(Qe) = 6.5568778965197343e-03
Test Summary:                                                                                                                                      | Pass  Total
Test Summary:                                                                                                                                      | Test Summary:                                                                                                                                      | Pass  Pass  /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/Euler/isentropicvortex_mrigark_implicit.jl |    2      2
Total
Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/Euler/isentropicvortex_mrigark_implicit.jl |    2      2
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/Euler/isentropicvortex_mrigark_implicit.jl |    2      2

out_10192347

+ hostname
hpc-90-18.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192347_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192347_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/Euler/acousticwave_1d_imex.jl
┌ Info: Starting
│ ArrayType       = Array
│ FT              = Float32
│ polynomialorder = 5
│ numelem_horz    = 10
│ numelem_vert    = 5
│ dt              = 1.0000000000000000e+02
└ norm(Q₀)        = 9.5070132895744000e+13
┌ Info: Update
│ simtime = 3.4000000000000000e+03
│ runtime = 00:01:00
└ norm(Q) = 9.5069973512192000e+13
┌ Info: Finished
│ norm(Q)                 = 9.5069965123584000e+13
│ norm(Q) / norm(Q₀)      = 9.9999821186065674e-01
└ norm(Q) - norm(Q₀)      = -1.6777216000000000e+08
┌ Info: Starting
│ ArrayType       = Array
│ FT              = Float32
│ polynomialorder = 5
│ numelem_horz    = 10
│ numelem_vert    = 5
│ dt              = 1.0000000000000000e+02
└ norm(Q₀)        = 9.5070132895744000e+13
┌ Info: Finished
│ norm(Q)                 = 9.5069981900800000e+13
│ norm(Q) / norm(Q₀)      = 9.9999839067459106e-01
└ norm(Q) - norm(Q₀)      = -1.5099494400000000e+08
┌ Info: Starting
│ ArrayType       = Array
│ FT              = Float64
│ polynomialorder = 5
│ numelem_horz    = 10
│ numelem_vert    = 5
│ dt              = 1.0000000000000000e+02
└ norm(Q₀)        = 9.5073561639227109e+13
┌ Info: Finished
│ norm(Q)                 = 9.5073452874026609e+13
│ norm(Q) / norm(Q₀)      = 9.9999885598899818e-01
└ norm(Q) - norm(Q₀)      = -1.0876520050000000e+08
┌ Info: Starting
│ ArrayType       = Array
│ FT              = Float64
│ polynomialorder = 5
│ numelem_horz    = 10
│ numelem_vert    = 5
│ dt              = 1.0000000000000000e+02
└ norm(Q₀)        = 9.5073561639227109e+13
┌ Info: Finished
│ norm(Q)                 = 9.5073452874026578e+13
│ norm(Q) / norm(Q₀)      = 9.9999885598899785e-01
└ norm(Q) - norm(Q₀)      = -1.0876520053125000e+08

out_10192348

+ hostname
hpc-90-18.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192348_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192348_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/Euler/acousticwave_mrigark.jl
┌ Info: Starting
│ ArrayType       = Array
│ FT              = Float64
│ polynomialorder = 5
│ numelem_horz    = 10
│ numelem_vert    = 5
│ acoustic dt     = 6.7670313998798237e-01
│ remainder_dt    = 3.3834586466165413e+00
└ norm(Q₀)        = 9.5073561639227109e+13
┌ Info: Update
│ simtime = 2.0300751879699249e+01
│ runtime = 00:01:00
└ norm(Q) = 9.5073402385126359e+13
┌ Info: Update
│ simtime = 1.7255639097744356e+02
│ runtime = 00:02:02
└ norm(Q) = 9.5073558462836391e+13
┌ Info: Update
│ simtime = 3.3157894736842059e+02
│ runtime = 00:03:03
└ norm(Q) = 9.5073436352200344e+13
┌ Info: Update
│ simtime = 4.8383458646616452e+02
│ runtime = 00:04:04
└ norm(Q) = 9.5073433217799719e+13
┌ Info: Update
│ simtime = 6.3609022556390846e+02
│ runtime = 00:05:04
└ norm(Q) = 9.5073485412005563e+13
┌ Info: Update
│ simtime = 7.8834586466165240e+02
│ runtime = 00:06:04
└ norm(Q) = 9.5073441768808016e+13
┌ Info: Update
│ simtime = 9.4060150375939634e+02
│ runtime = 00:07:05
└ norm(Q) = 9.5073416450525672e+13
┌ Info: Update
│ simtime = 1.0928571428571427e+03
│ runtime = 00:08:05
└ norm(Q) = 9.5073485095511016e+13
┌ Info: Update
│ simtime = 1.2451127819548917e+03
│ runtime = 00:09:06
└ norm(Q) = 9.5073451738690438e+13
┌ Info: Update
│ simtime = 1.3973684210526408e+03
│ runtime = 00:10:06
└ norm(Q) = 9.5073406602004656e+13
┌ Info: Update
│ simtime = 1.5530075187970065e+03
│ runtime = 00:11:07
└ norm(Q) = 9.5073441783415297e+13
┌ Info: Update
│ simtime = 1.7086466165413722e+03
│ runtime = 00:12:08
└ norm(Q) = 9.5073431468540609e+13
┌ Info: Update
│ simtime = 1.8642857142857379e+03
│ runtime = 00:13:09
└ norm(Q) = 9.5073538288378031e+13
┌ Info: Update
│ simtime = 2.0199248120301036e+03
│ runtime = 00:14:10
└ norm(Q) = 9.5073364701826656e+13
┌ Info: Update
│ simtime = 2.1755639097744606e+03
│ runtime = 00:15:11
└ norm(Q) = 9.5073438966291109e+13
┌ Info: Update
│ simtime = 2.3312030075188159e+03
│ runtime = 00:16:12
└ norm(Q) = 9.5073483233982813e+13
┌ Info: Update
│ simtime = 2.4902255639097875e+03
│ runtime = 00:17:13
└ norm(Q) = 9.5073514359890188e+13
┌ Info: Update
│ simtime = 2.6458646616541428e+03
│ runtime = 00:18:14
└ norm(Q) = 9.5073427259503594e+13
┌ Info: Update
│ simtime = 2.8015037593984980e+03
│ runtime = 00:19:15
└ norm(Q) = 9.5073369616804406e+13
┌ Info: Update
│ simtime = 2.9571428571428532e+03
│ runtime = 00:20:16
└ norm(Q) = 9.5073514499925672e+13
┌ Info: Update
│ simtime = 3.1127819548872085e+03
│ runtime = 00:21:17
└ norm(Q) = 9.5073436102596531e+13
┌ Info: Update
│ simtime = 3.2684210526315637e+03
│ runtime = 00:22:18
└ norm(Q) = 9.5073467484045063e+13
┌ Info: Update
│ simtime = 3.4240601503759190e+03
│ runtime = 00:23:19
└ norm(Q) = 9.5073370414125750e+13
┌ Info: Update
│ simtime = 3.5796992481202742e+03
│ runtime = 00:24:19
└ norm(Q) = 9.5073488689584047e+13
┌ Info: Finished
│ norm(Q)                 = 9.5073337896207594e+13
│ norm(Q) / norm(Q₀)      = 9.9999764663261104e-01
└ norm(Q) - norm(Q₀)      = -2.2374301951562500e+08
┌ Info: Starting
│ ArrayType       = Array
│ FT              = Float64
│ polynomialorder = 5
│ numelem_horz    = 10
│ numelem_vert    = 5
│ acoustic dt     = 1.3333333333333334e+02
│ remainder_dt    = 2.3494467606676028e+02
└ norm(Q₀)        = 9.5073561639227109e+13
┌ Info: Update
│ simtime = 2.6666666666666670e+03
│ runtime = 00:01:01
└ norm(Q) = 9.5073455529521969e+13
┌ Info: Finished
│ norm(Q)                 = 9.5073455097536063e+13
│ norm(Q) / norm(Q₀)      = 9.9999887937625132e-01
└ norm(Q) - norm(Q₀)      = -1.0654169104687500e+08
Test Summary: | Test Summary: | Test Summary: | Pass  Pass  Pass  Total
Total
Total
acoustic wave | acoustic wave | acoustic wave |    2     2     2      2
    2
    2

out_10192349

+ hostname
hpc-90-18.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192349_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192349_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/compressible_Navier_Stokes/mms_bc_atmos.jl
[ Info: (Array, Float64, 2, 400, 0.0025)
┌ Info: Starting
└ norm(Q₀) = 1.4718488046151103e+02
┌ Info: Finished
│ norm(Q)                 = 1.4749573751578203e+02
│ norm(Q) / norm(Q₀)      = 1.0021120175747420e+00
│ norm(Q) - norm(Q₀)      = 3.1085705427099697e-01
│ norm(Q - Qe)            = 1.6931876910307275e-01
└ norm(Q - Qe) / norm(Qe) = 1.1479958671721494e-03
[ Info: (Array, Float64, 3, 800, 0.00125)
┌ Info: Starting
└ norm(Q₀) = 1.1341641833287579e+02
┌ Info: Update
│ simtime = 5.3874999999999407e-01
│ runtime = 00:01:00
└ norm(Q) = 1.1337591596252379e+02
┌ Info: Finished
│ norm(Q)                 = 1.1338036285141952e+02
│ norm(Q) / norm(Q₀)      = 9.9968209645493789e-01
│ norm(Q) - norm(Q₀)      = -3.6055481456273242e-02
│ norm(Q - Qe)            = 3.3983777728925427e-02
└ norm(Q - Qe) / norm(Qe) = 2.9973197445392285e-04
Test Summary: | Test Summary: | Test Summary: | Pass  Pass  Pass  TotalTotalTotal


mms_bc_atmos  | mms_bc_atmos  | mms_bc_atmos  |    4     4     4      4
    4
    4

out_10192350

+ hostname
hpc-90-18.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192350_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192350_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/compressible_Navier_Stokes/mms_bc_dgmodel.jl
[ Info: (Array, Float64, 2)
┌ Info: Starting
└ norm(Q₀) = 1.4718488046151103e+02
┌ Info: Finished
│ norm(Q)                 = 1.4749698909866967e+02
│ norm(Q) / norm(Q₀)      = 1.0021205210493089e+00
│ norm(Q) - norm(Q₀)      = 3.1210863715864434e-01
│ norm(Q - Qe)            = 1.6694721292985901e-01
└ norm(Q - Qe) / norm(Qe) = 1.1319165116462501e-03
[ Info: (Array, Float64, 3)
┌ Info: Starting
└ norm(Q₀) = 1.1341641833287579e+02
┌ Info: Update
│ simtime = 6.9124999999999082e-01
│ runtime = 00:01:00
└ norm(Q) = 1.1337429862411264e+02
┌ Info: Finished
│ norm(Q)                 = 1.1338058691327828e+02
│ norm(Q) / norm(Q₀)      = 9.9968407202304388e-01
│ norm(Q) - norm(Q₀)      = -3.5831419597514014e-02
│ norm(Q - Qe)            = 3.3672443923201310e-02
└ norm(Q - Qe) / norm(Qe) = 2.9698605559086140e-04
Test Summary:  | Test Summary:  | Pass  Pass  TotalTotal

Test Summary:  | Pass  Total
mms_bc_dgmodel |    2      2
mms_bc_dgmodel |    2      2
mms_bc_dgmodel |    2      2

out_10192351

+ hostname
hpc-22-12.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192351_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192351_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/compressible_Navier_Stokes/density_current_model.jl
┌ Info: Starting
│ norm(Q₀) = 4.3975577600000000e+09
│ ArrayType = Array
└ FloatType = Float32
┌ Info: Update
│ simtime = 9.9999997764825821e-03
│ runtime = 00:00:52
└ norm(Q) = 4.3975577600000000e+09
┌ Info: Update
│ simtime = 2.9999999329447746e-02
│ runtime = 00:01:03
└ norm(Q) = 4.3975572480000000e+09
┌ Info: Update
│ simtime = 4.9999997019767761e-02
│ runtime = 00:01:14
└ norm(Q) = 4.3975572480000000e+09
┌ Info: Update
│ simtime = 6.9999992847442627e-02
│ runtime = 00:01:24
└ norm(Q) = 4.3975567360000000e+09
┌ Info: Update
│ simtime = 9.9999986588954926e-02
│ runtime = 00:01:39
└ norm(Q) = 4.3975562240000000e+09
┌ Info: Finished
│ norm(Q)                 = 4.3975562240000000e+09
│ norm(Q) / norm(Q₀)      = 9.9999964237213135e-01
│ norm(Q) - norm(Q₀)      = -1.5360000000000000e+03
│ norm(Q - Qe)            = 3.5917015625000000e+04
└ norm(Q - Qe) / norm(Qe) = 8.1674916145857424e-06
┌ Info: Starting
│ norm(Q₀) = 4.3981206422004299e+09
│ ArrayType = Array
└ FloatType = Float64
┌ Info: Update
│ simtime = 1.0000000000000000e-02
│ runtime = 00:00:33
└ norm(Q) = 4.3981206294029999e+09
┌ Info: Update
│ simtime = 2.9999999999999999e-02
│ runtime = 00:00:45
└ norm(Q) = 4.3981205270634966e+09
┌ Info: Update
│ simtime = 5.0000000000000003e-02
│ runtime = 00:00:57
└ norm(Q) = 4.3981203224109554e+09
┌ Info: Update
│ simtime = 7.0000000000000007e-02
│ runtime = 00:01:10
└ norm(Q) = 4.3981200154792986e+09
┌ Info: Update
│ simtime = 8.9999999999999997e-02
│ runtime = 00:01:22
└ norm(Q) = 4.3981196063201733e+09
┌ Info: Update
│ simtime = 1.0000000000000001e-01
│ runtime = 00:01:35
└ norm(Q) = 4.3981193634261818e+09
┌ Info: Finished
│ norm(Q)                 = 4.3981193634261818e+09
│ norm(Q) / norm(Q₀)      = 9.9999970924529991e-01
│ norm(Q) - norm(Q₀)      = -1.2787742481231689e+03
│ norm(Q - Qe)            = 3.5863022189957599e+04
└ norm(Q - Qe) / norm(Qe) = 8.1541697255523485e-06

out_10192352

+ hostname
hpc-22-12.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192352_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192352_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/advection_diffusion/direction_splitting_advection_diffusion.jl --fix-rng-seed
Test Summary:                                                                                                                                                          | Test Summary:                                                                                                                                                          | Pass  Pass  Test Summary:                                                                                                                                                          | Pass  Total
Total
Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/direction_splitting_advection_diffusion.jl | /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/direction_splitting_advection_diffusion.jl |  162   162    162
  162
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/direction_splitting_advection_diffusion.jl |  162    162

out_10192353

+ hostname
hpc-22-12.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192353_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192353_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/conservation/sphere.jl --fix-rng-seed
[ Info: (Array, Float64)
┌ Info: Starting
│ norm(Q₀) = 4.4181152030340334e+00
└ sum(Q₀) = 2.9255250415427088e+01
┌ Info: Finished
│ norm(Q)            = 4.4304590011362377e+00
│ norm(Q) / norm(Q₀) = 1.0027939058931119e+00
│ norm(Q) - norm(Q₀) = 1.2343798102204318e-02
│ max mass loss      = 0.0000000000000000e+00
│ max mass gain      = 0.0000000000000000e+00
└ initial mass       = 2.9255250415427088e+01
Test Summary:                                                                                                                  | Pass  Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/conservation/sphere.jl |    1  Test Summary:                                                                                                                  |     1
Pass  Test Summary:                                                                                                                  | Total
Pass  Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/conservation/sphere.jl |    1      1
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/conservation/sphere.jl |    1      1
[ Info: (Array, Float32)
┌ Info: Starting
│ norm(Q₀) = 4.4181084632873535e+00
└ sum(Q₀) = 2.9255237579345703e+01
┌ Info: Finished
│ norm(Q)            = 4.4304566383361816e+00
│ norm(Q) / norm(Q₀) = 1.0027948617935181e+00
│ norm(Q) - norm(Q₀) = 1.2348175048828125e-02
│ max mass loss      = 0.0000000000000000e+00
│ max mass gain      = 0.0000000000000000e+00
└ initial mass       = 2.9255237579345703e+01
Test Summary:                                                                                                                  | Pass  Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/conservation/sphere.jl |    1      1
Test Summary:                                                                                                                  | Pass  Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/conservation/sphere.jl |    1      1
Test Summary:                                                                                                                  | Pass  Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/conservation/sphere.jl |    1      1

out_10192354

+ hostname
hpc-90-18.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192354_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192354_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/advection_diffusion/advection_sphere.jl
[ Info: (Array, Float64)
┌ Info: Starting
│ problem   = SolidBodyRotation
│ method    = LSRK144NiegemannDiehlBusch
│ time step = 7.1428571428571425e-02
└ norm(Q₀)  = 6.3810679373927848e-01
┌ Info: Finished
│ Δmass                   = 7.0103401895551229e-15
│ norm(Q)                 = 6.0927759884214716e-01
│ norm(Q) / norm(Q₀)      = 9.5482073662279399e-01
│ norm(Q) - norm(Q₀)      = -2.8829194897131316e-02
│ norm(Q - Qe)            = 1.3199024557832187e-01
└ norm(Q - Qe) / norm(Qe) = 2.0684663895343394e-01
[ Info: 
[ Info: (Array, Float64)
┌ Info: Starting
│ problem   = SolidBodyRotation
│ method    = SSPRK33ShuOsher
│ time step = 1.5151515151515152e-02
└ norm(Q₀)  = 6.3810679373927848e-01
┌ Info: Finished
│ Δmass                   = 2.6639292720309469e-15
│ norm(Q)                 = 6.0758654980256122e-01
│ norm(Q) / norm(Q₀)      = 9.5217063313513728e-01
│ norm(Q) - norm(Q₀)      = -3.0520243936717262e-02
│ norm(Q - Qe)            = 1.1055145388897815e-01
└ norm(Q - Qe) / norm(Qe) = 1.7324914101157168e-01
[ Info: 
[ Info: (Array, Float64)
┌ Info: Starting
│ problem   = SolidBodyRotation
│ method    = SSPRK34SpiteriRuuth
│ time step = 2.2727272727272728e-02
└ norm(Q₀)  = 6.3810679373927848e-01
┌ Info: Finished
│ Δmass                   = 1.9628952530754343e-15
│ norm(Q)                 = 5.9932132891912659e-01
│ norm(Q) / norm(Q₀)      = 9.3921790960276297e-01
│ norm(Q) - norm(Q₀)      = -3.8785464820151883e-02
│ norm(Q - Qe)            = 1.2641959922001447e-01
└ norm(Q - Qe) / norm(Qe) = 1.9811667962223226e-01
[ Info: 
[ Info: (Array, Float64)
┌ Info: Starting
│ problem   = ReversingDeformationalFlow
│ method    = LSRK144NiegemannDiehlBusch
│ time step = 1.6129032258064516e-01
└ norm(Q₀)  = 7.5338057383963131e-01
┌ Info: Finished
│ Δmass                   = 1.8614343182663521e-16
│ norm(Q)                 = 5.1909887102433183e-01
│ norm(Q) / norm(Q₀)      = 6.8902608993317371e-01
│ norm(Q) - norm(Q₀)      = -2.3428170281529948e-01
│ norm(Q - Qe)            = 5.5387951598735496e-01
└ norm(Q - Qe) / norm(Qe) = 7.3519219265833735e-01
[ Info: 
[ Info: (Array, Float64)
┌ Info: Starting
│ problem   = ReversingDeformationalFlow
│ method    = SSPRK33ShuOsher
│ time step = 3.3112582781456956e-02
└ norm(Q₀)  = 7.5338057383963131e-01
┌ Info: Finished
│ Δmass                   = 5.5843029547990565e-15
│ norm(Q)                 = 5.1716060651384743e-01
│ norm(Q) / norm(Q₀)      = 6.8645333377541140e-01
│ norm(Q) - norm(Q₀)      = -2.3621996732578388e-01
│ norm(Q - Qe)            = 5.5353962032596327e-01
└ norm(Q - Qe) / norm(Qe) = 7.3474103201895502e-01
[ Info: 
[ Info: (Array, Float64)
┌ Info: Starting
│ problem   = ReversingDeformationalFlow
│ method    = SSPRK34SpiteriRuuth
│ time step = 4.9504950495049507e-02
└ norm(Q₀)  = 7.5338057383963131e-01
┌ Info: Finished
│ Δmass                   = 5.0258726593191508e-15
│ norm(Q)                 = 5.1534369870022134e-01
│ norm(Q) / norm(Q₀)      = 6.8404166047679404e-01
│ norm(Q) - norm(Q₀)      = -2.3803687513940996e-01
│ norm(Q - Qe)            = 5.5404045660832868e-01
└ norm(Q - Qe) / norm(Qe) = 7.3540581725467313e-01
[ Info: 
Test Summary:                                                                                                                                   | Pass  Test Summary:                                                                                                                                   | Pass  Total
Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/advection_sphere.jl |   12     12
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/advection_sphere.jl |   12     12

out_10192355

+ hostname
hpc-89-06.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192355_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192355_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Driver/gcm_driver_test.jl
┌ Info: Model composition
│     param_set = EarthParameterSet()
│     orientation = SphericalOrientation()
│     ref_state = HydrostaticState{DecayingTemperatureProfile{Float64},Float64}(DecayingTemperatureProfile{Float64}(300.0, 300.0, 8776.832091407436), 0.0)
│     turbulence = ConstantViscosityWithDivergence{Float64}(0.0)
│     hyperdiffusion = NoHyperDiffusion()
│     moisture = DryModel()
│     precipitation = NoPrecipitation()
│     radiation = NoRadiation()
│     source = Gravity()
│     tracers = NoTracers()
│     boundarycondition = AtmosBC{Impenetrable{FreeSlip},Insulating,Impermeable,ImpermeableTracer}(Impenetrable{FreeSlip}(FreeSlip()), Insulating(), Impermeable(), ImpermeableTracer())
│     init_state_conservative = AcousticWaveSetup{Float64}(10000.0, 300.0, 3.0, 100.0, 1)
└     data_config = nothing
┌ Info: Establishing Atmos GCM configuration for GCM Driver test
│     precision        = Float64
│     polynomial order = 4
│     #horiz elems     = 4
│     #vert elems      = 6
│     domain height    = 1.00e+04 m
│     MPI ranks        = 1
│     min(Δ_horz)      = 305681.05 m
└     min(Δ_vert)      = 287.79 m
[ Info: Initializing GCM Driver test
┌ Info: Starting GCM Driver test
│     dt              = 6.00000e+02
│     timeend         =  1800.00
│     number of steps = 3
└     norm(Q)         = 9.5073564673382391e+13
┌ Info: Update
│     simtime =   600.00 /  1800.00
│     runtime = 00:01:09
└     norm(Q) = 9.5073453551766703e+13
┌ Info: Update
│     simtime =  1800.00 /  1800.00
│     runtime = 00:02:14
└     norm(Q) = 9.5073454833172078e+13
┌ Info: Finished
│     norm(Q)            = 9.5073454833172078e+13
│     norm(Q) / norm(Q₀) = 9.9999884468189770e-01
└     norm(Q) - norm(Q₀) = -1.0984021031250000e+08

out_10192356

+ hostname
hpc-91-10.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192356_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192356_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/SystemSolvers/poisson.jl
[ Info: (Array, Float64, 1, 2)
┌ Info: Finished
│ error = 5.0540243605694528e-02
└ iters = 49
[ Info: (Array, Float64, 1, 3)
┌ Info: Finished
│ error = 1.4957957659447905e-02
└ iters = 39
[ Info: (Array, Float64, 2, 2)
┌ Info: Finished
│ error = 5.0540243614350271e-02
└ iters = 38
[ Info: (Array, Float64, 2, 3)
┌ Info: Finished
│ error = 1.4957957667511290e-02
└ iters = 35

out_10192357

+ hostname
hpc-89-25.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192357_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192357_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/SystemSolvers/columnwiselu.jl

out_10192358

+ hostname
hpc-22-12.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192358_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192358_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/SystemSolvers/bandedsystem.jl
Test Summary:                                                                                                                              | Pass  Test Summary:                                                                                                                              | Test Summary:                                                                                                                              | Pass  Total
Total
Pass  /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/SystemSolvers/bandedsystem.jl DGModel matrix | Total
  72     72
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/SystemSolvers/bandedsystem.jl DGModel matrix |   72     72
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/SystemSolvers/bandedsystem.jl DGModel matrix |   72     72

out_10192359

+ hostname
hpc-22-12.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192359_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192359_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/Mesh/interpolation.jl
Test Summary:       | Test Summary:       | Pass  Test Summary:       | Total
Pass  Pass  Total
Total
Interpolation tests |    4      4
Interpolation tests |    4  Interpolation tests |     4
   4      4

out_10192360

+ hostname
hpc-90-09.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192360_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192360_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Ocean/ShallowWater/GyreDriver.jl
[ Info: running Ne 10 and N 4 with
┌ Info: 
└  dt = 63.102347
┌ Info: Finished
└ error = 1.6068814534513961e-03

out_10192361

+ hostname
hpc-89-18.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192361_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192361_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Ocean/ShallowWater/2D_hydrostatic_spindown.jl
┌ Info: doing VTK output
└   outprefix = "vtk_shallow_spindown/fast/mpirank0000_step0000"
# SC Start: creating state check callback
# SC Creating state check callback labeled "2D state" for symbols
# SC η
# SC U
# SC Finish: creating state check callback
┌ Info: Starting
│ norm(Q₀) = 9.9999999999999790e+05
└ ArrayType = Array
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000000|    2D state|        η | -9.570004283478e-01|  9.572452611952e-01| -2.223998762929e-16|  6.772836616366e-01|
# SC 0000000|    2D state|     U[1] | -1.792532800489e+01|  1.792532800489e+01|  4.401704695178e-15|  1.274390739934e+01|
# SC 0000000|    2D state|     U[2] | -4.772516078078e-13|  4.391217053759e-13|  8.848192699530e-15|  8.803775620991e-14|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000024|    2D state|        η | -8.244119230668e-01|  8.244339910020e-01|  1.296740492762e-17|  5.834075469111e-01|
# SC 0000024|    2D state|     U[1] | -3.512801331247e+01|  3.512801331247e+01| -9.709283377290e-15|  2.497982254591e+01|
# SC 0000024|    2D state|     U[2] | -7.410666166209e-13|  7.255267079459e-13| -1.182685111407e-14|  1.801707136582e-13|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000048|    2D state|        η | -6.144793034641e-01|  6.144946933273e-01| -3.327116360197e-16|  4.348410147654e-01|
# SC 0000048|    2D state|     U[1] | -4.901995498250e+01|  4.901995498250e+01|  1.367816970799e-14|  3.485484383011e+01|
# SC 0000048|    2D state|     U[2] | -9.500318323447e-13|  1.203941377544e-12|  2.088137013298e-14|  2.591934639704e-13|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000072|    2D state|        η | -3.472229442154e-01|  3.471049320739e-01| -8.331113576787e-17|  2.456489143135e-01|
# SC 0000072|    2D state|     U[1] | -5.827752340806e+01|  5.827752340806e+01| -1.719880349249e-14|  4.144038160451e+01|
# SC 0000072|    2D state|     U[2] | -1.747596833435e-12|  1.398020011498e-12| -2.940465101593e-14|  3.616986091314e-13|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000096|    2D state|        η | -4.763924420977e-02|  4.752491380495e-02| -1.073807709417e-16|  3.369680773001e-02|
# SC 0000096|    2D state|     U[1] | -6.205731222939e+01|  6.205731222939e+01|  1.796026105616e-14|  4.412697895910e+01|
# SC 0000096|    2D state|     U[2] | -1.567957523617e-12|  2.167993103752e-12|  2.515572152539e-14|  4.066302497418e-13|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000120|    2D state|        η | -2.560178592636e-01|  2.559630305845e-01| -8.988365607365e-17|  1.811621769315e-01|
# SC 0000120|    2D state|     U[1] | -6.000444094081e+01|  6.000444094081e+01| -2.163675349998e-14|  4.266661476095e+01|
# SC 0000120|    2D state|     U[2] | -2.254836126721e-12|  1.675663676571e-12| -2.889252493813e-14|  4.300300322781e-13|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000144|    2D state|        η | -5.351443527053e-01|  5.351153344761e-01|  4.494182803683e-17|  3.786780380728e-01|
# SC 0000144|    2D state|     U[1] | -5.231521203845e+01|  5.231521203845e+01|  1.461591958574e-14|  3.720030613681e+01|
# SC 0000144|    2D state|     U[2] | -1.634289270883e-12|  2.233235546136e-12|  2.636325998057e-14|  4.074640577261e-13|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000168|    2D state|        η | -7.635599856548e-01|  7.636623115743e-01| -2.584599201327e-16|  5.403552154437e-01|
# SC 0000168|    2D state|     U[1] | -3.973138903400e+01|  3.973138903400e+01| -1.244694902702e-14|  2.825071776819e+01|
# SC 0000168|    2D state|     U[2] | -2.033758708095e-12|  1.441571646608e-12| -1.681507464642e-14|  3.523050408050e-13|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000192|    2D state|        η | -9.200822132533e-01|  9.200384241042e-01| -7.371880883511e-17|  6.510808852866e-01|
# SC 0000192|    2D state|     U[1] | -2.342978002845e+01|  2.342978002845e+01|  9.401007750043e-15|  1.666038251575e+01|
# SC 0000192|    2D state|     U[2] | -1.101681432640e-12|  1.537008325302e-12|  1.000031119873e-14|  2.585927167164e-13|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000216|    2D state|        η | -9.898734969061e-01|  9.899710495454e-01| -3.344879928591e-16|  7.005041344611e-01|
# SC 0000216|    2D state|     U[1] | -4.953768384280e+00|  4.953768384280e+00| -1.794265083732e-15|  3.522268485124e+00|
# SC 0000216|    2D state|     U[2] | -1.014211395524e-12|  6.428588136244e-13| -7.664557986271e-15|  1.491627971341e-13|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000240|    2D state|        η | -9.667308999352e-01|  9.666829448740e-01|  5.524469770535e-17|  6.840902369194e-01|
# SC 0000240|    2D state|     U[1] | -1.396139302125e+01|  1.396139302125e+01| -4.543734832829e-15|  9.927994563815e+00|
# SC 0000240|    2D state|     U[2] | -3.615464441996e-13|  2.848411729338e-13| -7.999040308422e-15|  6.845633380951e-14|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000264|    2D state|        η | -8.527228636775e-01|  8.528281863846e-01| -2.279065824951e-16|  6.034549743954e-01|
# SC 0000264|    2D state|     U[1] | -3.154026150718e+01|  3.154026150718e+01|  4.236663797563e-15|  2.242694051910e+01|
# SC 0000264|    2D state|     U[2] | -5.136586033612e-13|  5.375975735033e-13|  1.329040550200e-14|  1.386173431094e-13|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
2D error = 0.00011327920483882387
# BEGIN SCPRINT
# varr - reference values (from reference run)    
# parr - digits match precision (hand edit as needed) 
#
# [
#  [ MPIStateArray Name, Field Name, Maximum, Minimum, Mean, Standard Deviation ],
#  [         :                :          :        :      :          :           ],
# ]
varr = [
 [ "2D state",     :η, -8.52722863677453979392e-01,  8.52828186384614772564e-01, -2.27906582495052129964e-16,  6.03454974395413956678e-01 ],
 [ "2D state", "U[1]", -3.15402615071803609226e+01,  3.15402615071800944690e+01,  4.23666379756326684746e-15,  2.24269405191038053715e+01 ],
 [ "2D state", "U[2]", -5.13658603361190211547e-13,  5.37597573503267133351e-13,  1.32904055020046349575e-14,  1.38617343109420055294e-13 ],
]
parr = [
 [ "2D state",     :η,    16,    16,    16,    16 ],
 [ "2D state", "U[1]",    16,    16,    16,    16 ],
 [ "2D state", "U[2]",    16,    16,    16,    16 ],
]
# END SCPRINT
# SC +++++++++++ClimateMachine StateCheck ref val check start+++++++++++++++++
# SC "N( )" bracketing indicates field failed to match      
# SC "P="  row pass count      
# SC "F="  row pass count      
# SC "NA=" row not checked count      
# SC 
# SC        Label         Field      min()      max()     mean()      std() 
# SC     2D state,            η,        15,        15,         0,        15 :: P=5, F=0, NA=1
# SC     2D state,         U[1],        15,        15,         0,        15 :: P=5, F=0, NA=1
# SC     2D state,         U[2],         0,         0,         0,         0 :: P=2, F=0, NA=4
# SC +++++++++++ClimateMachine StateCheck ref val check end+++++++++++++++++
Test Summary:                                                                                                                      | Pass  Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Ocean/ShallowWater/2D_hydrostatic_spindown.jl |    2      2

out_10192362

+ hostname
hpc-89-19.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192362_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192362_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Ocean/HydrostaticBoussinesq/3D_hydrostatic_spindown.jl
[ Info: Initializing hydrostatic_spindown
# SC Start: creating state check callback
# SC Creating state check callback labeled "state" for symbols
# SC u
# SC η
# SC θ
# SC Creating state check callback labeled "aux" for symbols
# SC y
# SC w
# SC pkin
# SC wz0
# SC Creating state check callback labeled "gradient" for symbols
# SC ν∇u
# SC κ∇θ
# SC Finish: creating state check callback
┌ Info: Starting hydrostatic_spindown
│     dt              = 1.20000e+02
│     timeend         = 86400.00
│     number of steps = 720
└     norm(Q)         = 1.7320537548398674e+07
┌ Info: Update
│     simtime = 19080.00 / 86400.00
│     runtime = 00:01:00
└     norm(Q) = 1.0946054997300336e+07
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000000|       state|     u[1] | -1.087951509677e+00|  1.087951509677e+00| -1.318767317571e-16|  4.926008335114e-01|
# SC 0000000|       state|     u[2] | -4.001370537643e-14|  2.998356888627e-14| -9.403109212613e-17|  3.865679625351e-15|
# SC 0000000|       state|        η | -6.145234312582e-01|  6.144952745945e-01|  5.338733899407e-15|  4.345142206664e-01|
# SC 0000000|       state|        θ |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000000|         aux|        y |  0.000000000000e+00|  1.000000000000e+06|  5.000000000000e+05|  2.927758774607e+05|
# SC 0000000|         aux|        w | -4.683964160146e-04|  4.685473240467e-04|  1.431077478742e-18|  1.868552330777e-04|
# SC 0000000|         aux|     pkin |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000000|         aux|      wz0 | -3.109753456934e-04|  3.108100054090e-04|  2.184918912462e-18|  2.198651094847e-04|
# SC 0000000|    gradient|   ν∇u[1] | -3.432942108233e-02|  3.434288004063e-02|  1.751487843649e-17|  1.547360652068e-02|
# SC 0000000|    gradient|   ν∇u[2] | -3.501821261551e-14|  2.896455062079e-14| -6.720215291169e-18|  1.824162861148e-15|
# SC 0000000|    gradient|   ν∇u[3] | -7.606685607826e-05|  7.606685607826e-05| -1.193489751472e-20|  3.811894706291e-05|
# SC 0000000|    gradient|   ν∇u[4] | -1.710575995383e-14|  1.534967722129e-14|  1.224962311321e-18|  1.310612500294e-15|
# SC 0000000|    gradient|   ν∇u[5] | -8.411865100937e-15|  9.931713890876e-15|  3.569871162584e-17|  1.000855376565e-15|
# SC 0000000|    gradient|   ν∇u[6] | -2.898609239633e-17|  1.809035240413e-17|  1.588334346827e-20|  2.140030369901e-18|
# SC 0000000|    gradient|    κ∇θ[1 | -0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000000|    gradient|    κ∇θ[2 | -0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000000|    gradient|    κ∇θ[3 | -0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000180|       state|     u[1] | -1.085847524443e+00|  1.085847524443e+00|  2.273736754432e-16|  4.819856489613e-01|
# SC 0000180|       state|     u[2] | -1.106616112876e-13|  1.247901648051e-13|  1.667852691391e-17|  9.051871640838e-15|
# SC 0000180|       state|        η | -2.560215899912e-01|  2.559366968685e-01|  1.239527591679e-14|  1.810174277921e-01|
# SC 0000180|       state|        θ |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000180|         aux|        y |  0.000000000000e+00|  1.000000000000e+06|  5.000000000000e+05|  2.927758774607e+05|
# SC 0000180|         aux|        w | -6.642488075190e-04|  6.642028953680e-04| -9.992007221626e-20|  2.854191070386e-04|
# SC 0000180|         aux|     pkin |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000180|         aux|      wz0 | -3.781878501380e-04|  3.783776483443e-04|  6.461498003318e-19|  2.674812779706e-04|
# SC 0000180|    gradient|   ν∇u[1] | -3.425251105146e-02|  3.423963460959e-02|  7.815970093361e-19|  1.513687104664e-02|
# SC 0000180|    gradient|   ν∇u[2] | -6.116861654793e-14|  5.695598415239e-14|  2.287408245595e-17|  3.062221521090e-15|
# SC 0000180|    gradient|   ν∇u[3] | -7.375828788387e-05|  7.375828788388e-05|  6.383782391595e-21|  3.697050602310e-05|
# SC 0000180|    gradient|   ν∇u[4] | -5.176059652753e-14|  4.365943631248e-14| -6.594837452693e-19|  3.135534051397e-15|
# SC 0000180|    gradient|   ν∇u[5] | -3.356872715390e-14|  2.337664069384e-14| -5.340838379631e-17|  2.480995060901e-15|
# SC 0000180|    gradient|   ν∇u[6] | -8.845906038638e-17|  1.355638502558e-16|  9.944446213746e-21|  8.007035066431e-18|
# SC 0000180|    gradient|    κ∇θ[1 | -0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000180|    gradient|    κ∇θ[2 | -0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000180|    gradient|    κ∇θ[3 | -0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000360|       state|     u[1] | -9.658959801489e-01|  9.658959801489e-01| -2.842170943040e-16|  4.576350104791e-01|
# SC 0000360|       state|     u[2] | -1.129250330047e-13|  8.701817407398e-14| -1.214316627899e-16|  7.836513004076e-15|
# SC 0000360|       state|        η | -9.200563582487e-01|  9.200557619157e-01|  1.867192622740e-14|  6.505752825496e-01|
# SC 0000360|       state|        θ |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000360|         aux|        y |  0.000000000000e+00|  1.000000000000e+06|  5.000000000000e+05|  2.927758774607e+05|
# SC 0000360|         aux|        w | -4.007038841854e-04|  4.009442568433e-04|  3.291256156501e-18|  1.687253907435e-04|
# SC 0000360|         aux|     pkin |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000360|         aux|      wz0 | -1.456785026757e-04|  1.458579822630e-04|  4.946043574705e-18|  1.030479734173e-04|
# SC 0000360|    gradient|   ν∇u[1] | -3.043131433466e-02|  3.044891231978e-02|  4.540368081507e-17|  1.437116731263e-02|
# SC 0000360|    gradient|   ν∇u[2] | -4.693414241050e-14|  4.179671771518e-14|  9.126071325761e-18|  2.600379035392e-15|
# SC 0000360|    gradient|   ν∇u[3] | -7.151371913685e-05|  7.151371913684e-05| -1.609823385706e-20|  3.584617851406e-05|
# SC 0000360|    gradient|   ν∇u[4] | -3.790750414259e-14|  4.629026294172e-14|  1.047899337908e-17|  2.647620783108e-15|
# SC 0000360|    gradient|   ν∇u[5] | -2.228490723292e-14|  2.951339278130e-14|  7.190037035497e-17|  2.174120450653e-15|
# SC 0000360|    gradient|   ν∇u[6] | -1.338682098376e-16|  1.103803977208e-16|  9.576198518664e-21|  8.098878178160e-18|
# SC 0000360|    gradient|    κ∇θ[1 | -0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000360|    gradient|    κ∇θ[2 | -0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000360|    gradient|    κ∇θ[3 | -0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
┌ Info: Update
│     simtime = 70680.00 / 86400.00
│     runtime = 00:02:00
└     norm(Q) = 1.5623489246786220e+07
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000540|       state|     u[1] | -9.585440660500e-01|  9.585440660500e-01|  1.500666257925e-16|  4.454002636873e-01|
# SC 0000540|       state|     u[2] | -4.348591190864e-14|  2.344385549716e-14|  4.355077186156e-17|  2.914322147192e-15|
# SC 0000540|       state|        η | -8.527328861547e-01|  8.528455869392e-01|  2.243268681923e-14|  6.029920885230e-01|
# SC 0000540|       state|        θ |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000540|         aux|        y |  0.000000000000e+00|  1.000000000000e+06|  5.000000000000e+05|  2.927758774607e+05|
# SC 0000540|         aux|        w | -4.045534600638e-04|  4.047143584632e-04|  3.685940441756e-19|  1.639586556819e-04|
# SC 0000540|         aux|     pkin |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000540|         aux|      wz0 | -2.011646847993e-04|  2.010419681595e-04| -8.215650382226e-20|  1.422284202445e-04|
# SC 0000540|    gradient|   ν∇u[1] | -3.025334369064e-02|  3.026720638465e-02| -2.724931391640e-17|  1.399088500814e-02|
# SC 0000540|    gradient|   ν∇u[2] | -2.075760849478e-14|  1.592275402931e-14| -1.762353678576e-18|  1.349489434489e-15|
# SC 0000540|    gradient|   ν∇u[3] | -6.933654440203e-05|  6.933654440203e-05| -5.134781488891e-21|  3.475498578206e-05|
# SC 0000540|    gradient|   ν∇u[4] | -1.440501588220e-14|  1.823624573916e-14|  9.156437841509e-18|  8.878829247065e-16|
# SC 0000540|    gradient|   ν∇u[5] | -1.152036335046e-14|  9.023308457985e-15|  2.801121334210e-17|  7.459824525364e-16|
# SC 0000540|    gradient|   ν∇u[6] | -3.916557083857e-17|  3.224335512488e-17|  1.974580713868e-20|  2.307791885332e-18|
# SC 0000540|    gradient|    κ∇θ[1 | -0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000540|    gradient|    κ∇θ[2 | -0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000540|    gradient|    κ∇θ[3 | -0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
┌ Info: Finished
│     norm(Q)            = 1.4992679913550535e+07
│     norm(Q) / norm(Q₀) = 8.6560130548238357e-01
└     norm(Q) - norm(Q₀) = -2.3278576348481383e+06
error = 0.0011289879366366948
# BEGIN SCPRINT
# varr - reference values (from reference run)    
# parr - digits match precision (hand edit as needed) 
#
# [
#  [ MPIStateArray Name, Field Name, Maximum, Minimum, Mean, Standard Deviation ],
#  [         :                :          :        :      :          :           ],
# ]
varr = [
 [    "state",   "u[1]", -9.58544066050006970947e-01,  9.58544066050012522062e-01,  1.50066625792533158436e-16,  4.45400263687339537100e-01 ],
 [    "state",   "u[2]", -4.34859119086353955511e-14,  2.34438554971582318433e-14,  4.35507718615627065067e-17,  2.91432214719231860938e-15 ],
 [    "state",       :η, -8.52732886154698666026e-01,  8.52845586939244948432e-01,  2.24326868192292744651e-14,  6.02992088522952829344e-01 ],
 [    "state",       :θ,  0.00000000000000000000e+00,  0.00000000000000000000e+00,  0.00000000000000000000e+00,  0.00000000000000000000e+00 ],
 [      "aux",       :y,  0.00000000000000000000e+00,  1.00000000000000011642e+06,  5.00000000000000000000e+05,  2.92775877460665535182e+05 ],
 [      "aux",       :w, -4.04553460063759536859e-04,  4.04714358463228313090e-04,  3.68594044175551975242e-19,  1.63958655681887627764e-04 ],
 [      "aux",    :pkin,  0.00000000000000000000e+00,  0.00000000000000000000e+00,  0.00000000000000000000e+00,  0.00000000000000000000e+00 ],
 [      "aux",     :wz0, -2.01164684799335768007e-04,  2.01041968159456848914e-04, -8.21565038222615828127e-20,  1.42228420244468965186e-04 ],
 [ "gradient", "ν∇u[1]", -3.02533436906376219933e-02,  3.02672063846511317509e-02, -2.72493139163998411855e-17,  1.39908850081350826650e-02 ],
 [ "gradient", "ν∇u[2]", -2.07576084947807916761e-14,  1.59227540293146197996e-14, -1.76235367857575537748e-18,  1.34948943448918087842e-15 ],
 [ "gradient", "ν∇u[3]", -6.93365444020260785060e-05,  6.93365444020259429807e-05, -5.13478148889134892579e-21,  3.47549857820555530875e-05 ],
 [ "gradient", "ν∇u[4]", -1.44050158821990506269e-14,  1.82362457391604441708e-14,  9.15643784150947173434e-18,  8.87882924706467725803e-16 ],
 [ "gradient", "ν∇u[5]", -1.15203633504597294929e-14,  9.02330845798548410036e-15,  2.80112133420985224408e-17,  7.45982452536400405862e-16 ],
 [ "gradient", "ν∇u[6]", -3.91655708385736866951e-17,  3.22433551248755236026e-17,  1.97458071386837399204e-20,  2.30779188533240584197e-18 ],
 [ "gradient", "κ∇θ[1]", -0.00000000000000000000e+00,  0.00000000000000000000e+00,  0.00000000000000000000e+00,  0.00000000000000000000e+00 ],
 [ "gradient", "κ∇θ[2]", -0.00000000000000000000e+00,  0.00000000000000000000e+00,  0.00000000000000000000e+00,  0.00000000000000000000e+00 ],
 [ "gradient", "κ∇θ[3]", -0.00000000000000000000e+00,  0.00000000000000000000e+00,  0.00000000000000000000e+00,  0.00000000000000000000e+00 ],
]
parr = [
 [    "state",   "u[1]",    16,    16,    16,    16 ],
 [    "state",   "u[2]",    16,    16,    16,    16 ],
 [    "state",       :η,    16,    16,    16,    16 ],
 [    "state",       :θ,    16,    16,    16,    16 ],
 [      "aux",       :y,    16,    16,    16,    16 ],
 [      "aux",       :w,    16,    16,    16,    16 ],
 [      "aux",    :pkin,    16,    16,    16,    16 ],
 [      "aux",     :wz0,    16,    16,    16,    16 ],
 [ "gradient", "ν∇u[1]",    16,    16,    16,    16 ],
 [ "gradient", "ν∇u[2]",    16,    16,    16,    16 ],
 [ "gradient", "ν∇u[3]",    16,    16,    16,    16 ],
 [ "gradient", "ν∇u[4]",    16,    16,    16,    16 ],
 [ "gradient", "ν∇u[5]",    16,    16,    16,    16 ],
 [ "gradient", "ν∇u[6]",    16,    16,    16,    16 ],
 [ "gradient", "κ∇θ[1]",    16,    16,    16,    16 ],
 [ "gradient", "κ∇θ[2]",    16,    16,    16,    16 ],
 [ "gradient", "κ∇θ[3]",    16,    16,    16,    16 ],
]
# END SCPRINT
# SC +++++++++++ClimateMachine StateCheck ref val check start+++++++++++++++++
# SC "N( )" bracketing indicates field failed to match      
# SC "P="  row pass count      
# SC "F="  row pass count      
# SC "NA=" row not checked count      
# SC 
# SC        Label         Field      min()      max()     mean()      std() 
# SC        state,         u[1],        12,        12,         0,        14 :: P=5, F=0, NA=1
# SC        state,         u[2],         0,         0,         0,         0 :: P=2, F=0, NA=4
# SC        state,            η,        15,        15,         0,        15 :: P=5, F=0, NA=1
# SC        state,            θ,        18,        18,        18,        18 :: P=6, F=0, NA=0
# SC          aux,            y,        18,        18,        18,        18 :: P=6, F=0, NA=0
# SC          aux,            w,        15,        15,         0,        15 :: P=5, F=0, NA=1
# SC          aux,         pkin,        18,        18,        18,        18 :: P=6, F=0, NA=0
# SC          aux,          wz0,        15,        15,         0,        15 :: P=5, F=0, NA=1
# SC     gradient,       ν∇u[1],        14,        14,         0,        15 :: P=5, F=0, NA=1
# SC     gradient,       ν∇u[2],         0,         0,         0,         0 :: P=2, F=0, NA=4
# SC     gradient,       ν∇u[3],        15,        15,         0,        15 :: P=5, F=0, NA=1
# SC     gradient,       ν∇u[4],         0,         0,         0,         0 :: P=2, F=0, NA=4
# SC     gradient,       ν∇u[5],         0,         0,         0,         0 :: P=2, F=0, NA=4
# SC     gradient,       ν∇u[6],         0,         0,         0,         0 :: P=2, F=0, NA=4
# SC     gradient,       κ∇θ[1],        18,        18,        18,        18 :: P=6, F=0, NA=0
# SC     gradient,       κ∇θ[2],        18,        18,        18,        18 :: P=6, F=0, NA=0
# SC     gradient,       κ∇θ[3],        18,        18,        18,        18 :: P=6, F=0, NA=0
# SC +++++++++++ClimateMachine StateCheck ref val check end+++++++++++++++++
[ Info: Initializing hydrostatic_spindown
# SC Start: creating state check callback
# SC Creating state check callback labeled "state" for symbols
# SC u
# SC η
# SC θ
# SC Creating state check callback labeled "aux" for symbols
# SC y
# SC w
# SC pkin
# SC wz0
# SC Creating state check callback labeled "gradient" for symbols
# SC ν∇u
# SC κ∇θ
# SC Finish: creating state check callback
┌ Info: Starting hydrostatic_spindown
│     dt              = 1.20000e+02
│     timeend         = 86400.00
│     number of steps = 720
└     norm(Q)         = 1.7320537548398674e+07
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000000|       state|     u[1] | -1.087768882099e+00|  1.087768882099e+00| -4.547473508865e-18|  4.925774623372e-01|
# SC 0000000|       state|     u[2] | -3.701804258313e-14|  3.279118161540e-14| -3.042285954007e-18|  3.458756395126e-15|
# SC 0000000|       state|        η | -6.156894560513e-01|  6.157144586653e-01|  5.470610631164e-15|  4.353677156349e-01|
# SC 0000000|       state|        θ |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000000|         aux|        y |  0.000000000000e+00|  1.000000000000e+06|  5.000000000000e+05|  2.927758774607e+05|
# SC 0000000|         aux|        w | -4.678084017937e-04|  4.680779201629e-04|  7.283063041541e-19|  1.866431940665e-04|
# SC 0000000|         aux|     pkin |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000000|         aux|      wz0 | -3.088409155002e-04|  3.091123490701e-04|  3.941291737419e-19|  2.185849758837e-04|
# SC 0000000|    gradient|   ν∇u[1] | -3.430207163076e-02|  3.432098942425e-02| -1.104893954107e-17|  1.547044526318e-02|
# SC 0000000|    gradient|   ν∇u[2] | -3.453939197838e-14|  2.861359896684e-14|  1.097483885890e-19|  1.726931357005e-15|
# SC 0000000|    gradient|   ν∇u[3] | -7.606522667894e-05|  7.606522667894e-05| -6.938893903907e-21|  3.811811610497e-05|
# SC 0000000|    gradient|   ν∇u[4] | -1.630253923479e-14|  1.552064855716e-14|  3.249378436544e-18|  1.230252426711e-15|
# SC 0000000|    gradient|   ν∇u[5] | -7.851353855505e-15|  1.003241066083e-14|  3.585906452439e-17|  9.022746281580e-16|
# SC 0000000|    gradient|   ν∇u[6] | -2.721479765042e-17|  1.721481718195e-17|  8.156301754883e-21|  2.011116526127e-18|
# SC 0000000|    gradient|    κ∇θ[1 | -0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000000|    gradient|    κ∇θ[2 | -0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000000|    gradient|    κ∇θ[3 | -0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000180|       state|     u[1] | -1.085947036901e+00|  1.085947036901e+00| -2.273736754432e-17|  4.820011547514e-01|
# SC 0000180|       state|     u[2] | -1.068286516141e-13|  1.224111621444e-13| -4.609577728768e-18|  8.750224325909e-15|
# SC 0000180|       state|        η | -2.530767298060e-01|  2.530206005217e-01|  1.126863935497e-14|  1.789384386274e-01|
# SC 0000180|       state|        θ |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000180|         aux|        y |  0.000000000000e+00|  1.000000000000e+06|  5.000000000000e+05|  2.927758774607e+05|
# SC 0000180|         aux|        w | -6.650614756960e-04|  6.647690195237e-04| -7.960299086562e-19|  2.857388443319e-04|
# SC 0000180|         aux|     pkin |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000180|         aux|      wz0 | -3.792515912056e-04|  3.791218608349e-04| -5.662137425588e-19|  2.681857121789e-04|
# SC 0000180|    gradient|   ν∇u[1] | -3.426516584024e-02|  3.424829522480e-02|  1.723066134218e-17|  1.513850538778e-02|
# SC 0000180|    gradient|   ν∇u[2] | -5.981252920744e-14|  5.698409967344e-14|  1.291122293512e-17|  3.019641355548e-15|
# SC 0000180|    gradient|   ν∇u[3] | -7.375670146146e-05|  7.375670146147e-05| -1.665334536938e-21|  3.696967811037e-05|
# SC 0000180|    gradient|   ν∇u[4] | -5.101795508604e-14|  4.133179549656e-14| -5.128066898766e-18|  3.080073932312e-15|
# SC 0000180|    gradient|   ν∇u[5] | -3.251341024338e-14|  2.269722487114e-14| -8.512975206870e-17|  2.412294390901e-15|
# SC 0000180|    gradient|   ν∇u[6] | -8.690870766581e-17|  1.347057905118e-16|  1.035587240357e-20|  7.930833362878e-18|
# SC 0000180|    gradient|    κ∇θ[1 | -0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000180|    gradient|    κ∇θ[2 | -0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000180|    gradient|    κ∇θ[3 | -0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000360|       state|     u[1] | -9.665326488789e-01|  9.665326488789e-01|  1.818989403546e-17|  4.576732195034e-01|
# SC 0000360|       state|     u[2] | -1.131243848909e-13|  8.690018773141e-14| -2.225131476373e-17|  7.568752410047e-15|
# SC 0000360|       state|        η | -9.184026758328e-01|  9.184140437861e-01|  1.673015503911e-14|  6.494135774897e-01|
# SC 0000360|       state|        θ |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000360|         aux|        y |  0.000000000000e+00|  1.000000000000e+06|  5.000000000000e+05|  2.927758774607e+05|
# SC 0000360|         aux|        w | -4.017259387126e-04|  4.019053880433e-04|  1.133537708142e-18|  1.686324793750e-04|
# SC 0000360|         aux|     pkin |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000360|         aux|      wz0 | -1.497984383068e-04|  1.497342215510e-04|  1.073585664813e-18|  1.058632095637e-04|
# SC 0000360|    gradient|   ν∇u[1] | -3.048219204414e-02|  3.049674388526e-02| -9.876544027065e-18|  1.437405441127e-02|
# SC 0000360|    gradient|   ν∇u[2] | -4.543561742741e-14|  4.200138801310e-14|  4.873784804466e-19|  2.529958586363e-15|
# SC 0000360|    gradient|   ν∇u[3] | -7.151217939010e-05|  7.151217939009e-05| -9.575673587392e-21|  3.584537392098e-05|
# SC 0000360|    gradient|   ν∇u[4] | -3.685891304987e-14|  4.598144531404e-14|  9.240631356829e-19|  2.609805471893e-15|
# SC 0000360|    gradient|   ν∇u[5] | -2.112698748681e-14|  2.953534945948e-14|  7.874431682757e-17|  2.119430240765e-15|
# SC 0000360|    gradient|   ν∇u[6] | -1.336392145267e-16|  1.083757147242e-16| -5.665942992365e-22|  8.099155448655e-18|
# SC 0000360|    gradient|    κ∇θ[1 | -0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000360|    gradient|    κ∇θ[2 | -0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000360|    gradient|    κ∇θ[3 | -0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
┌ Info: Update
│     simtime = 71160.00 / 86400.00
│     runtime = 00:01:00
└     norm(Q) = 1.6354973570685845e+07
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000540|       state|     u[1] | -9.577053596858e-01|  9.577053596858e-01|  4.092726157978e-17|  4.453237279479e-01|
# SC 0000540|       state|     u[2] | -3.977633260696e-14|  2.682839480460e-14|  9.867391712933e-18|  2.683956598681e-15|
# SC 0000540|       state|        η | -8.559417167785e-01|  8.560149087988e-01|  2.175966073992e-14|  6.052608142966e-01|
# SC 0000540|       state|        θ |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000540|         aux|        y |  0.000000000000e+00|  1.000000000000e+06|  5.000000000000e+05|  2.927758774607e+05|
# SC 0000540|         aux|        w | -4.034193018246e-04|  4.036269595962e-04| -3.885780586188e-20|  1.638162374855e-04|
# SC 0000540|         aux|     pkin |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000540|         aux|      wz0 | -1.970166309227e-04|  1.970660869349e-04| -6.861178292183e-19|  1.393933185745e-04|
# SC 0000540|    gradient|   ν∇u[1] | -3.020089063279e-02|  3.021690548632e-02| -1.243449787580e-17|  1.398612664214e-02|
# SC 0000540|    gradient|   ν∇u[2] | -2.059783673844e-14|  1.573791806011e-14|  3.620002750511e-18|  1.329395383308e-15|
# SC 0000540|    gradient|   ν∇u[3] | -6.933505133186e-05|  6.933505133186e-05| -6.245004513517e-21|  3.475420544945e-05|
# SC 0000540|    gradient|   ν∇u[4] | -1.568547992692e-14|  1.886925914901e-14|  1.926121724337e-18|  9.674402235974e-16|
# SC 0000540|    gradient|   ν∇u[5] | -1.213934659142e-14|  9.933758043984e-15| -1.006871912988e-18|  8.406619642015e-16|
# SC 0000540|    gradient|   ν∇u[6] | -3.927556818203e-17|  3.251061074571e-17|  9.951721932612e-21|  2.265859772746e-18|
# SC 0000540|    gradient|    κ∇θ[1 | -0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000540|    gradient|    κ∇θ[2 | -0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000540|    gradient|    κ∇θ[3 | -0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
┌ Info: Finished
│     norm(Q)            = 1.5028312562790792e+07
│     norm(Q) / norm(Q₀) = 8.6765855394483393e-01
└     norm(Q) - norm(Q₀) = -2.2922249856078811e+06
error = 0.00330630717736293
# BEGIN SCPRINT
# varr - reference values (from reference run)    
# parr - digits match precision (hand edit as needed) 
#
# [
#  [ MPIStateArray Name, Field Name, Maximum, Minimum, Mean, Standard Deviation ],
#  [         :                :          :        :      :          :           ],
# ]
varr = [
 [    "state",   "u[1]", -9.57705359685843249373e-01,  9.57705359685846246975e-01,  4.09272615797817732838e-17,  4.45323727947876057964e-01 ],
 [    "state",   "u[2]", -3.97763326069570017285e-14,  2.68283948045981341472e-14,  9.86739171293257329595e-18,  2.68395659868081566943e-15 ],
 [    "state",       :η, -8.55941716778495953477e-01,  8.56014908798836460235e-01,  2.17596607399173065927e-14,  6.05260814296590510253e-01 ],
 [    "state",       :θ,  0.00000000000000000000e+00,  0.00000000000000000000e+00,  0.00000000000000000000e+00,  0.00000000000000000000e+00 ],
 [      "aux",       :y,  0.00000000000000000000e+00,  1.00000000000000011642e+06,  5.00000000000000000000e+05,  2.92775877460665535182e+05 ],
 [      "aux",       :w, -4.03419301824649713210e-04,  4.03626959596158638513e-04, -3.88578058618804785200e-20,  1.63816237485524089645e-04 ],
 [      "aux",    :pkin,  0.00000000000000000000e+00,  0.00000000000000000000e+00,  0.00000000000000000000e+00,  0.00000000000000000000e+00 ],
 [      "aux",     :wz0, -1.97016630922690853941e-04,  1.97066086934920091768e-04, -6.86117829218346711968e-19,  1.39393318574472545920e-04 ],
 [ "gradient", "ν∇u[1]", -3.02008906327931453406e-02,  3.02169054863199518235e-02, -1.24344978758017535116e-17,  1.39861266421407951649e-02 ],
 [ "gradient", "ν∇u[2]", -2.05978367384372508080e-14,  1.57379180601147416964e-14,  3.62000275051089740942e-18,  1.32939538330788498606e-15 ],
 [ "gradient", "ν∇u[3]", -6.93350513318618968776e-05,  6.93350513318595522904e-05, -6.24500451351650536896e-21,  3.47542054494542898492e-05 ],
 [ "gradient", "ν∇u[4]", -1.56854799269231918512e-14,  1.88692591490102842832e-14,  1.92612172433741593399e-18,  9.67440223597421045503e-16 ],
 [ "gradient", "ν∇u[5]", -1.21393465914193108045e-14,  9.93375804398401605853e-15, -1.00687191298751067721e-18,  8.40661964201546130337e-16 ],
 [ "gradient", "ν∇u[6]", -3.92755681820295500928e-17,  3.25106107457148034953e-17,  9.95172193261159797674e-21,  2.26585977274648721388e-18 ],
 [ "gradient", "κ∇θ[1]", -0.00000000000000000000e+00,  0.00000000000000000000e+00,  0.00000000000000000000e+00,  0.00000000000000000000e+00 ],
 [ "gradient", "κ∇θ[2]", -0.00000000000000000000e+00,  0.00000000000000000000e+00,  0.00000000000000000000e+00,  0.00000000000000000000e+00 ],
 [ "gradient", "κ∇θ[3]", -0.00000000000000000000e+00,  0.00000000000000000000e+00,  0.00000000000000000000e+00,  0.00000000000000000000e+00 ],
]
parr = [
 [    "state",   "u[1]",    16,    16,    16,    16 ],
 [    "state",   "u[2]",    16,    16,    16,    16 ],
 [    "state",       :η,    16,    16,    16,    16 ],
 [    "state",       :θ,    16,    16,    16,    16 ],
 [      "aux",       :y,    16,    16,    16,    16 ],
 [      "aux",       :w,    16,    16,    16,    16 ],
 [      "aux",    :pkin,    16,    16,    16,    16 ],
 [      "aux",     :wz0,    16,    16,    16,    16 ],
 [ "gradient", "ν∇u[1]",    16,    16,    16,    16 ],
 [ "gradient", "ν∇u[2]",    16,    16,    16,    16 ],
 [ "gradient", "ν∇u[3]",    16,    16,    16,    16 ],
 [ "gradient", "ν∇u[4]",    16,    16,    16,    16 ],
 [ "gradient", "ν∇u[5]",    16,    16,    16,    16 ],
 [ "gradient", "ν∇u[6]",    16,    16,    16,    16 ],
 [ "gradient", "κ∇θ[1]",    16,    16,    16,    16 ],
 [ "gradient", "κ∇θ[2]",    16,    16,    16,    16 ],
 [ "gradient", "κ∇θ[3]",    16,    16,    16,    16 ],
]
# END SCPRINT
# SC +++++++++++ClimateMachine StateCheck ref val check start+++++++++++++++++
# SC "N( )" bracketing indicates field failed to match      
# SC "P="  row pass count      
# SC "F="  row pass count      
# SC "NA=" row not checked count      
# SC 
# SC        Label         Field      min()      max()     mean()      std() 
# SC        state,         u[1],        15,        15,         0,        15 :: P=5, F=0, NA=1
# SC        state,         u[2],         0,         0,         0,         0 :: P=2, F=0, NA=4
# SC        state,            η,        14,        15,         0,        15 :: P=5, F=0, NA=1
# SC        state,            θ,        18,        18,        18,        18 :: P=6, F=0, NA=0
# SC          aux,            y,        18,        18,        18,        18 :: P=6, F=0, NA=0
# SC          aux,            w,        15,        15,         0,        15 :: P=5, F=0, NA=1
# SC          aux,         pkin,        18,        18,        18,        18 :: P=6, F=0, NA=0
# SC          aux,          wz0,        15,        15,         0,        15 :: P=5, F=0, NA=1
# SC     gradient,       ν∇u[1],        15,        14,         0,        15 :: P=5, F=0, NA=1
# SC     gradient,       ν∇u[2],         0,         0,         0,         0 :: P=2, F=0, NA=4
# SC     gradient,       ν∇u[3],        15,        15,         0,        15 :: P=5, F=0, NA=1
# SC     gradient,       ν∇u[4],         0,         0,         0,         0 :: P=2, F=0, NA=4
# SC     gradient,       ν∇u[5],         0,         0,         0,         0 :: P=2, F=0, NA=4
# SC     gradient,       ν∇u[6],         0,         0,         0,         0 :: P=2, F=0, NA=4
# SC     gradient,       κ∇θ[1],        18,        18,        18,        18 :: P=6, F=0, NA=0
# SC     gradient,       κ∇θ[2],        18,        18,        18,        18 :: P=6, F=0, NA=0
# SC     gradient,       κ∇θ[3],        18,        18,        18,        18 :: P=6, F=0, NA=0
# SC +++++++++++ClimateMachine StateCheck ref val check end+++++++++++++++++
Test Summary:                                                                                                                               | Pass  Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Ocean/HydrostaticBoussinesq/3D_hydrostatic_spindown.jl |    4      4

out_10192363

+ hostname
hpc-91-24.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192363_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192363_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Ocean/SplitExplicit/hydrostatic_spindown.jl
┌ Info: doing VTK output
└   outprefix = "vtk_split/slow/mpirank0000_step0000"
┌ Info: doing VTK output
└   outprefix = "vtk_split/fast/mpirank0000_step0000"
# SC Start: creating state check callback
# SC Creating state check callback labeled "3D state" for symbols
# SC u
# SC η
# SC θ
# SC Creating state check callback labeled "3D aux" for symbols
# SC y
# SC w
# SC pkin
# SC wz0
# SC Creating state check callback labeled "2D state" for symbols
# SC η
# SC U
# SC Finish: creating state check callback
┌ Info: Starting
│ norm(Q₀) = 1.7320544160043247e+07
└ ArrayType = Array
┌ Info: doing VTK output
└   outprefix = "vtk_split/slow/mpirank0000_step0000"
┌ Info: doing VTK output
└   outprefix = "vtk_split/fast/mpirank0000_step0000"
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000000|    3D state|     u[1] | -1.030849901770e+00|  1.030849901770e+00|  9.094947017729e-18|  4.960604425050e-01|
# SC 0000000|    3D state|     u[2] | -4.967916160951e-14|  5.486625198699e-14|  2.027280739225e-17|  3.656198266806e-15|
# SC 0000000|    3D state|        η | -9.569889064320e-01|  9.572485127703e-01|  1.873559085652e-15|  6.767523540914e-01|
# SC 0000000|    3D state|        θ |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000000|      3D aux|        y |  0.000000000000e+00|  1.000000000000e+06|  5.000000000000e+05|  2.927758774607e+05|
# SC 0000000|      3D aux|        w | -4.242833560905e-04|  4.245094828326e-04|  2.275957200482e-19|  1.851101862209e-04|
# SC 0000000|      3D aux|     pkin |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000000|      3D aux|      wz0 | -1.149895395653e-04|  1.149770630109e-04|  1.182387521226e-19|  8.135886648546e-05|
# SC 0000000|    2D state|        η | -9.569891904437e-01|  9.572461475299e-01| -3.549160965122e-16|  6.772813599671e-01|
# SC 0000000|    2D state|     U[1] | -1.792547171126e+01|  1.792547171126e+01|  2.585546880762e-15|  1.274366126549e+01|
# SC 0000000|    2D state|     U[2] | -4.083475257973e-13|  4.659046156900e-13|  8.932801159977e-15|  8.678760389051e-14|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
┌ Info: doing VTK output
└   outprefix = "vtk_split/slow/mpirank0000_step0001"
┌ Info: doing VTK output
└   outprefix = "vtk_split/fast/mpirank0000_step0001"
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000024|    3D state|     u[1] | -1.063414661605e+00|  1.063414661605e+00|  0.000000000000e+00|  4.938608165201e-01|
# SC 0000024|    3D state|     u[2] | -2.129062988655e-14|  4.232253674540e-14| -3.527629601589e-17|  2.417602870242e-15|
# SC 0000024|    3D state|        η | -8.244114109667e-01|  8.244420496554e-01|  3.838067641482e-15|  5.829541785963e-01|
# SC 0000024|    3D state|        θ |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000024|      3D aux|        y |  0.000000000000e+00|  1.000000000000e+06|  5.000000000000e+05|  2.927758774607e+05|
# SC 0000024|      3D aux|        w | -5.607309778931e-04|  5.600943253429e-04|  2.220446049250e-21|  2.457264742782e-04|
# SC 0000024|      3D aux|     pkin |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000024|      3D aux|      wz0 | -2.237226387793e-04|  2.231625619443e-04| -3.219646771413e-20|  1.579634386612e-04|
# SC 0000024|    2D state|        η | -8.244141785601e-01|  8.244202932402e-01| -2.486899575160e-16|  5.834054579671e-01|
# SC 0000024|    2D state|     U[1] | -3.512762310561e+01|  3.512762310561e+01| -1.554499862166e-14|  2.497975041665e+01|
# SC 0000024|    2D state|     U[2] | -8.109135078406e-13|  7.210305729427e-13| -1.446918450409e-14|  1.868782900008e-13|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
┌ Info: doing VTK output
└   outprefix = "vtk_split/slow/mpirank0000_step0002"
┌ Info: doing VTK output
└   outprefix = "vtk_split/fast/mpirank0000_step0002"
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000048|    3D state|     u[1] | -1.087952057868e+00|  1.087952057868e+00|  3.637978807092e-17|  4.926012399818e-01|
# SC 0000048|    3D state|     u[2] | -3.882066450383e-14|  3.292991009959e-14| -7.356612386379e-18|  3.531816139322e-15|
# SC 0000048|    3D state|        η | -6.144945461622e-01|  6.144888145231e-01|  5.466063157655e-15|  4.345054706788e-01|
# SC 0000048|    3D state|        θ |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000048|      3D aux|        y |  0.000000000000e+00|  1.000000000000e+06|  5.000000000000e+05|  2.927758774607e+05|
# SC 0000048|      3D aux|        w | -4.683795284143e-04|  4.685186927529e-04|  6.561418075535e-19|  1.868460187638e-04|
# SC 0000048|      3D aux|     pkin |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000048|      3D aux|      wz0 | -3.109185751031e-04|  3.107100014918e-04|  7.549516567451e-20|  2.198124729873e-04|
# SC 0000048|    2D state|        η | -6.144998792793e-01|  6.145002616428e-01| -5.345057729755e-16|  4.348478809593e-01|
# SC 0000048|    2D state|     U[1] | -4.901996541150e+01|  4.901996541150e+01|  8.738987311574e-15|  3.485444011943e+01|
# SC 0000048|    2D state|     U[2] | -8.747690527241e-13|  1.179475712763e-12|  1.854693536768e-14|  2.605184660591e-13|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
┌ Info: doing VTK output
└   outprefix = "vtk_split/slow/mpirank0000_step0003"
┌ Info: doing VTK output
└   outprefix = "vtk_split/fast/mpirank0000_step0003"
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000072|    3D state|     u[1] | -1.101094633621e+00|  1.101094633621e+00| -4.547473508865e-18|  4.908955394026e-01|
# SC 0000072|    3D state|     u[2] | -6.929315542373e-14|  7.329786169974e-14| -3.088628351401e-17|  5.877963285666e-15|
# SC 0000072|    3D state|        η | -3.472357815297e-01|  3.471064155602e-01|  7.455582817784e-15|  2.454570203728e-01|
# SC 0000072|    3D state|        θ |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000072|      3D aux|        y |  0.000000000000e+00|  1.000000000000e+06|  5.000000000000e+05|  2.927758774607e+05|
# SC 0000072|      3D aux|        w | -6.648280052304e-04|  6.645837534357e-04| -9.769962616701e-20|  2.857703298814e-04|
# SC 0000072|      3D aux|     pkin |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000072|      3D aux|      wz0 | -3.688901311789e-04|  3.688235022847e-04|  4.241051954068e-19|  2.607900313386e-04|
# SC 0000072|    2D state|        η | -3.472280747623e-01|  3.471093873592e-01| -2.880362615088e-16|  2.456533567197e-01|
# SC 0000072|    2D state|     U[1] | -5.827750016193e+01|  5.827750016193e+01| -2.159611378616e-14|  4.144019439961e+01|
# SC 0000072|    2D state|     U[2] | -1.830756375639e-12|  1.470057524424e-12| -3.069873369092e-14|  3.662851907989e-13|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
┌ Info: doing VTK output
└   outprefix = "vtk_split/slow/mpirank0000_step0004"
┌ Info: doing VTK output
└   outprefix = "vtk_split/fast/mpirank0000_step0004"
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000096|    3D state|     u[1] | -1.100698393690e+00|  1.100698393690e+00|  1.818989403546e-17|  4.875519569987e-01|
# SC 0000096|    3D state|     u[2] | -9.990960544410e-14|  8.144025500303e-14| -5.416761552022e-18|  7.510490697731e-15|
# SC 0000096|    3D state|        η | -4.764930990793e-02|  4.750650117194e-02|  9.410996426595e-15|  3.367366313844e-02|
# SC 0000096|    3D state|        θ |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000096|      3D aux|        y |  0.000000000000e+00|  1.000000000000e+06|  5.000000000000e+05|  2.927758774607e+05|
# SC 0000096|      3D aux|        w | -4.826158513266e-04|  4.827914424847e-04|  1.078026556911e-18|  1.962228236503e-04|
# SC 0000096|      3D aux|     pkin |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000096|      3D aux|      wz0 | -3.921689419540e-04|  3.921079000381e-04|  4.352074256531e-19|  2.772682179948e-04|
# SC 0000096|    2D state|        η | -4.766190044388e-02|  4.753405839796e-02| -2.321920433701e-16|  3.370637815801e-02|
# SC 0000096|    2D state|     U[1] | -6.205712582683e+01|  6.205712582683e+01|  1.149923489763e-14|  4.412712862452e+01|
# SC 0000096|    2D state|     U[2] | -1.572238547670e-12|  2.259869114211e-12|  2.836348669329e-14|  4.086760412524e-13|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
┌ Info: doing VTK output
└   outprefix = "vtk_split/slow/mpirank0000_step0005"
┌ Info: doing VTK output
└   outprefix = "vtk_split/fast/mpirank0000_step0005"
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000120|    3D state|     u[1] | -1.085847232593e+00|  1.085847232593e+00| -2.273736754432e-17|  4.819856694890e-01|
# SC 0000120|    3D state|     u[2] | -1.063443005651e-13|  1.228554806650e-13| -5.766327516608e-18|  8.757116486006e-15|
# SC 0000120|    3D state|        η | -2.560352594349e-01|  2.559459824676e-01|  1.123225956690e-14|  1.810255712824e-01|
# SC 0000120|    3D state|        θ |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000120|      3D aux|        y |  0.000000000000e+00|  1.000000000000e+06|  5.000000000000e+05|  2.927758774607e+05|
# SC 0000120|      3D aux|        w | -6.644745219146e-04|  6.643281981857e-04| -2.220446049250e-19|  2.855011818948e-04|
# SC 0000120|      3D aux|     pkin |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000120|      3D aux|      wz0 | -3.784822344753e-04|  3.785418959328e-04|  5.062616992291e-19|  2.676615239127e-04|
# SC 0000120|    2D state|        η | -2.560062764875e-01|  2.559531056618e-01| -8.890665981198e-17|  1.811545594929e-01|
# SC 0000120|    2D state|     U[1] | -6.000478742874e+01|  6.000478742874e+01| -2.672822518868e-14|  4.266678165849e+01|
# SC 0000120|    2D state|     U[2] | -2.300012131418e-12|  1.770945862628e-12| -2.529693840729e-14|  4.335421358007e-13|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
┌ Info: doing VTK output
└   outprefix = "vtk_split/slow/mpirank0000_step0006"
┌ Info: doing VTK output
└   outprefix = "vtk_split/fast/mpirank0000_step0006"
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000144|    3D state|     u[1] | -1.057012603414e+00|  1.057012603414e+00|  1.818989403546e-17|  4.744216931560e-01|
# SC 0000144|    3D state|     u[2] | -1.296446226968e-13|  1.008881280258e-13| -1.518059036762e-17|  9.073782785751e-15|
# SC 0000144|    3D state|        η | -5.351494151782e-01|  5.351175482997e-01|  1.318085196544e-14|  3.783828437171e-01|
# SC 0000144|    3D state|        θ |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000144|      3D aux|        y |  0.000000000000e+00|  1.000000000000e+06|  5.000000000000e+05|  2.927758774607e+05|
# SC 0000144|      3D aux|        w | -4.578776292365e-04|  4.581271663229e-04|  9.969802761134e-19|  1.833114368455e-04|
# SC 0000144|      3D aux|     pkin |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000144|      3D aux|      wz0 | -3.293453317860e-04|  3.295541981653e-04|  2.353672812205e-19|  2.329163472408e-04|
# SC 0000144|    2D state|        η | -5.351332624160e-01|  5.351116163412e-01|  4.032330025439e-17|  3.786723371258e-01|
# SC 0000144|    2D state|     U[1] | -5.231555356022e+01|  5.231555356022e+01|  7.471967489181e-15|  3.720073742149e+01|
# SC 0000144|    2D state|     U[2] | -1.519120786030e-12|  2.165393935368e-12|  2.837069711311e-14|  4.054018561731e-13|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
┌ Info: doing VTK output
└   outprefix = "vtk_split/slow/mpirank0000_step0007"
┌ Info: doing VTK output
└   outprefix = "vtk_split/fast/mpirank0000_step0007"
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000168|    3D state|     u[1] | -1.016054170724e+00|  1.016054170724e+00|  0.000000000000e+00|  4.658370806400e-01|
# SC 0000168|    3D state|     u[2] | -1.064565603348e-13|  1.287547095509e-13| -4.132313496959e-19|  8.710916270887e-15|
# SC 0000168|    3D state|        η | -7.635558864508e-01|  7.636706046541e-01|  1.470198185416e-14|  5.399322649117e-01|
# SC 0000168|    3D state|        θ |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000168|      3D aux|        y |  0.000000000000e+00|  1.000000000000e+06|  5.000000000000e+05|  2.927758774607e+05|
# SC 0000168|      3D aux|        w | -5.572480080511e-04|  5.571718535075e-04| -3.891331701311e-19|  2.422044219195e-04|
# SC 0000168|      3D aux|     pkin |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000168|      3D aux|      wz0 | -2.493592414581e-04|  2.495110529698e-04| -3.219646771413e-20|  1.764175851246e-04|
# SC 0000168|    2D state|        η | -7.635508714690e-01|  7.636545126511e-01| -5.591971330432e-16|  5.403490137017e-01|
# SC 0000168|    2D state|     U[1] | -3.973213335592e+01|  3.973213335592e+01| -1.263300852816e-14|  2.825122806280e+01|
# SC 0000168|    2D state|     U[2] | -2.053163557891e-12|  1.467389279025e-12| -1.915911931448e-14|  3.561237123567e-13|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
┌ Info: doing VTK output
└   outprefix = "vtk_split/slow/mpirank0000_step0008"
┌ Info: doing VTK output
└   outprefix = "vtk_split/fast/mpirank0000_step0008"
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000192|    3D state|     u[1] | -9.658939290454e-01|  9.658939290454e-01|  9.094947017729e-18|  4.576350160423e-01|
# SC 0000192|    3D state|     u[2] | -1.110760059514e-13|  8.106565197887e-14| -3.501828704657e-17|  7.470562742106e-15|
# SC 0000192|    3D state|        η | -9.200627437540e-01|  9.200457998264e-01|  1.669832272455e-14|  6.505753848104e-01|
# SC 0000192|    3D state|        θ |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000192|      3D aux|        y |  0.000000000000e+00|  1.000000000000e+06|  5.000000000000e+05|  2.927758774607e+05|
# SC 0000192|      3D aux|        w | -4.008169841299e-04|  4.010525136073e-04|  6.961098364400e-19|  1.687139643092e-04|
# SC 0000192|      3D aux|     pkin |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000192|      3D aux|      wz0 | -1.461364927311e-04|  1.462965990385e-04|  9.159339953158e-20|  1.033633898282e-04|
# SC 0000192|    2D state|        η | -9.200761360448e-01|  9.200374458572e-01| -4.934719299854e-16|  6.510782538781e-01|
# SC 0000192|    2D state|     U[1] | -2.343081988666e+01|  2.343081988666e+01|  1.418077599791e-14|  1.666103705499e+01|
# SC 0000192|    2D state|     U[2] | -1.083027582660e-12|  1.514995270865e-12|  7.775128239417e-15|  2.615616589758e-13|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
┌ Info: doing VTK output
└   outprefix = "vtk_split/slow/mpirank0000_step0009"
┌ Info: doing VTK output
└   outprefix = "vtk_split/fast/mpirank0000_step0009"
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000216|    3D state|     u[1] | -9.104002084349e-01|  9.104002084349e-01|  2.273736754432e-18|  4.511514585364e-01|
# SC 0000216|    3D state|     u[2] | -6.934311728573e-14|  8.738109138666e-14|  7.378776007413e-18|  5.721453418549e-15|
# SC 0000216|    3D state|        η | -9.898595680788e-01|  9.899941544807e-01|  1.816715666791e-14|  6.999533120657e-01|
# SC 0000216|    3D state|        θ |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000216|      3D aux|        y |  0.000000000000e+00|  1.000000000000e+06|  5.000000000000e+05|  2.927758774607e+05|
# SC 0000216|      3D aux|        w | -3.827658570659e-04|  3.825611159184e-04|  7.785438960184e-20|  1.853005672861e-04|
# SC 0000216|      3D aux|     pkin |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000216|      3D aux|      wz0 | -2.936549477064e-05|  2.935357477093e-05|  2.477185123695e-19|  2.075801270640e-05|
# SC 0000216|    2D state|        η | -9.898723197696e-01|  9.899709386910e-01| -7.855049943828e-16|  7.005037547305e-01|
# SC 0000216|    2D state|     U[1] | -4.954826959985e+00|  4.954826959985e+00| -1.193814734568e-14|  3.523050736447e+00|
# SC 0000216|    2D state|     U[2] | -1.046825335866e-12|  6.448337657272e-13| -9.390961598498e-15|  1.558983663210e-13|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
┌ Info: doing VTK output
└   outprefix = "vtk_split/slow/mpirank0000_step0010"
┌ Info: doing VTK output
└   outprefix = "vtk_split/fast/mpirank0000_step0010"
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000240|    3D state|     u[1] | -9.237102932312e-01|  9.237102932312e-01| -1.023181539495e-17|  4.471289658508e-01|
# SC 0000240|    3D state|     u[2] | -5.176449146151e-14|  5.374572415076e-14| -2.198260851981e-17|  3.590527861107e-15|
# SC 0000240|    3D state|        η | -9.667136810087e-01|  9.666903847370e-01|  2.004526322708e-14|  6.835588992074e-01|
# SC 0000240|    3D state|        θ |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000240|      3D aux|        y |  0.000000000000e+00|  1.000000000000e+06|  5.000000000000e+05|  2.927758774607e+05|
# SC 0000240|      3D aux|        w | -4.246686831976e-04|  4.243591435855e-04|  4.168887457467e-19|  1.955813770688e-04|
# SC 0000240|      3D aux|     pkin |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000240|      3D aux|      wz0 | -9.016704910559e-05|  9.001563631095e-05|  2.953193245503e-19|  6.367775864208e-05|
# SC 0000240|    2D state|        η | -9.667375283295e-01|  9.666866284766e-01| -2.037481294792e-16|  6.840938220260e-01|
# SC 0000240|    2D state|     U[1] | -1.396013622166e+01|  1.396013622166e+01| -7.256868717054e-15|  9.927160589808e+00|
# SC 0000240|    2D state|     U[2] | -3.613430715602e-13|  3.244793380678e-13| -5.372585087706e-15|  8.351189185950e-14|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
┌ Info: doing VTK output
└   outprefix = "vtk_split/slow/mpirank0000_step0011"
┌ Info: doing VTK output
└   outprefix = "vtk_split/fast/mpirank0000_step0011"
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000264|    3D state|     u[1] | -9.585474282467e-01|  9.585474282467e-01|  6.821210263297e-18|  4.454006553253e-01|
# SC 0000264|    3D state|     u[2] | -4.149063678878e-14|  1.871165461679e-14|  2.159920325784e-17|  2.126555374397e-15|
# SC 0000264|    3D state|        η | -8.527217343958e-01|  8.528342593823e-01|  2.185061021009e-14|  6.029835731686e-01|
# SC 0000264|    3D state|        θ |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000264|      3D aux|        y |  0.000000000000e+00|  1.000000000000e+06|  5.000000000000e+05|  2.927758774607e+05|
# SC 0000264|      3D aux|        w | -4.044892228289e-04|  4.046499790483e-04|  5.684341886081e-19|  1.639495855752e-04|
# SC 0000264|      3D aux|     pkin |  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|  0.000000000000e+00|
# SC 0000264|      3D aux|      wz0 | -2.009330996605e-04|  2.008098594697e-04|  3.697042672002e-19|  1.420646826978e-04|
# SC 0000264|    2D state|        η | -8.527356543685e-01|  8.528396109882e-01| -2.819078304128e-16|  6.034638471078e-01|
# SC 0000264|    2D state|     U[1] | -3.153917220077e+01|  3.153917220077e+01| -1.672095795158e-15|  2.242613663716e+01|
# SC 0000264|    2D state|     U[2] | -5.286601966402e-13|  5.536548923354e-13|  1.421613752331e-14|  1.420474645186e-13|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
3D error = 0.0011297069946050026
2D error = 9.928972389302695e-5
# BEGIN SCPRINT
# varr - reference values (from reference run)    
# parr - digits match precision (hand edit as needed) 
#
# [
#  [ MPIStateArray Name, Field Name, Maximum, Minimum, Mean, Standard Deviation ],
#  [         :                :          :        :      :          :           ],
# ]
varr = [
 [ "3D state", "u[1]", -9.58547428246721122669e-01,  9.58547428246723454137e-01,  6.82121026329696195717e-18,  4.45400655325267236240e-01 ],
 [ "3D state", "u[2]", -4.14906367887818936843e-14,  1.87116546167932601190e-14,  2.15992032578401374106e-17,  2.12655537439710835119e-15 ],
 [ "3D state",     :η, -8.52721734395836761244e-01,  8.52834259382325488019e-01,  2.18506102100946020216e-14,  6.02983573168642816853e-01 ],
 [ "3D state",     :θ,  0.00000000000000000000e+00,  0.00000000000000000000e+00,  0.00000000000000000000e+00,  0.00000000000000000000e+00 ],
 [   "3D aux",     :y,  0.00000000000000000000e+00,  1.00000000000000011642e+06,  5.00000000000000000000e+05,  2.92775877460665535182e+05 ],
 [   "3D aux",     :w, -4.04489222828861654489e-04,  4.04649979048276892818e-04,  5.68434188608080195197e-19,  1.63949585575227561006e-04 ],
 [   "3D aux",  :pkin,  0.00000000000000000000e+00,  0.00000000000000000000e+00,  0.00000000000000000000e+00,  0.00000000000000000000e+00 ],
 [   "3D aux",   :wz0, -2.00933099660528401067e-04,  2.00809859469733319789e-04,  3.69704267200177134694e-19,  1.42064682697768177293e-04 ],
 [ "2D state",     :η, -8.52735654368523166013e-01,  8.52839610988159746441e-01, -2.81907830412819762471e-16,  6.03463847107759798760e-01 ],
 [ "2D state", "U[1]", -3.15391722007734429667e+01,  3.15391722007731125643e+01, -1.67209579515770203353e-15,  2.24261366371601056358e+01 ],
 [ "2D state", "U[2]", -5.28660196640225304769e-13,  5.53654892335427414531e-13,  1.42161375233088450725e-14,  1.42047464518599267171e-13 ],
]
parr = [
 [ "3D state", "u[1]",    16,    16,    16,    16 ],
 [ "3D state", "u[2]",    16,    16,    16,    16 ],
 [ "3D state",     :η,    16,    16,    16,    16 ],
 [ "3D state",     :θ,    16,    16,    16,    16 ],
 [   "3D aux",     :y,    16,    16,    16,    16 ],
 [   "3D aux",     :w,    16,    16,    16,    16 ],
 [   "3D aux",  :pkin,    16,    16,    16,    16 ],
 [   "3D aux",   :wz0,    16,    16,    16,    16 ],
 [ "2D state",     :η,    16,    16,    16,    16 ],
 [ "2D state", "U[1]",    16,    16,    16,    16 ],
 [ "2D state", "U[2]",    16,    16,    16,    16 ],
]
# END SCPRINT
# SC +++++++++++ClimateMachine StateCheck ref val check start+++++++++++++++++
# SC "N( )" bracketing indicates field failed to match      
# SC "P="  row pass count      
# SC "F="  row pass count      
# SC "NA=" row not checked count      
# SC 
# SC        Label         Field      min()      max()     mean()      std() 
# SC     3D state,         u[1],        14,        14,         0,        14 :: P=5, F=0, NA=1
# SC     3D state,         u[2],         0,         0,         0,         0 :: P=2, F=0, NA=4
# SC     3D state,            η,        15,        15,         0,        15 :: P=5, F=0, NA=1
# SC     3D state,            θ,        18,        18,        18,        18 :: P=6, F=0, NA=0
# SC       3D aux,            y,        18,        18,        18,        18 :: P=6, F=0, NA=0
# SC       3D aux,            w,        14,        14,         0,        15 :: P=5, F=0, NA=1
# SC       3D aux,         pkin,        18,        18,        18,        18 :: P=6, F=0, NA=0
# SC       3D aux,          wz0,        15,        15,         0,        15 :: P=5, F=0, NA=1
# SC     2D state,            η,        15,        15,         0,        15 :: P=5, F=0, NA=1
# SC     2D state,         U[1],        15,        15,         0,        15 :: P=5, F=0, NA=1
# SC     2D state,         U[2],         0,         0,         0,         0 :: P=2, F=0, NA=4
# SC +++++++++++ClimateMachine StateCheck ref val check end+++++++++++++++++
Test Summary:                                                                                                                    | Pass  Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Ocean/SplitExplicit/hydrostatic_spindown.jl |    3      3

out_10192364

+ hostname
hpc-22-12.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project tutorials/Numerics/DGMethods/nonnegative.jl
┌ Info: Starting
│ FT               = Float64
│ dim              = 2
│ Ne               = 20
│ polynomial order = 4
│ final time       = 5.0000000000000000e+00
└ time step        = 1.5625000000000001e-03
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0000
┌ Info: Step  = 1
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 9.9432020767041762e-01
└ sum error   = 1.6048083315449916e-15
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0001
┌ Info: Step  = 2
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 1.0000143365966552e+00
└ sum error   = 3.3555083295940734e-15
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0002
┌ Info: Step  = 3
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 9.8176263118350948e-01
└ sum error   = 4.8144249946349749e-15
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0003
┌ Info: Step  = 4
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 9.9738987314021454e-01
└ sum error   = 6.4192333261799665e-15
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0004
┌ Info: Step  = 5
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 9.8647851628764271e-01
└ sum error   = 7.8781499912208684e-15
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0005
┌ Info: Step  = 6
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 9.9088076596995966e-01
└ sum error   = 9.3370666562617687e-15
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0006
┌ Info: Step  = 7
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 9.9197942278285756e-01
└ sum error   = 1.0941874987806760e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0007
┌ Info: Step  = 8
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 9.9163733089472494e-01
└ sum error   = 1.2400791652847662e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0008
┌ Info: Step  = 9
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 9.9443826407826330e-01
└ sum error   = 1.3859708317888564e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0009
┌ Info: Step  = 10
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 1.0010541856657944e+00
└ sum error   = 1.5464516649433554e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0010
┌ Info: Step  = 11
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 9.9668589801826857e-01
└ sum error   = 1.7069324980978546e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0011
┌ Info: Step  = 12
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 1.0022902785045398e+00
└ sum error   = 1.8382349979515358e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0012
┌ Info: Step  = 13
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 1.0131223189593199e+00
└ sum error   = 2.0133049977564441e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0013
┌ Info: Step  = 14
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 1.0176430010623181e+00
└ sum error   = 2.1591966642605341e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0014
┌ Info: Step  = 15
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 9.6624828674644880e-01
└ sum error   = 2.3342666640654424e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0015
┌ Info: Step  = 16
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 9.1765628524907183e-01
└ sum error   = 2.4947474972199416e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0016
┌ Info: Step  = 17
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 9.5497589935317007e-01
└ sum error   = 2.6406391637240316e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0017
┌ Info: Step  = 18
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 9.7045737402253995e-01
└ sum error   = 2.8011199968785308e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0018
┌ Info: Step  = 19
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 9.4250261512808642e-01
└ sum error   = 2.9616008300330299e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0019
┌ Info: Step  = 20
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 1.0123754684640098e+00
└ sum error   = 3.1366708298379382e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0020
┌ Info: Step  = 21
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 1.0160201122294243e+00
└ sum error   = 3.2971516629924374e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0021
┌ Info: Step  = 22
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 9.5685514584470932e-01
└ sum error   = 3.4430433294965274e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0022
┌ Info: Step  = 23
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 8.9287330603654669e-01
└ sum error   = 3.6181133293014357e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0023
┌ Info: Step  = 24
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 8.5044037149808971e-01
└ sum error   = 3.7785941624559349e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0024
┌ Info: Step  = 25
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 8.3865331836336532e-01
└ sum error   = 3.9536641622608432e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0025
┌ Info: Step  = 26
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 8.2356691639937674e-01
└ sum error   = 4.1141449954153423e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0026
┌ Info: Step  = 27
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 8.1019244554659542e-01
└ sum error   = 4.2892149952202500e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0027
┌ Info: Step  = 28
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 7.9113739209057643e-01
└ sum error   = 4.4788741616755674e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0028
┌ Info: Step  = 29
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 7.9863354612519843e-01
└ sum error   = 4.6685333281308848e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0029
┌ Info: Step  = 30
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 8.3171343703180367e-01
└ sum error   = 4.8436033279357931e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0030
┌ Info: Step  = 31
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 8.1519156019367400e-01
└ sum error   = 5.0040841610902923e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0031
┌ Info: Step  = 32
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 8.1538575261162827e-01
└ sum error   = 5.1791541608951999e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0032
┌ Info: Step  = 33
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 7.9434916759559560e-01
└ sum error   = 5.3396349940496991e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0033
┌ Info: Step  = 34
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 7.6749630959436099e-01
└ sum error   = 5.5147049938546074e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0034
┌ Info: Step  = 35
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 7.6463177249978098e-01
└ sum error   = 5.6897749936595151e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0035
┌ Info: Step  = 36
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 7.5487621796555571e-01
└ sum error   = 5.8502558268140142e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0036
┌ Info: Step  = 37
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 7.4465086828894156e-01
└ sum error   = 6.0253258266189225e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0037
┌ Info: Step  = 38
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 7.4607836982882969e-01
└ sum error   = 6.1858066597734217e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0038
┌ Info: Step  = 39
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 7.4606463225455300e-01
└ sum error   = 6.3754658262287391e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0039
┌ Info: Step  = 40
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 7.4588913905435972e-01
└ sum error   = 6.5359466593832383e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0040
┌ Info: Step  = 41
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 7.3338795387654188e-01
└ sum error   = 6.6964274925377374e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0041
┌ Info: Step  = 42
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 7.0116620629161319e-01
└ sum error   = 6.8569083256922366e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0042
┌ Info: Step  = 43
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.8853508573783040e-01
└ sum error   = 7.0319783254971449e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0043
┌ Info: Step  = 44
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.8181915727974196e-01
└ sum error   = 7.1778699920012349e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0044
┌ Info: Step  = 45
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.9714249155896013e-01
└ sum error   = 7.3383508251557341e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0045
┌ Info: Step  = 46
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 7.1373493773062968e-01
└ sum error   = 7.5134208249606424e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0046
┌ Info: Step  = 47
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 7.0589709549120550e-01
└ sum error   = 7.6739016581151415e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0047
┌ Info: Step  = 48
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.7561835188200325e-01
└ sum error   = 7.8343824912696407e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0048
┌ Info: Step  = 49
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.6799633670242475e-01
└ sum error   = 7.9948633244241398e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0049
┌ Info: Step  = 50
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.6317595158268261e-01
└ sum error   = 8.1699333242290481e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0050
┌ Info: Step  = 51
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.7131152798099103e-01
└ sum error   = 8.3304141573835473e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0051
┌ Info: Step  = 52
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.6378067520168349e-01
└ sum error   = 8.5054841571884556e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0052
┌ Info: Step  = 53
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.9869715023895917e-01
└ sum error   = 8.6659649903429548e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0053
┌ Info: Step  = 54
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.5297914322790285e-01
└ sum error   = 8.8410349901478630e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0054
┌ Info: Step  = 55
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.5949707880381958e-01
└ sum error   = 9.0015158233023622e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0055
┌ Info: Step  = 56
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.7760891760906261e-01
└ sum error   = 9.1765858231072705e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0056
┌ Info: Step  = 57
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.9141938302931616e-01
└ sum error   = 9.3370666562617697e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0057
┌ Info: Step  = 58
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.9215686852713587e-01
└ sum error   = 9.5267258227170871e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0058
┌ Info: Step  = 59
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.9411474744347224e-01
└ sum error   = 9.7017958225219941e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0059
┌ Info: Step  = 60
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.7943739252826685e-01
└ sum error   = 9.8768658223269024e-14
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0060
┌ Info: Step  = 61
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.3388282089000025e-01
└ sum error   = 1.0051935822131811e-13
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0061
┌ Info: Step  = 62
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.3707600261526320e-01
└ sum error   = 1.0212416655286310e-13
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0062
┌ Info: Step  = 63
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 5.9847900007097110e-01
└ sum error   = 1.0387486655091218e-13
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0063
┌ Info: Step  = 64
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.1884186669957453e-01
└ sum error   = 1.0562556654896126e-13
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0064
┌ Info: Step  = 65
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.2154672529481547e-01
└ sum error   = 1.0723037488050626e-13
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0065
┌ Info: Step  = 66
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.2506070108437728e-01
└ sum error   = 1.0898107487855534e-13
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0066
┌ Info: Step  = 67
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.1743383359325055e-01
└ sum error   = 1.1087766654310851e-13
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0067
┌ Info: Step  = 68
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.1902486341166585e-01
└ sum error   = 1.1262836654115760e-13
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0068
┌ Info: Step  = 69
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.1969947826326266e-01
└ sum error   = 1.1437906653920667e-13
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0069
┌ Info: Step  = 70
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.1841812756151116e-01
└ sum error   = 1.1627565820375984e-13
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0070
┌ Info: Step  = 71
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.2270482223621626e-01
└ sum error   = 1.1802635820180894e-13
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0071
┌ Info: Step  = 72
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.2018447181832426e-01
└ sum error   = 1.1977705819985801e-13
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0072
┌ Info: Step  = 73
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.2119855246752198e-01
└ sum error   = 1.2167364986441118e-13
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0073
┌ Info: Step  = 74
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.2613101937125781e-01
└ sum error   = 1.2357024152896435e-13
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0074
┌ Info: Step  = 75
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.2378142047610285e-01
└ sum error   = 1.2546683319351753e-13
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0075
┌ Info: Step  = 76
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.2211409024629449e-01
└ sum error   = 1.2721753319156660e-13
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0076
┌ Info: Step  = 77
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.1966300075277103e-01
└ sum error   = 1.2896823318961569e-13
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0077
┌ Info: Step  = 78
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.1825712813613931e-01
└ sum error   = 1.3086482485416887e-13
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0078
┌ Info: Step  = 79
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.1858716713682049e-01
└ sum error   = 1.3246963318571386e-13
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0079
┌ Info: Step  = 80
│ minimum(Q)  = 0.0000000000000000e+00
│ maximum(Q)  = 6.1492238702067981e-01
└ sum error   = 1.3422033318376293e-13
[ Info: Done writing VTK: /central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192364_output/vtk_nonnegative/nonnegative_step0080
┌ Info: Finished
│ minimum(Q) = 0.0000000000000000e+00
│ maximum(Q) = 6.1492238702067981e-01
│ L2 error   = 4.2308574875971561e-02
└ sum error  = 1.3422033318376293e-13

out_10192365

+ hostname
hpc-22-12.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192365_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192365_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Atmos/Parameterizations/Microphysics/KM_saturation_adjustment.jl
┌ Info: Model composition
│     param_set = EarthParameterSet{MicropysicsParameterSet{LiquidParameterSet,IceParameterSet,RainParameterSet,SnowParameterSet}}(MicropysicsParameterSet{LiquidParameterSet,IceParameterSet,RainParameterSet,SnowParameterSet}(LiquidParameterSet(), IceParameterSet(), RainParameterSet(), SnowParameterSet()))
│     orientation = FlatOrientation()
│     moisture = nothing
│     precipitation = nothing
│     source = nothing
│     boundarycondition = nothing
│     init_state_conservative = init_kinematic_eddy!
└     data_config = KinematicModelConfig{Float64}(1500.0, 10.0, 1500.0, 0.6, 289.0, 101500.0, 100000.0, 0.0075, 0.0, true, true, false, 0, 0, 0, 0, 1, 2)
┌ Info: Establishing Atmos LES configuration for KinematicModel
│     precision        = Float64
│     polynomial order = 4
│     domain           = 1500.00 m x10.00 m x1500.00 m
│     resolution       = 20x1x20
│     MPI ranks        = 3
│     min(Δ_horz)      = 0.86 m
└     min(Δ_vert)      = 13.63 m
[ Info: Initializing KinematicModel
┌ Info: Starting KinematicModel
│     dt              = 4.00000e+01
│     timeend         =  1800.00
│     number of steps = 45
└     norm(Q)         = 1.7391051223178524e+08
┌ Info: doing VTK output
└   out_path_prefix = "/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192365_output/vtk/new_ex_1_mpirank0000_step0000"
┌ Info: doing VTK output
└   out_path_prefix = "/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192365_output/vtk/new_ex_1_mpirank0000_step0001"
┌ Info: doing VTK output
└   out_path_prefix = "/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192365_output/vtk/new_ex_1_mpirank0000_step0002"
┌ Info: doing VTK output
└   out_path_prefix = "/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192365_output/vtk/new_ex_1_mpirank0000_step0003"
┌ Info: doing VTK output
└   out_path_prefix = "/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192365_output/vtk/new_ex_1_mpirank0000_step0004"
┌ Info: Finished
│     norm(Q)            = 1.7391051217590663e+08
│     norm(Q) / norm(Q₀) = 9.9999999967869335e-01
└     norm(Q) - norm(Q₀) = -5.5878609418869019e-02
┌ Info: Euclidean distance
│     norm(Q - Qe)            = 7.7535673575889223e+03
└     norm(Q - Qe) / norm(Qe) = 4.4583661206489277e-05

out_10192366

+ hostname
hpc-22-12.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192366_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192366_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Atmos/Parameterizations/Microphysics/KM_warm_rain.jl
┌ Info: Model composition
│     param_set = EarthParameterSet{MicropysicsParameterSet{LiquidParameterSet,IceParameterSet,RainParameterSet,SnowParameterSet}}(MicropysicsParameterSet{LiquidParameterSet,IceParameterSet,RainParameterSet,SnowParameterSet}(LiquidParameterSet(), IceParameterSet(), RainParameterSet(), SnowParameterSet()))
│     orientation = FlatOrientation()
│     moisture = nothing
│     precipitation = nothing
│     source = nothing
│     boundarycondition = nothing
│     init_state_conservative = init_kinematic_eddy!
└     data_config = KinematicModelConfig{Float64}(1500.0, 10.0, 1500.0, 0.6, 289.0, 101500.0, 100000.0, 0.0075, 0.0, true, true, false, 0, 0, 0, 0, 1, 2)
┌ Info: Establishing Atmos LES configuration for KinematicModel
│     precision        = Float64
│     polynomial order = 4
│     domain           = 1500.00 m x10.00 m x1500.00 m
│     resolution       = 20x1x20
│     MPI ranks        = 3
│     min(Δ_horz)      = 0.86 m
└     min(Δ_vert)      = 13.63 m
[ Info: Initializing KinematicModel
┌ Info: Starting KinematicModel
│     dt              = 5.00000e+00
│     timeend         =  1800.00
│     number of steps = 360
└     norm(Q)         = 1.7391051223178524e+08
┌ Info: Update
│     simtime =   135.00 /  1800.00
│     runtime = 00:01:00
└     norm(Q) = 1.7383498794969824e+08
┌ Info: doing VTK output
└   out_path_prefix = "/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192366_output/vtk/microphysics_test_3_mpirank0000_step0000"
┌ Info: Update
│     simtime =   360.00 /  1800.00
│     runtime = 00:02:03
└     norm(Q) = 1.7373918123467049e+08
┌ Info: Update
│     simtime =   545.00 /  1800.00
│     runtime = 00:03:04
└     norm(Q) = 1.7374027848611286e+08
┌ Info: Update
│     simtime =   710.00 /  1800.00
│     runtime = 00:04:04
└     norm(Q) = 1.7374602613458058e+08
┌ Info: doing VTK output
└   out_path_prefix = "/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192366_output/vtk/microphysics_test_3_mpirank0000_step0001"
┌ Info: Update
│     simtime =   875.00 /  1800.00
│     runtime = 00:05:06
└     norm(Q) = 1.7375094661547285e+08
┌ Info: Update
│     simtime =  1030.00 /  1800.00
│     runtime = 00:06:06
└     norm(Q) = 1.7375460511889982e+08
┌ Info: doing VTK output
└   out_path_prefix = "/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192366_output/vtk/microphysics_test_3_mpirank0000_step0002"
┌ Info: Update
│     simtime =  1185.00 /  1800.00
│     runtime = 00:07:06
└     norm(Q) = 1.7375723936354479e+08
┌ Info: Update
│     simtime =  1345.00 /  1800.00
│     runtime = 00:08:07
└     norm(Q) = 1.7375880676253846e+08
┌ Info: doing VTK output
└   out_path_prefix = "/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192366_output/vtk/microphysics_test_3_mpirank0000_step0003"
┌ Info: Update
│     simtime =  1510.00 /  1800.00
│     runtime = 00:09:08
└     norm(Q) = 1.7375920745061007e+08
┌ Info: Update
│     simtime =  1685.00 /  1800.00
│     runtime = 00:10:10
└     norm(Q) = 1.7375829416357905e+08
┌ Info: doing VTK output
└   out_path_prefix = "/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192366_output/vtk/microphysics_test_3_mpirank0000_step0004"
┌ Info: Finished
│     norm(Q)            = 1.7375700427664909e+08
│     norm(Q) / norm(Q₀) = 9.9911731641080126e-01
└     norm(Q) - norm(Q₀) = -1.5350795513615012e+05
┌ Info: Euclidean distance
│     norm(Q - Qe)            = 1.8820344817866494e+06
└     norm(Q - Qe) / norm(Qe) = 1.0821855778782958e-02

out_10192367

+ hostname
hpc-22-12.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_cpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ CLIMATEMACHINE_SETTINGS_DISABLE_GPU=true
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192367_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192367_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 openmpi/4.0.3 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include ';export' 'CPATH;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3//include:/central/software/julia/1.4.2//include
++ export CPATH
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3//lib
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3//lib
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:openmpi/4.0.3:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3//share/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3//bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/mpi/openmpi/4.0.3:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Atmos/Parameterizations/Microphysics/KM_ice.jl
┌ Info: Model composition
│     param_set = EarthParameterSet{MicropysicsParameterSet{LiquidParameterSet,IceParameterSet,RainParameterSet,SnowParameterSet}}(MicropysicsParameterSet{LiquidParameterSet,IceParameterSet,RainParameterSet,SnowParameterSet}(LiquidParameterSet(), IceParameterSet(), RainParameterSet(), SnowParameterSet()))
│     orientation = FlatOrientation()
│     moisture = nothing
│     precipitation = nothing
│     source = nothing
│     boundarycondition = nothing
│     init_state_conservative = init_kinematic_eddy!
└     data_config = KinematicModelConfig{Float64}(90000.0, 10.0, 16000.0, 0.6, 289.0, 101500.0, 100000.0, 0.0075, 0.0, false, true, false, 1, 2, 3, 4, 5, 6)
┌ Info: Establishing Atmos LES configuration for KinematicModel
│     precision        = Float64
│     polynomial order = 4
│     domain           = 90000.00 m x10.00 m x16000.00 m
│     resolution       = 500x1x250
│     MPI ranks        = 3
│     min(Δ_horz)      = 0.86 m
└     min(Δ_vert)      = 172.67 m
[ Info: Initializing KinematicModel
┌ Info: Starting KinematicModel
│     dt              = 1.50000e+01
│     timeend         =   300.00
│     number of steps = 20
└     norm(Q)         = 4.2991622816819172e+09
┌ Info: doing VTK output
└   out_path_prefix = "/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192367_output/vtk/microphysics_test_4_mpirank0000_step0000"
┌ Info: doing VTK output
└   out_path_prefix = "/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192367_output/vtk/microphysics_test_4_mpirank0000_step0001"
┌ Info: Update
│     simtime =   150.00 /   300.00
│     runtime = 00:01:00
└     norm(Q) = 4.3012290919816294e+09
┌ Info: doing VTK output
└   out_path_prefix = "/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192367_output/vtk/microphysics_test_4_mpirank0000_step0002"
┌ Info: doing VTK output
└   out_path_prefix = "/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192367_output/vtk/microphysics_test_4_mpirank0000_step0003"
┌ Info: doing VTK output
└   out_path_prefix = "/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192367_output/vtk/microphysics_test_4_mpirank0000_step0004"
┌ Info: Finished
│     norm(Q)            = 4.3034600134843979e+09
│     norm(Q) / norm(Q₀) = 1.0009996672655956e+00
└     norm(Q) - norm(Q₀) = 4.2977318024806976e+06
┌ Info: Euclidean distance
│     norm(Q - Qe)            = 5.6475959073576167e+07
└     norm(Q - Qe) / norm(Qe) = 1.3136503200684404e-02

out_10192368

+ hostname
hpc-23-28.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
++ pwd
+ export CURL_HOME=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
+ CURL_HOME=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
+ printf -- '--silent\n--show-error\n'
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ julia --color=no --project -e 'using Pkg; Pkg.instantiate(); Pkg.build(;verbose=true)'
[?25l    Fetching: [>                                        ]  0.0 %
    Fetching: [===================>                     ]  45.7 %
    Resolving Deltas: [===================================>     ]  85.8 %
[?25h[ Info: using system MPI
┌ Info: Using implementation
│   libmpi = "libmpi"
└   mpiexec_cmd = `mpiexec`
┌ Info: MPI implementation detected
│   impl = OpenMPI::MPIImpl = 2
│   version = v"4.0.3"
└   abi = "OpenMPI"
africa
antarctica
asia
australasia
europe
northamerica
southamerica
etcetera
backward
pacificnew
[ Info: Extracting 2020a tzdata archive
[ Info: Converting tz source files into TimeZone data
[ Info: Successfully built TimeZones
    Cloning default registries into `/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu`
    Cloning registry from "https://github.com/JuliaRegistries/General.git"
      Added registry `General` to `/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu/registries/General`
  Installed XML2_jll ───────────────────── v2.9.9+5
  Installed OpenMPI_jll ────────────────── v4.0.2+2
  Installed OpenSpecFun_jll ────────────── v0.5.3+3
  Installed CondaBinDeps ───────────────── v0.2.0
  Installed GaussQuadrature ────────────── v0.5.4
  Installed LightXML ───────────────────── v0.9.0
  Installed DataAPI ────────────────────── v1.3.0
  Installed HTTP ───────────────────────── v0.8.15
  Installed OpenBLAS_jll ───────────────── v0.3.9+4
  Installed Adapt ──────────────────────── v2.0.2
  Installed URIParser ──────────────────── v0.4.1
  Installed DiffResults ────────────────── v1.0.2
  Installed CLIMAParameters ────────────── v0.1.6
  Installed ArgParse ───────────────────── v1.1.0
  Installed StatsBase ──────────────────── v0.33.0
  Installed ForwardDiff ────────────────── v0.10.10
  Installed PDMats ─────────────────────── v0.9.12
  Installed CodecZlib ──────────────────── v0.7.0
  Installed Combinatorics ──────────────── v1.0.2
  Installed MicrosoftMPI_jll ───────────── v10.1.2+3
  Installed NNPACK_jll ─────────────────── v2018.6.22+0
  Installed VersionParsing ─────────────── v1.2.0
  Installed Rmath_jll ──────────────────── v0.2.2+1
  Installed PackageCompiler ────────────── v1.1.1
  Installed Mocking ────────────────────── v0.7.1
  Installed DataStructures ─────────────── v0.17.18
  Installed KernelAbstractions ─────────── v0.3.0
  Installed Arpack ─────────────────────── v0.4.0
  Installed StaticNumbers ──────────────── v0.3.3
  Installed Distributions ──────────────── v0.23.4
  Installed GenericSchur ───────────────── v0.4.0
  Installed BinDeps ────────────────────── v1.0.1
  Installed Coverage ───────────────────── v1.1.1
  Installed MatrixFactorizations ───────── v0.4.1
  Installed BinaryProvider ─────────────── v0.5.10
  Installed Intervals ──────────────────── v1.2.0
  Installed GPUCompiler ────────────────── v0.4.0
  Installed IniFile ────────────────────── v0.5.0
  Installed Rmath ──────────────────────── v0.6.1
  Installed Literate ───────────────────── v2.5.0
  Installed EzXML ──────────────────────── v1.1.0
  Installed Arpack_jll ─────────────────── v3.5.0+3
  Installed WriteVTK ───────────────────── v1.6.0
  Installed MbedTLS ────────────────────── v1.0.2
  Installed SortingAlgorithms ──────────── v0.3.1
  Installed TimerOutputs ───────────────── v0.5.6
  Installed MPICH_jll ──────────────────── v3.3.2+10
  Installed QuadGK ─────────────────────── v2.3.1
  Installed DoubleFloats ───────────────── v1.1.12
  Installed ArrayLayouts ───────────────── v0.3.5
  Installed LazyArrays ─────────────────── v0.16.13
  Installed Polynomials ────────────────── v1.1.1
  Installed Libiconv_jll ───────────────── v1.16.0+4
  Installed MacroTools ─────────────────── v0.5.5
  Installed OrderedCollections ─────────── v1.2.0
  Installed Reexport ───────────────────── v0.2.0
  Installed MPI ────────────────────────── v0.15.0
  Installed ExprTools ──────────────────── v0.1.1
  Installed Cassette ───────────────────── v0.3.3
  Installed FillArrays ─────────────────── v0.8.11
  Installed CEnum ──────────────────────── v0.4.1
  Installed TranscodingStreams ─────────── v0.9.5
  Installed StatsFuns ──────────────────── v0.9.5
  Installed StaticArrays ───────────────── v0.12.3
  Installed GenericSVD ─────────────────── v0.3.0
  Installed NNlib ──────────────────────── v0.7.0
  Installed NaNMath ────────────────────── v0.3.3
  Installed CompilerSupportLibraries_jll ─ v0.3.3+0
  Installed DiffRules ──────────────────── v1.0.1
  Installed LLVM ───────────────────────── v1.6.0
  Installed Parsers ────────────────────── v1.0.5
  Installed Missings ───────────────────── v0.4.3
  Installed Dierckx ────────────────────── v0.4.2
  Installed Zlib_jll ───────────────────── v1.2.11+12
  Installed MbedTLS_jll ────────────────── v2.16.0+2
  Installed FileIO ─────────────────────── v1.3.0
  Installed AbstractFFTs ───────────────── v0.5.0
  Installed DocStringExtensions ────────── v0.8.2
  Installed SpecialFunctions ───────────── v0.10.3
  Installed TimeZones ──────────────────── v1.2.0
  Installed CUDA ───────────────────────── v1.0.2
  Installed TextWrap ───────────────────── v1.0.1
  Installed CFTime ─────────────────────── v0.1.0
  Installed RootSolvers ────────────────── v0.2.0
  Installed Conda ──────────────────────── v1.4.1
  Installed CommonSubexpressions ───────── v0.2.0
  Installed Formatting ─────────────────── v0.4.1
  Installed Quadmath ───────────────────── v0.5.4
  Installed NCDatasets ─────────────────── v0.10.2
  Installed GPUArrays ──────────────────── v4.0.0
  Installed Requires ───────────────────── v1.0.1
  Installed JSON ───────────────────────── v0.21.0
  Installed CoverageTools ──────────────── v1.1.0
  Installed RecipesBase ────────────────── v1.0.1
  Installed JLD2 ───────────────────────── v0.1.13
   Building MPI ───────→ `/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu/packages/MPI/k7f4E/deps/build.log`
   Building Dierckx ───→ `/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu/packages/Dierckx/ZTanR/deps/build.log`
   Building TimeZones ─→ `/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu/packages/TimeZones/OjMoF/deps/build.log`
   Building Conda ─────→ `/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu/packages/Conda/3rPhK/deps/build.log`
   Building NCDatasets → `/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu/packages/NCDatasets/KjkFI/deps/build.log`
   Building MPI ───────→ `/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu/packages/MPI/k7f4E/deps/build.log`
   Building TimeZones ─→ `/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu/packages/TimeZones/OjMoF/deps/build.log`
   Building Dierckx ───→ `/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu/packages/Dierckx/ZTanR/deps/build.log`
   Building Conda ─────→ `/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu/packages/Conda/3rPhK/deps/build.log`
   Building NCDatasets → `/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu/packages/NCDatasets/KjkFI/deps/build.log`
+ julia --color=no --project -e 'using Pkg; Pkg.precompile()'
Precompiling project...
+ cat Manifest.toml
# This file is machine-generated - editing it directly is not advised

[[AbstractFFTs]]
deps = ["LinearAlgebra"]
git-tree-sha1 = "051c95d6836228d120f5f4b984dd5aba1624f716"
uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c"
version = "0.5.0"

[[Adapt]]
deps = ["LinearAlgebra"]
git-tree-sha1 = "0fac443759fa829ed8066db6cf1077d888bb6573"
uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
version = "2.0.2"

[[ArgParse]]
deps = ["Logging", "TextWrap"]
git-tree-sha1 = "a8fc2e149cd6db276c76faebe197ccd3a92fb9ff"
uuid = "c7e460c6-2fb9-53a9-8c5b-16f535851c63"
version = "1.1.0"

[[Arpack]]
deps = ["Arpack_jll", "Libdl", "LinearAlgebra"]
git-tree-sha1 = "2ff92b71ba1747c5fdd541f8fc87736d82f40ec9"
uuid = "7d9fca2a-8960-54d3-9f78-7d1dccf2cb97"
version = "0.4.0"

[[Arpack_jll]]
deps = ["Libdl", "OpenBLAS_jll", "Pkg"]
git-tree-sha1 = "e214a9b9bd1b4e1b4f15b22c0994862b66af7ff7"
uuid = "68821587-b530-5797-8361-c406ea357684"
version = "3.5.0+3"

[[ArrayLayouts]]
deps = ["FillArrays", "LinearAlgebra"]
git-tree-sha1 = "a3254b3780a3544838ca0b7e23b1e9b06eb71bd8"
uuid = "4c555306-a7a7-4459-81d9-ec55ddd5c99a"
version = "0.3.5"

[[Base64]]
uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

[[BinDeps]]
deps = ["Libdl", "Pkg", "SHA", "URIParser", "Unicode"]
git-tree-sha1 = "46cf2c1668ad07aba5a9d331bdeea994a1f13856"
uuid = "9e28174c-4ba2-5203-b857-d8d62c4213ee"
version = "1.0.1"

[[BinaryProvider]]
deps = ["Libdl", "Logging", "SHA"]
git-tree-sha1 = "ecdec412a9abc8db54c0efc5548c64dfce072058"
uuid = "b99e7846-7c00-51b0-8f62-c81ae34c0232"
version = "0.5.10"

[[CEnum]]
git-tree-sha1 = "215a9aa4a1f23fbd05b92769fdd62559488d70e9"
uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82"
version = "0.4.1"

[[CFTime]]
deps = ["Dates", "Printf"]
git-tree-sha1 = "143ef231a14c2ab1ac7e838c559ee1e45c0d1b57"
uuid = "179af706-886a-5703-950a-314cd64e0468"
version = "0.1.0"

[[CLIMAParameters]]
deps = ["Test"]
git-tree-sha1 = "314f0bdd5817abb2df61cc8d3e857c496d421109"
uuid = "6eacf6c3-8458-43b9-ae03-caf5306d3d53"
version = "0.1.6"

[[CUDA]]
deps = ["AbstractFFTs", "Adapt", "BinaryProvider", "CEnum", "DataStructures", "ExprTools", "GPUArrays", "GPUCompiler", "LLVM", "Libdl", "LinearAlgebra", "Logging", "MacroTools", "NNlib", "Pkg", "Printf", "Random", "Reexport", "Requires", "SparseArrays", "Statistics", "TimerOutputs"]
git-tree-sha1 = "d658b3881a25b317ea7fb128efbd82b5e63396ad"
uuid = "052768ef-5323-5732-b1bb-66c8b64840ba"
version = "1.0.2"

[[Cassette]]
git-tree-sha1 = "ff6f5109371926beb67ec3101be17d2c211e497d"
uuid = "7057c7e9-c182-5462-911a-8362d720325c"
version = "0.3.3"

[[CodecZlib]]
deps = ["TranscodingStreams", "Zlib_jll"]
git-tree-sha1 = "ded953804d019afa9a3f98981d99b33e3db7b6da"
uuid = "944b1d66-785c-5afd-91f1-9de20f533193"
version = "0.7.0"

[[Combinatorics]]
git-tree-sha1 = "08c8b6831dc00bfea825826be0bc8336fc369860"
uuid = "861a8166-3701-5b0c-9a16-15d98fcdc6aa"
version = "1.0.2"

[[CommonSubexpressions]]
deps = ["Test"]
git-tree-sha1 = "efdaf19ab11c7889334ca247ff4c9f7c322817b0"
uuid = "bbf7d656-a473-5ed7-a52c-81e309532950"
version = "0.2.0"

[[CompilerSupportLibraries_jll]]
deps = ["Libdl", "Pkg"]
git-tree-sha1 = "7c4f882c41faa72118841185afc58a2eb00ef612"
uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"
version = "0.3.3+0"

[[Conda]]
deps = ["JSON", "VersionParsing"]
git-tree-sha1 = "7a58bb32ce5d85f8bf7559aa7c2842f9aecf52fc"
uuid = "8f4d0f93-b110-5947-807f-2305c1781a2d"
version = "1.4.1"

[[CondaBinDeps]]
deps = ["BinDeps", "Conda"]
git-tree-sha1 = "25f750df2893991f2c9b18425bfac6f2ce855154"
uuid = "a9693cdc-2bc8-5703-a9cd-1da358117377"
version = "0.2.0"

[[Coverage]]
deps = ["CoverageTools", "HTTP", "JSON", "LibGit2", "MbedTLS"]
git-tree-sha1 = "aad69c0c2d8080427f507c36b289c77fe465c9fe"
uuid = "a2441757-f6aa-5fb2-8edb-039e3f45d037"
version = "1.1.1"

[[CoverageTools]]
deps = ["LibGit2"]
git-tree-sha1 = "328154f6062b0053588295df6bffe22e1436597a"
uuid = "c36e975a-824b-4404-a568-ef97ca766997"
version = "1.1.0"

[[DataAPI]]
git-tree-sha1 = "176e23402d80e7743fc26c19c681bfb11246af32"
uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
version = "1.3.0"

[[DataStructures]]
deps = ["InteractiveUtils", "OrderedCollections"]
git-tree-sha1 = "be680f1ad03c0a03796aa3fda5a2180df7f83b46"
uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
version = "0.17.18"

[[Dates]]
deps = ["Printf"]
uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"

[[DelimitedFiles]]
deps = ["Mmap"]
uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab"

[[Dierckx]]
deps = ["BinaryProvider", "Libdl", "Random", "Test"]
git-tree-sha1 = "956bfe5c4f9b794545f6527424fc0294cc5decbe"
uuid = "39dd38d3-220a-591b-8e3c-4c3a8c710a94"
version = "0.4.2"

[[DiffResults]]
deps = ["StaticArrays"]
git-tree-sha1 = "da24935df8e0c6cf28de340b958f6aac88eaa0cc"
uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5"
version = "1.0.2"

[[DiffRules]]
deps = ["NaNMath", "Random", "SpecialFunctions"]
git-tree-sha1 = "eb0c34204c8410888844ada5359ac8b96292cfd1"
uuid = "b552c78f-8df3-52c6-915a-8e097449b14b"
version = "1.0.1"

[[Distributed]]
deps = ["Random", "Serialization", "Sockets"]
uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"

[[Distributions]]
deps = ["FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SpecialFunctions", "Statistics", "StatsBase", "StatsFuns"]
git-tree-sha1 = "78c4c32a2357a00a0a7d614880f02c2c6e1ec73c"
uuid = "31c24e10-a181-5473-b8eb-7969acd0382f"
version = "0.23.4"

[[DocStringExtensions]]
deps = ["LibGit2", "Markdown", "Pkg", "Test"]
git-tree-sha1 = "c5714d9bcdba66389612dc4c47ed827c64112997"
uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
version = "0.8.2"

[[DoubleFloats]]
deps = ["GenericSVD", "GenericSchur", "LinearAlgebra", "Polynomials", "Printf", "Quadmath", "Random", "Requires", "SpecialFunctions"]
git-tree-sha1 = "a73770c1418f8a7c4c127328e0cd34fdcf0a1d29"
uuid = "497a8b3b-efae-58df-a0af-a86822472b78"
version = "1.1.12"

[[ExprTools]]
git-tree-sha1 = "6f0517056812fd6aa3af23d4b70d5325a2ae4e95"
uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04"
version = "0.1.1"

[[EzXML]]
deps = ["Printf", "XML2_jll"]
git-tree-sha1 = "0fa3b52a04a4e210aeb1626def9c90df3ae65268"
uuid = "8f5d6c58-4d21-5cfd-889c-e3ad7ee6a615"
version = "1.1.0"

[[FileIO]]
deps = ["Pkg"]
git-tree-sha1 = "202335fd24c2776493e198d6c66a6d910400a895"
uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
version = "1.3.0"

[[FillArrays]]
deps = ["LinearAlgebra", "Random", "SparseArrays"]
git-tree-sha1 = "bf726ba7ce99e00d10bf63c031285fb9ab3676ae"
uuid = "1a297f60-69ca-5386-bcde-b61e274b549b"
version = "0.8.11"

[[Formatting]]
deps = ["Printf"]
git-tree-sha1 = "a0c901c29c0e7c763342751c0a94211d56c0de5c"
uuid = "59287772-0a20-5a39-b81b-1366585eb4c0"
version = "0.4.1"

[[ForwardDiff]]
deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "NaNMath", "Random", "SpecialFunctions", "StaticArrays"]
git-tree-sha1 = "869540e4367122fbffaace383a5bdc34d6e5e5ac"
uuid = "f6369f11-7733-5829-9624-2563aa707210"
version = "0.10.10"

[[GPUArrays]]
deps = ["AbstractFFTs", "Adapt", "LinearAlgebra", "Printf", "Random", "Serialization"]
git-tree-sha1 = "ae20accf251c6da038fe33cbc2c53d5af7f51344"
uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7"
version = "4.0.0"

[[GPUCompiler]]
deps = ["DataStructures", "InteractiveUtils", "LLVM", "Libdl", "TimerOutputs", "UUIDs"]
git-tree-sha1 = "65f7395a1245635f0c2279649fdbef09a1b0aa7b"
uuid = "61eb1bfa-7361-4325-ad38-22787b887f55"
version = "0.4.0"

[[GaussQuadrature]]
deps = ["SpecialFunctions"]
git-tree-sha1 = "ff30c61ad1ff425d0d8bac6b77833216741c2e68"
uuid = "d54b0c1a-921d-58e0-8e36-89d8069c0969"
version = "0.5.4"

[[GenericSVD]]
deps = ["LinearAlgebra"]
git-tree-sha1 = "62909c3eda8a25b5673a367d1ad2392ebb265211"
uuid = "01680d73-4ee2-5a08-a1aa-533608c188bb"
version = "0.3.0"

[[GenericSchur]]
deps = ["LinearAlgebra", "Printf"]
git-tree-sha1 = "43b4dc5648028be2c6e96201aa3653903bd1af21"
uuid = "c145ed77-6b09-5dd9-b285-bf645a82121e"
version = "0.4.0"

[[HTTP]]
deps = ["Base64", "Dates", "IniFile", "MbedTLS", "Sockets"]
git-tree-sha1 = "ec87d5e2acbe1693789efbbe14f5ea7525758f71"
uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3"
version = "0.8.15"

[[IniFile]]
deps = ["Test"]
git-tree-sha1 = "098e4d2c533924c921f9f9847274f2ad89e018b8"
uuid = "83e8ac13-25f8-5344-8a64-a9f2b223428f"
version = "0.5.0"

[[InteractiveUtils]]
deps = ["Markdown"]
uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"

[[Intervals]]
deps = ["Dates", "Printf", "RecipesBase", "TimeZones"]
git-tree-sha1 = "11555ad4a66a93e2268364f2ea11139634f9dda3"
uuid = "d8418881-c3e1-53bb-8760-2df7ec849ed5"
version = "1.2.0"

[[JLD2]]
deps = ["CodecZlib", "DataStructures", "FileIO", "Mmap", "Pkg", "Printf", "UUIDs"]
git-tree-sha1 = "e03e697cf84c275ece9cbefd1eabaf49bf5e7254"
uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
version = "0.1.13"

[[JSON]]
deps = ["Dates", "Mmap", "Parsers", "Unicode"]
git-tree-sha1 = "b34d7cef7b337321e97d22242c3c2b91f476748e"
uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
version = "0.21.0"

[[KernelAbstractions]]
deps = ["Adapt", "CUDA", "Cassette", "LLVM", "MacroTools", "SpecialFunctions", "StaticArrays"]
git-tree-sha1 = "6e812284039b7df4c32845d6d23fff9afc29d0ee"
uuid = "63c18a36-062a-441e-b654-da1e3ab1ce7c"
version = "0.3.0"

[[LLVM]]
deps = ["CEnum", "Libdl", "Printf", "Unicode"]
git-tree-sha1 = "0efb86dc7db298459186546bab008f3b53157582"
uuid = "929cbde3-209d-540e-8aea-75f648917ca0"
version = "1.6.0"

[[LazyArrays]]
deps = ["ArrayLayouts", "FillArrays", "LinearAlgebra", "MacroTools", "MatrixFactorizations", "StaticArrays"]
git-tree-sha1 = "eb02bd7606db9516205b400da8ba01e829ed5f7b"
uuid = "5078a376-72f3-5289-bfd5-ec5146d43c02"
version = "0.16.13"

[[LibGit2]]
deps = ["Printf"]
uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"

[[Libdl]]
uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"

[[Libiconv_jll]]
deps = ["Libdl", "Pkg"]
git-tree-sha1 = "a06937d9aa48855f52484c9a77a2670158fa90e2"
uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531"
version = "1.16.0+4"

[[LightXML]]
deps = ["Libdl", "XML2_jll"]
git-tree-sha1 = "e129d9391168c677cd4800f5c0abb1ed8cb3794f"
uuid = "9c8b4983-aa76-5018-a973-4c85ecc9e179"
version = "0.9.0"

[[LinearAlgebra]]
deps = ["Libdl"]
uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"

[[Literate]]
deps = ["Base64", "JSON", "REPL"]
git-tree-sha1 = "422133037d6dc5df9f9b97c2cb81fcd9e35ddffe"
uuid = "98b081ad-f1c9-55d3-8b20-4c87d4299306"
version = "2.5.0"

[[Logging]]
uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"

[[MPI]]
deps = ["Distributed", "DocStringExtensions", "Libdl", "MPICH_jll", "MicrosoftMPI_jll", "OpenMPI_jll", "Pkg", "Random", "Requires", "Serialization", "Sockets"]
git-tree-sha1 = "925f4e00b51d7a68ecb88c1b60d3b7aefb8c8fcb"
uuid = "da04e1cc-30fd-572f-bb4f-1f8673147195"
version = "0.15.0"

[[MPICH_jll]]
deps = ["CompilerSupportLibraries_jll", "Libdl", "Pkg"]
git-tree-sha1 = "4d37f1e07b4e2a74462eebf9ee48c626d15ffdac"
uuid = "7cb0a576-ebde-5e09-9194-50597f1243b4"
version = "3.3.2+10"

[[MacroTools]]
deps = ["Markdown", "Random"]
git-tree-sha1 = "f7d2e3f654af75f01ec49be82c231c382214223a"
uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
version = "0.5.5"

[[Markdown]]
deps = ["Base64"]
uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"

[[MatrixFactorizations]]
deps = ["ArrayLayouts", "LinearAlgebra", "Random"]
git-tree-sha1 = "69454f168628adeff74264837fc4c9d258dcc444"
uuid = "a3b82374-2e81-5b9e-98ce-41277c0e4c87"
version = "0.4.1"

[[MbedTLS]]
deps = ["Dates", "MbedTLS_jll", "Random", "Sockets"]
git-tree-sha1 = "426a6978b03a97ceb7ead77775a1da066343ec6e"
uuid = "739be429-bea8-5141-9913-cc70e7f3736d"
version = "1.0.2"

[[MbedTLS_jll]]
deps = ["Libdl", "Pkg"]
git-tree-sha1 = "c83f5a1d038f034ad0549f9ee4d5fac3fb429e33"
uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
version = "2.16.0+2"

[[MicrosoftMPI_jll]]
deps = ["Libdl", "Pkg"]
git-tree-sha1 = "720de13004e416f2864c92593d3839e062703985"
uuid = "9237b28f-5490-5468-be7b-bb81f5f5e6cf"
version = "10.1.2+3"

[[Missings]]
deps = ["DataAPI"]
git-tree-sha1 = "de0a5ce9e5289f27df672ffabef4d1e5861247d5"
uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28"
version = "0.4.3"

[[Mmap]]
uuid = "a63ad114-7e13-5084-954f-fe012c677804"

[[Mocking]]
deps = ["ExprTools"]
git-tree-sha1 = "916b850daad0d46b8c71f65f719c49957e9513ed"
uuid = "78c3b35d-d492-501b-9361-3d52fe80e533"
version = "0.7.1"

[[NCDatasets]]
deps = ["BinDeps", "CFTime", "CondaBinDeps", "DataStructures", "Dates", "Libdl", "Printf"]
git-tree-sha1 = "271e4670a6b30ec1e5daf6ba8b30240b7d85f535"
uuid = "85f8d34a-cbdd-5861-8df4-14fed0d494ab"
version = "0.10.2"

[[NNPACK_jll]]
deps = ["Libdl", "Pkg"]
git-tree-sha1 = "c3d1a616362645754b18e12dbba96ec311b0867f"
uuid = "a6bfbf70-4841-5cb9-aa18-3a8ad3c413ee"
version = "2018.6.22+0"

[[NNlib]]
deps = ["Libdl", "LinearAlgebra", "NNPACK_jll", "Pkg", "Requires", "Statistics"]
git-tree-sha1 = "04bae5dcd446ee45d6a3b9e84adca85ee33fb20c"
uuid = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
version = "0.7.0"

[[NaNMath]]
git-tree-sha1 = "928b8ca9b2791081dc71a51c55347c27c618760f"
uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3"
version = "0.3.3"

[[OpenBLAS_jll]]
deps = ["CompilerSupportLibraries_jll", "Libdl", "Pkg"]
git-tree-sha1 = "1887096f6897306a4662f7c5af936da7d5d1a062"
uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"
version = "0.3.9+4"

[[OpenMPI_jll]]
deps = ["Libdl", "Pkg"]
git-tree-sha1 = "41b983e26a7ab8c9bf05f7d70c274b817d541b46"
uuid = "fe0851c0-eecd-5654-98d4-656369965a5c"
version = "4.0.2+2"

[[OpenSpecFun_jll]]
deps = ["CompilerSupportLibraries_jll", "Libdl", "Pkg"]
git-tree-sha1 = "d51c416559217d974a1113522d5919235ae67a87"
uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e"
version = "0.5.3+3"

[[OrderedCollections]]
git-tree-sha1 = "12ce190210d278e12644bcadf5b21cbdcf225cd3"
uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
version = "1.2.0"

[[PDMats]]
deps = ["Arpack", "LinearAlgebra", "SparseArrays", "SuiteSparse", "Test"]
git-tree-sha1 = "2fc6f50ddd959e462f0a2dbc802ddf2a539c6e35"
uuid = "90014a1f-27ba-587c-ab20-58faa44d9150"
version = "0.9.12"

[[PackageCompiler]]
deps = ["Libdl", "Pkg", "UUIDs"]
git-tree-sha1 = "8989a56e4506b7b6a050f2b912e7856742d6998c"
uuid = "9b87118b-4619-50d2-8e1e-99f35a4d4d9d"
version = "1.1.1"

[[Parsers]]
deps = ["Dates", "Test"]
git-tree-sha1 = "eb3e09940c0d7ae01b01d9291ebad7b081c844d3"
uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
version = "1.0.5"

[[Pkg]]
deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"]
uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"

[[Polynomials]]
deps = ["Intervals", "LinearAlgebra", "RecipesBase"]
git-tree-sha1 = "05f4b8b55595b86b252b28fb1e7c402fb4df98d2"
uuid = "f27b6e38-b328-58d1-80ce-0feddd5e7a45"
version = "1.1.1"

[[Printf]]
deps = ["Unicode"]
uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"

[[QuadGK]]
deps = ["DataStructures", "LinearAlgebra"]
git-tree-sha1 = "dc84e810393cfc6294248c9032a9cdacc14a3db4"
uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc"
version = "2.3.1"

[[Quadmath]]
deps = ["Printf", "Random", "Requires"]
git-tree-sha1 = "cd993c45147a8432bf24358f14bf2cfd4aeb14df"
uuid = "be4d8f0f-7fa4-5f49-b795-2f01399ab2dd"
version = "0.5.4"

[[REPL]]
deps = ["InteractiveUtils", "Markdown", "Sockets"]
uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"

[[Random]]
deps = ["Serialization"]
uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"

[[RecipesBase]]
git-tree-sha1 = "54f8ceb165a0f6d083f0d12cb4996f5367c6edbc"
uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01"
version = "1.0.1"

[[Reexport]]
deps = ["Pkg"]
git-tree-sha1 = "7b1d07f411bc8ddb7977ec7f377b97b158514fe0"
uuid = "189a3867-3050-52da-a836-e630ba90ab69"
version = "0.2.0"

[[Requires]]
deps = ["UUIDs"]
git-tree-sha1 = "d37400976e98018ee840e0ca4f9d20baa231dc6b"
uuid = "ae029012-a4dd-5104-9daa-d747884805df"
version = "1.0.1"

[[Rmath]]
deps = ["Random", "Rmath_jll"]
git-tree-sha1 = "86c5647b565873641538d8f812c04e4c9dbeb370"
uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa"
version = "0.6.1"

[[Rmath_jll]]
deps = ["Libdl", "Pkg"]
git-tree-sha1 = "d76185aa1f421306dec73c057aa384bad74188f0"
uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f"
version = "0.2.2+1"

[[RootSolvers]]
deps = ["DocStringExtensions", "ForwardDiff", "Test"]
git-tree-sha1 = "0e5b394adc5c6fb39b3964bce2a259a44cc312d3"
uuid = "7181ea78-2dcb-4de3-ab41-2b8ab5a31e74"
version = "0.2.0"

[[SHA]]
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"

[[Serialization]]
uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"

[[Sockets]]
uuid = "6462fe0b-24de-5631-8697-dd941f90decc"

[[SortingAlgorithms]]
deps = ["DataStructures", "Random", "Test"]
git-tree-sha1 = "03f5898c9959f8115e30bc7226ada7d0df554ddd"
uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c"
version = "0.3.1"

[[SparseArrays]]
deps = ["LinearAlgebra", "Random"]
uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"

[[SpecialFunctions]]
deps = ["OpenSpecFun_jll"]
git-tree-sha1 = "d8d8b8a9f4119829410ecd706da4cc8594a1e020"
uuid = "276daf66-3868-5448-9aa4-cd146d93841b"
version = "0.10.3"

[[StaticArrays]]
deps = ["LinearAlgebra", "Random", "Statistics"]
git-tree-sha1 = "5c06c0aeb81bef54aed4b3f446847905eb6cbda0"
uuid = "90137ffa-7385-5640-81b9-e52037218182"
version = "0.12.3"

[[StaticNumbers]]
deps = ["Requires"]
git-tree-sha1 = "a0df7d5ade3fd0f0e6c93ad63facc05b12c40e6a"
uuid = "c5e4b96a-f99f-5557-8ed2-dc63ef9b5131"
version = "0.3.3"

[[Statistics]]
deps = ["LinearAlgebra", "SparseArrays"]
uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"

[[StatsBase]]
deps = ["DataAPI", "DataStructures", "LinearAlgebra", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics"]
git-tree-sha1 = "a6102b1f364befdb05746f386b67c6b7e3262c45"
uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
version = "0.33.0"

[[StatsFuns]]
deps = ["Rmath", "SpecialFunctions"]
git-tree-sha1 = "04a5a8e6ab87966b43f247920eab053fd5fdc925"
uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c"
version = "0.9.5"

[[SuiteSparse]]
deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"]
uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9"

[[Test]]
deps = ["Distributed", "InteractiveUtils", "Logging", "Random"]
uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

[[TextWrap]]
git-tree-sha1 = "9250ef9b01b66667380cf3275b3f7488d0e25faf"
uuid = "b718987f-49a8-5099-9789-dcd902bef87d"
version = "1.0.1"

[[TimeZones]]
deps = ["Dates", "EzXML", "Mocking", "Printf", "RecipesBase", "Serialization", "Unicode"]
git-tree-sha1 = "db7bc2051d4c2e5f336409224df81485c00de6cb"
uuid = "f269a46b-ccf7-5d73-abea-4c690281aa53"
version = "1.2.0"

[[TimerOutputs]]
deps = ["Printf"]
git-tree-sha1 = "f458ca23ff80e46a630922c555d838303e4b9603"
uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f"
version = "0.5.6"

[[TranscodingStreams]]
deps = ["Random", "Test"]
git-tree-sha1 = "7c53c35547de1c5b9d46a4797cf6d8253807108c"
uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"
version = "0.9.5"

[[URIParser]]
deps = ["Unicode"]
git-tree-sha1 = "53a9f49546b8d2dd2e688d216421d050c9a31d0d"
uuid = "30578b45-9adc-5946-b283-645ec420af67"
version = "0.4.1"

[[UUIDs]]
deps = ["Random", "SHA"]
uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"

[[Unicode]]
uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"

[[VersionParsing]]
git-tree-sha1 = "80229be1f670524750d905f8fc8148e5a8c4537f"
uuid = "81def892-9a0e-5fdd-b105-ffc91e053289"
version = "1.2.0"

[[WriteVTK]]
deps = ["Base64", "CodecZlib", "LightXML", "Random", "TranscodingStreams"]
git-tree-sha1 = "46c5cc2f4e094eb57685f5fb0629b7e14b6ad18b"
uuid = "64499a7a-5c06-52f2-abe2-ccb03c286192"
version = "1.6.0"

[[XML2_jll]]
deps = ["Libdl", "Libiconv_jll", "Pkg", "Zlib_jll"]
git-tree-sha1 = "6b2ffe6728bdba991da4fc1aa5980a53db46a23f"
uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a"
version = "2.9.9+5"

[[Zlib_jll]]
deps = ["Libdl", "Pkg"]
git-tree-sha1 = "4838361eb3ba796a57457bdc183e1a128639d0ec"
uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
version = "1.2.11+12"

out_10192369

+ hostname
hpc-23-28.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ julia --color=no --project test/runtests_gpu.jl
┌ Warning: Performing scalar operations on GPU arrays: This is very slow, consider disallowing these operations with `allowscalar(false)`
└ @ GPUArrays /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu/packages/GPUArrays/X4SqE/src/host/indexing.jl:43
Test Summary:        | Pass  Total
MPIStateArray basics |   15     15
Test Summary:              | Pass  Total
MPIStateArray broadcasting |    9      9
Test Summary:            | Pass  Total
MPIStateArray reductions |   14     14
Test Summary:            | Test Summary:            | Pass  Test Summary:            | Pass  Pass  Total
TotalTotal

MPIStateArray reductions |   14  MPIStateArray reductions   14 | MPIStateArray reductions
  14   |    14  14     14

Test Summary:           | Pass  Total
MPIStateArray varsindex |    9      9
┌ Warning: Performing scalar operations on GPU arrays: This is very slow, consider disallowing these operations with `allowscalar(false)`
└ @ GPUArrays /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu/packages/GPUArrays/X4SqE/src/host/indexing.jl:43
Test Summary:                | Pass  Total
MPIStateArray Reshape basics |    9      9
Test Summary:       | Pass  Total
Convergence/limited |  165    165
Test Summary:                          | Pass  Total
Explicit methods composition of solve! |    7      7
Test Summary:    | Pass  Total
GenericCallbacks |   48     48
┌ Info: Running MPI test...
│   file = "/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Arrays/basics.jl"
└   ntasks = 1
┌ Info: Running MPI test...
│   file = "/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Arrays/broadcasting.jl"
└   ntasks = 1
┌ Info: Running MPI test...
│   file = "/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Arrays/reductions.jl"
└   ntasks = 1
┌ Info: Running MPI test...
│   file = "/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Arrays/reductions.jl"
└   ntasks = 3
┌ Info: Running MPI test...
│   file = "/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Arrays/varsindex.jl"
└   ntasks = 1
┌ Info: Running MPI test...
│   file = "/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Arrays/reshape.jl"
└   ntasks = 1
┌ Info: Running MPI test...
│   file = "/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/ODESolvers/ode_tests_basic.jl"
└   ntasks = 1
┌ Info: Running MPI test...
│   file = "/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/ODESolvers/callbacks.jl"
└   ntasks = 1
Starting tests for Arrays
 45.187354 seconds (54.04 k allocations: 2.718 MiB, 0.04% gc time)
 74.409091 seconds (299 allocations: 16.672 KiB, 0.02% gc time)
 86.303472 seconds (315 allocations: 17.641 KiB, 0.02% gc time)
261.437835 seconds (596 allocations: 34.922 KiB, 0.03% gc time)
 41.928383 seconds (242 allocations: 12.953 KiB, 0.02% gc time)
 66.048776 seconds (291 allocations: 16.188 KiB, 0.02% gc time)
Test Summary:             | Pass  Total
MPIStateArrays reductions |    6      6
Completed tests for Arrays, 580 seconds elapsed
Starting tests for Numerics/ODESolvers
427.330561 seconds (862 allocations: 51.562 KiB, 0.02% gc time)
 36.426814 seconds (242 allocations: 12.953 KiB, 0.02% gc time)
Test Summary: | Pass  Total
ODE Solvers   |    2      2
Completed tests for Numerics/ODESolvers, 464 seconds elapsed

out_10192370

+ hostname
hpc-23-28.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192370_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192370_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/advection_diffusion/pseudo1D_advection_diffusion.jl --integration-testing
┌ Info: time step
└   dt = 0.00390625
[ Info: (CUDA.CuArray, Float64, 2, EveryDirection, true)
┌ Info: Starting
└ norm(Q₀) = 3.8737679305268846e-01
┌ Info: Finished
│ norm(Q)                 = 2.1231902141153636e-01
│ norm(Q) / norm(Q₀)      = 5.4809432371612954e-01
│ norm(Q) - norm(Q₀)      = -1.7505777164115210e-01
│ norm(Q - Qe)            = 1.2357162985295388e-02
└ norm(Q - Qe) / norm(Qe) = 5.9111207392709218e-02
[ Info: 
┌ Info: time step
└   dt = 0.00390625
[ Info: (CUDA.CuArray, Float64, 2, EveryDirection, false)
┌ Info: Starting
└ norm(Q₀) = 3.8737679305268846e-01
┌ Info: Finished
│ norm(Q)                 = 2.1231902141153636e-01
│ norm(Q) / norm(Q₀)      = 5.4809432371612954e-01
│ norm(Q) - norm(Q₀)      = -1.7505777164115210e-01
│ norm(Q - Qe)            = 1.2357162985295388e-02
└ norm(Q - Qe) / norm(Qe) = 5.9111207392709218e-02
[ Info: 
┌ Info: time step
└   dt = 0.00390625
[ Info: (CUDA.CuArray, Float64, 2, HorizontalDirection, true)
┌ Info: Starting
└ norm(Q₀) = 3.3630637465395119e-01
┌ Info: Finished
│ norm(Q)                 = 1.7559633237315536e-01
│ norm(Q) / norm(Q₀)      = 5.2213203675915609e-01
│ norm(Q) - norm(Q₀)      = -1.6071004228079583e-01
│ norm(Q - Qe)            = 4.6783743619257107e-02
└ norm(Q - Qe) / norm(Qe) = 2.1319129181721072e-01
[ Info: 
┌ Info: time step
└   dt = 0.00390625
[ Info: (CUDA.CuArray, Float64, 2, HorizontalDirection, false)
┌ Info: Starting
└ norm(Q₀) = 3.3630637465395119e-01
┌ Info: Finished
│ norm(Q)                 = 1.7559633237315536e-01
│ norm(Q) / norm(Q₀)      = 5.2213203675915609e-01
│ norm(Q) - norm(Q₀)      = -1.6071004228079583e-01
│ norm(Q - Qe)            = 4.6783743619257107e-02
└ norm(Q - Qe) / norm(Qe) = 2.1319129181721072e-01
[ Info: 
┌ Info: time step
└   dt = 0.00390625
[ Info: (CUDA.CuArray, Float64, 2, VerticalDirection, true)
┌ Info: Starting
└ norm(Q₀) = 3.3630637465395119e-01
┌ Info: Finished
│ norm(Q)                 = 1.7559633237315533e-01
│ norm(Q) / norm(Q₀)      = 5.2213203675915598e-01
│ norm(Q) - norm(Q₀)      = -1.6071004228079586e-01
│ norm(Q - Qe)            = 4.6783743619257127e-02
└ norm(Q - Qe) / norm(Qe) = 2.1319129181721080e-01
[ Info: 
┌ Info: time step
└   dt = 0.00390625
[ Info: (CUDA.CuArray, Float64, 2, VerticalDirection, false)
┌ Info: Starting
└ norm(Q₀) = 3.3630637465395119e-01
┌ Info: Finished
│ norm(Q)                 = 1.7559633237315533e-01
│ norm(Q) / norm(Q₀)      = 5.2213203675915598e-01
│ norm(Q) - norm(Q₀)      = -1.6071004228079586e-01
│ norm(Q - Qe)            = 4.6783743619257127e-02
└ norm(Q - Qe) / norm(Qe) = 2.1319129181721080e-01
[ Info: 
┌ Info: time step
└   dt = 0.00390625
[ Info: (CUDA.CuArray, Float64, 3, EveryDirection, true)
┌ Info: Starting
└ norm(Q₀) = 5.4103588436435490e-01
┌ Info: Finished
│ norm(Q)                 = 3.0178604704383455e-01
│ norm(Q) / norm(Q₀)      = 5.5779303326320573e-01
│ norm(Q) - norm(Q₀)      = -2.3924983732052035e-01
│ norm(Q - Qe)            = 9.6252415559793195e-03
└ norm(Q - Qe) / norm(Qe) = 3.1725960166418626e-02
[ Info: 
┌ Info: time step
└   dt = 0.00390625
[ Info: (CUDA.CuArray, Float64, 3, EveryDirection, false)
┌ Info: Starting
└ norm(Q₀) = 5.4103588436435490e-01
┌ Info: Finished
│ norm(Q)                 = 3.0178604704383455e-01
│ norm(Q) / norm(Q₀)      = 5.5779303326320573e-01
│ norm(Q) - norm(Q₀)      = -2.3924983732052035e-01
│ norm(Q - Qe)            = 9.6252415559793195e-03
└ norm(Q - Qe) / norm(Qe) = 3.1725960166418626e-02
[ Info: 
┌ Info: time step
└   dt = 0.00390625
[ Info: (CUDA.CuArray, Float64, 3, HorizontalDirection, true)
┌ Info: Starting
└ norm(Q₀) = 5.4783351448370776e-01
┌ Info: Finished
│ norm(Q)                 = 3.0026443962997829e-01
│ norm(Q) / norm(Q₀)      = 5.4809432371612954e-01
│ norm(Q) - norm(Q₀)      = -2.4756907485372948e-01
│ norm(Q - Qe)            = 1.7475667486259411e-02
└ norm(Q - Qe) / norm(Qe) = 5.9111207392708767e-02
[ Info: 
┌ Info: time step
└   dt = 0.00390625
[ Info: (CUDA.CuArray, Float64, 3, HorizontalDirection, false)
┌ Info: Starting
└ norm(Q₀) = 5.4783351448370776e-01
┌ Info: Finished
│ norm(Q)                 = 3.0026443962997829e-01
│ norm(Q) / norm(Q₀)      = 5.4809432371612954e-01
│ norm(Q) - norm(Q₀)      = -2.4756907485372948e-01
│ norm(Q - Qe)            = 1.7475667486259411e-02
└ norm(Q - Qe) / norm(Qe) = 5.9111207392708767e-02
[ Info: 
┌ Info: time step
└   dt = 0.00390625
[ Info: (CUDA.CuArray, Float64, 3, VerticalDirection, true)
┌ Info: Starting
└ norm(Q₀) = 4.7560903614814520e-01
┌ Info: Finished
│ norm(Q)                 = 2.4833071474509014e-01
│ norm(Q) / norm(Q₀)      = 5.2213203675915609e-01
│ norm(Q) - norm(Q₀)      = -2.2727832140305507e-01
│ norm(Q - Qe)            = 6.6162204724939166e-02
└ norm(Q - Qe) / norm(Qe) = 2.1319129181721072e-01
[ Info: 
┌ Info: time step
└   dt = 0.00390625
[ Info: (CUDA.CuArray, Float64, 3, VerticalDirection, false)
┌ Info: Starting
└ norm(Q₀) = 4.7560903614814520e-01
┌ Info: Finished
│ norm(Q)                 = 2.4833071474509014e-01
│ norm(Q) / norm(Q₀)      = 5.2213203675915609e-01
│ norm(Q) - norm(Q₀)      = -2.2727832140305507e-01
│ norm(Q - Qe)            = 6.6162204724939166e-02
└ norm(Q - Qe) / norm(Qe) = 2.1319129181721072e-01
[ Info: 
┌ Info: time step
└   dt = 0.00390625f0
[ Info: (CUDA.CuArray, Float32, 2, EveryDirection, true)
┌ Info: Starting
└ norm(Q₀) = 3.8737666606903076e-01
┌ Info: Finished
│ norm(Q)                 = 2.1231889724731445e-01
│ norm(Q) / norm(Q₀)      = 5.4809415340423584e-01
│ norm(Q) - norm(Q₀)      = -1.7505776882171631e-01
│ norm(Q - Qe)            = 1.2357021681964397e-02
└ norm(Q - Qe) / norm(Qe) = 5.9110544621944427e-02
[ Info: 
┌ Info: time step
└   dt = 0.00390625f0
[ Info: (CUDA.CuArray, Float32, 2, EveryDirection, false)
┌ Info: Starting
└ norm(Q₀) = 3.8737666606903076e-01
┌ Info: Finished
│ norm(Q)                 = 2.1231889724731445e-01
│ norm(Q) / norm(Q₀)      = 5.4809415340423584e-01
│ norm(Q) - norm(Q₀)      = -1.7505776882171631e-01
│ norm(Q - Qe)            = 1.2357021681964397e-02
└ norm(Q - Qe) / norm(Qe) = 5.9110544621944427e-02
[ Info: 
┌ Info: time step
└   dt = 0.00390625f0
[ Info: (CUDA.CuArray, Float32, 2, HorizontalDirection, true)
┌ Info: Starting
└ norm(Q₀) = 3.3630600571632385e-01
┌ Info: Finished
│ norm(Q)                 = 1.7559614777565002e-01
│ norm(Q) / norm(Q₀)      = 5.2213203907012939e-01
│ norm(Q) - norm(Q₀)      = -1.6070985794067383e-01
│ norm(Q - Qe)            = 4.6783886849880219e-02
└ norm(Q - Qe) / norm(Qe) = 2.1319194138050079e-01
[ Info: 
┌ Info: time step
└   dt = 0.00390625f0
[ Info: (CUDA.CuArray, Float32, 2, HorizontalDirection, false)
┌ Info: Starting
└ norm(Q₀) = 3.3630600571632385e-01
┌ Info: Finished
│ norm(Q)                 = 1.7559614777565002e-01
│ norm(Q) / norm(Q₀)      = 5.2213203907012939e-01
│ norm(Q) - norm(Q₀)      = -1.6070985794067383e-01
│ norm(Q - Qe)            = 4.6783886849880219e-02
└ norm(Q - Qe) / norm(Qe) = 2.1319194138050079e-01
[ Info: 
┌ Info: time step
└   dt = 0.00390625f0
[ Info: (CUDA.CuArray, Float32, 2, VerticalDirection, true)
┌ Info: Starting
└ norm(Q₀) = 3.3630603551864624e-01
┌ Info: Finished
│ norm(Q)                 = 1.7559617757797241e-01
│ norm(Q) / norm(Q₀)      = 5.2213209867477417e-01
│ norm(Q) - norm(Q₀)      = -1.6070985794067383e-01
│ norm(Q - Qe)            = 4.6783857047557831e-02
└ norm(Q - Qe) / norm(Qe) = 2.1319180727005005e-01
[ Info: 
┌ Info: time step
└   dt = 0.00390625f0
[ Info: (CUDA.CuArray, Float32, 2, VerticalDirection, false)
┌ Info: Starting
└ norm(Q₀) = 3.3630603551864624e-01
┌ Info: Finished
│ norm(Q)                 = 1.7559617757797241e-01
│ norm(Q) / norm(Q₀)      = 5.2213209867477417e-01
│ norm(Q) - norm(Q₀)      = -1.6070985794067383e-01
│ norm(Q - Qe)            = 4.6783857047557831e-02
└ norm(Q - Qe) / norm(Qe) = 2.1319180727005005e-01
[ Info: 
┌ Info: time step
└   dt = 0.00390625f0
[ Info: (CUDA.CuArray, Float32, 3, EveryDirection, true)
┌ Info: Starting
└ norm(Q₀) = 5.4103571176528931e-01
┌ Info: Finished
│ norm(Q)                 = 3.0178582668304443e-01
│ norm(Q) / norm(Q₀)      = 5.5779278278350830e-01
│ norm(Q) - norm(Q₀)      = -2.3924988508224487e-01
│ norm(Q - Qe)            = 9.6252309158444405e-03
└ norm(Q - Qe) / norm(Qe) = 3.1725939363241196e-02
[ Info: 
┌ Info: time step
└   dt = 0.00390625f0
[ Info: (CUDA.CuArray, Float32, 3, EveryDirection, false)
┌ Info: Starting
└ norm(Q₀) = 5.4103571176528931e-01
┌ Info: Finished
│ norm(Q)                 = 3.0178582668304443e-01
│ norm(Q) / norm(Q₀)      = 5.5779278278350830e-01
│ norm(Q) - norm(Q₀)      = -2.3924988508224487e-01
│ norm(Q - Qe)            = 9.6252309158444405e-03
└ norm(Q - Qe) / norm(Qe) = 3.1725939363241196e-02
[ Info: 
┌ Info: time step
└   dt = 0.00390625f0
[ Info: (CUDA.CuArray, Float32, 3, HorizontalDirection, true)
┌ Info: Starting
└ norm(Q₀) = 5.4783332347869873e-01
┌ Info: Finished
│ norm(Q)                 = 3.0026429891586304e-01
│ norm(Q) / norm(Q₀)      = 5.4809427261352539e-01
│ norm(Q) - norm(Q₀)      = -2.4756902456283569e-01
│ norm(Q - Qe)            = 1.7475545406341553e-02
└ norm(Q - Qe) / norm(Qe) = 5.9110812842845917e-02
[ Info: 
┌ Info: time step
└   dt = 0.00390625f0
[ Info: (CUDA.CuArray, Float32, 3, HorizontalDirection, false)
┌ Info: Starting
└ norm(Q₀) = 5.4783332347869873e-01
┌ Info: Finished
│ norm(Q)                 = 3.0026429891586304e-01
│ norm(Q) / norm(Q₀)      = 5.4809427261352539e-01
│ norm(Q) - norm(Q₀)      = -2.4756902456283569e-01
│ norm(Q - Qe)            = 1.7475545406341553e-02
└ norm(Q - Qe) / norm(Qe) = 5.9110812842845917e-02
[ Info: 
┌ Info: time step
└   dt = 0.00390625f0
[ Info: (CUDA.CuArray, Float32, 3, VerticalDirection, true)
┌ Info: Starting
└ norm(Q₀) = 4.7560843825340271e-01
┌ Info: Finished
│ norm(Q)                 = 2.4833044409751892e-01
│ norm(Q) / norm(Q₀)      = 5.2213209867477417e-01
│ norm(Q) - norm(Q₀)      = -2.2727799415588379e-01
│ norm(Q - Qe)            = 6.6162377595901489e-02
└ norm(Q - Qe) / norm(Qe) = 2.1319185197353363e-01
[ Info: 
┌ Info: time step
└   dt = 0.00390625f0
[ Info: (CUDA.CuArray, Float32, 3, VerticalDirection, false)
┌ Info: Starting
└ norm(Q₀) = 4.7560843825340271e-01
┌ Info: Finished
│ norm(Q)                 = 2.4833044409751892e-01
│ norm(Q) / norm(Q₀)      = 5.2213209867477417e-01
│ norm(Q) - norm(Q₀)      = -2.2727799415588379e-01
│ norm(Q - Qe)            = 6.6162377595901489e-02
└ norm(Q - Qe) / norm(Qe) = 2.1319185197353363e-01
[ Info: 
Test Summary:                                                                                                                                               | Test Summary:                                                                                                                                               | Pass  Pass  Test Summary:                                                                                                                                               | Pass  Total
Total
Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/pseudo1D_advection_diffusion.jl | /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/pseudo1D_advection_diffusion.jl |   24    24     24
   24
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/pseudo1D_advection_diffusion.jl |   24     24

out_10192371

+ hostname
hpc-89-38.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192371_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192371_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/advection_diffusion/pseudo1D_advection_diffusion_1dimex.jl
[ Info: (CUDA.CuArray, Float64, 2, SingleColumnLU, 1, true)
┌ Info: Starting
└ norm(Q₀) = 4.2914804456671035e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 2.7809865655333543e-01
│ norm(Q) / norm(Q₀)      = 6.4802498828607724e-01
│ norm(Q) - norm(Q₀)      = -1.5104938801337492e-01
│ norm(Q - Qe)            = 7.2801198255507432e-02
└ norm(Q - Qe) / norm(Qe) = 2.8371215264958716e-01
[ Info: 
[ Info: (CUDA.CuArray, Float64, 2, ManyColumnLU, 1, true)
┌ Info: Starting
└ norm(Q₀) = 4.2914804456671035e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 2.7809865655333543e-01
│ norm(Q) / norm(Q₀)      = 6.4802498828607724e-01
│ norm(Q) - norm(Q₀)      = -1.5104938801337492e-01
│ norm(Q - Qe)            = 7.2801198255507418e-02
└ norm(Q - Qe) / norm(Qe) = 2.8371215264958710e-01
[ Info: 
[ Info: (CUDA.CuArray, Float64, 2, SingleColumnLU, 1, false)
┌ Info: Starting
└ norm(Q₀) = 4.2914804456671035e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 2.7809865655333543e-01
│ norm(Q) / norm(Q₀)      = 6.4802498828607724e-01
│ norm(Q) - norm(Q₀)      = -1.5104938801337492e-01
│ norm(Q - Qe)            = 7.2801198255507432e-02
└ norm(Q - Qe) / norm(Qe) = 2.8371215264958716e-01
[ Info: 
[ Info: (CUDA.CuArray, Float64, 2, ManyColumnLU, 1, false)
┌ Info: Starting
└ norm(Q₀) = 4.2914804456671035e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 2.7809865655333543e-01
│ norm(Q) / norm(Q₀)      = 6.4802498828607724e-01
│ norm(Q) - norm(Q₀)      = -1.5104938801337492e-01
│ norm(Q - Qe)            = 7.2801198255507418e-02
└ norm(Q - Qe) / norm(Qe) = 2.8371215264958710e-01
[ Info: 
[ Info: (CUDA.CuArray, Float64, 3, SingleColumnLU, 1, true)
┌ Info: Starting
└ norm(Q₀) = 5.6541735527969450e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 3.6810030091336332e-01
│ norm(Q) / norm(Q₀)      = 6.5102405767377880e-01
│ norm(Q) - norm(Q₀)      = -1.9731705436633118e-01
│ norm(Q - Qe)            = 1.0462203776357536e-01
└ norm(Q - Qe) / norm(Qe) = 2.8922403248030559e-01
[ Info: 
[ Info: (CUDA.CuArray, Float64, 3, ManyColumnLU, 1, true)
┌ Info: Starting
└ norm(Q₀) = 5.6541735527969450e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 3.6810030091336332e-01
│ norm(Q) / norm(Q₀)      = 6.5102405767377880e-01
│ norm(Q) - norm(Q₀)      = -1.9731705436633118e-01
│ norm(Q - Qe)            = 1.0462203776357536e-01
└ norm(Q - Qe) / norm(Qe) = 2.8922403248030559e-01
[ Info: 
[ Info: (CUDA.CuArray, Float64, 3, SingleColumnLU, 1, false)
┌ Info: Starting
└ norm(Q₀) = 5.6541735527969450e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 3.6810030091336332e-01
│ norm(Q) / norm(Q₀)      = 6.5102405767377880e-01
│ norm(Q) - norm(Q₀)      = -1.9731705436633118e-01
│ norm(Q - Qe)            = 1.0462203776357536e-01
└ norm(Q - Qe) / norm(Qe) = 2.8922403248030559e-01
[ Info: 
[ Info: (CUDA.CuArray, Float64, 3, ManyColumnLU, 1, false)
┌ Info: Starting
└ norm(Q₀) = 5.6541735527969450e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 3.6810030091336332e-01
│ norm(Q) / norm(Q₀)      = 6.5102405767377880e-01
│ norm(Q) - norm(Q₀)      = -1.9731705436633118e-01
│ norm(Q - Qe)            = 1.0462203776357536e-01
└ norm(Q - Qe) / norm(Qe) = 2.8922403248030559e-01
[ Info: 
[ Info: (CUDA.CuArray, Float32, 2, SingleColumnLU, 1, true)
┌ Info: Starting
└ norm(Q₀) = 4.2914789915084839e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 2.7809852361679077e-01
│ norm(Q) / norm(Q₀)      = 6.4802491664886475e-01
│ norm(Q) - norm(Q₀)      = -1.5104937553405762e-01
│ norm(Q - Qe)            = 7.2801202535629272e-02
└ norm(Q - Qe) / norm(Qe) = 2.8371220827102661e-01
[ Info: 
[ Info: (CUDA.CuArray, Float32, 2, ManyColumnLU, 1, true)
┌ Info: Starting
└ norm(Q₀) = 4.2914789915084839e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 2.7809852361679077e-01
│ norm(Q) / norm(Q₀)      = 6.4802491664886475e-01
│ norm(Q) - norm(Q₀)      = -1.5104937553405762e-01
│ norm(Q - Qe)            = 7.2801217436790466e-02
└ norm(Q - Qe) / norm(Qe) = 2.8371226787567139e-01
[ Info: 
[ Info: (CUDA.CuArray, Float32, 2, SingleColumnLU, 1, false)
┌ Info: Starting
└ norm(Q₀) = 4.2914789915084839e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 2.7809852361679077e-01
│ norm(Q) / norm(Q₀)      = 6.4802491664886475e-01
│ norm(Q) - norm(Q₀)      = -1.5104937553405762e-01
│ norm(Q - Qe)            = 7.2801202535629272e-02
└ norm(Q - Qe) / norm(Qe) = 2.8371220827102661e-01
[ Info: 
[ Info: (CUDA.CuArray, Float32, 2, ManyColumnLU, 1, false)
┌ Info: Starting
└ norm(Q₀) = 4.2914789915084839e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 2.7809852361679077e-01
│ norm(Q) / norm(Q₀)      = 6.4802491664886475e-01
│ norm(Q) - norm(Q₀)      = -1.5104937553405762e-01
│ norm(Q - Qe)            = 7.2801217436790466e-02
└ norm(Q - Qe) / norm(Qe) = 2.8371226787567139e-01
[ Info: 
[ Info: (CUDA.CuArray, Float32, 3, SingleColumnLU, 1, true)
┌ Info: Starting
└ norm(Q₀) = 5.6541717052459717e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 3.6810007691383362e-01
│ norm(Q) / norm(Q₀)      = 6.5102386474609375e-01
│ norm(Q) - norm(Q₀)      = -1.9731709361076355e-01
│ norm(Q - Qe)            = 1.0462204366922379e-01
└ norm(Q - Qe) / norm(Qe) = 2.8922411799430847e-01
[ Info: 
[ Info: (CUDA.CuArray, Float32, 3, ManyColumnLU, 1, true)
┌ Info: Starting
└ norm(Q₀) = 5.6541717052459717e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 3.6810007691383362e-01
│ norm(Q) / norm(Q₀)      = 6.5102386474609375e-01
│ norm(Q) - norm(Q₀)      = -1.9731709361076355e-01
│ norm(Q - Qe)            = 1.0462204366922379e-01
└ norm(Q - Qe) / norm(Qe) = 2.8922411799430847e-01
[ Info: 
[ Info: (CUDA.CuArray, Float32, 3, SingleColumnLU, 1, false)
┌ Info: Starting
└ norm(Q₀) = 5.6541717052459717e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 3.6810007691383362e-01
│ norm(Q) / norm(Q₀)      = 6.5102386474609375e-01
│ norm(Q) - norm(Q₀)      = -1.9731709361076355e-01
│ norm(Q - Qe)            = 1.0462204366922379e-01
└ norm(Q - Qe) / norm(Qe) = 2.8922411799430847e-01
[ Info: 
[ Info: (CUDA.CuArray, Float32, 3, ManyColumnLU, 1, false)
┌ Info: Starting
└ norm(Q₀) = 5.6541717052459717e-01
┌ Info: time step
│   dt = 0.00390625
│   numberofsteps = 128
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 3.6810007691383362e-01
│ norm(Q) / norm(Q₀)      = 6.5102386474609375e-01
│ norm(Q) - norm(Q₀)      = -1.9731709361076355e-01
│ norm(Q - Qe)            = 1.0462204366922379e-01
└ norm(Q - Qe) / norm(Qe) = 2.8922411799430847e-01
[ Info: 
Test Summary:                                                                                                                                                      | Pass  Test Summary:                                                                                                                                                      | Test Summary:                                                                                                                                                      | Pass  Pass  Total
Total
Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/pseudo1D_advection_diffusion_1dimex.jl |   16  /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/pseudo1D_advection_diffusion_1dimex.jl |    16
  16     16
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/pseudo1D_advection_diffusion_1dimex.jl |   16     16

out_10192372

+ hostname
hpc-26-15.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192372_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192372_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/advection_diffusion/pseudo1D_advection_diffusion_mrigark_implicit.jl
[ Info: (CUDA.CuArray, Float64, 2, SingleColumnLU, 1, true)
┌ Info: Starting
└ norm(Q₀) = 4.2914804456671035e-01
┌ Info: time step
│   dt = 0.125
│   numberofsteps = 4
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 2.7858038837682986e-01
│ norm(Q) / norm(Q₀)      = 6.4914751891296341e-01
│ norm(Q) - norm(Q₀)      = -1.5056765618988049e-01
│ norm(Q - Qe)            = 7.3188989633307638e-02
└ norm(Q - Qe) / norm(Qe) = 2.8522340698620546e-01
[ Info: 
[ Info: (CUDA.CuArray, Float64, 2, SingleColumnLU, 1, false)
┌ Info: Starting
└ norm(Q₀) = 4.2914804456671035e-01
┌ Info: time step
│   dt = 0.125
│   numberofsteps = 4
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 2.7858038837682986e-01
│ norm(Q) / norm(Q₀)      = 6.4914751891296341e-01
│ norm(Q) - norm(Q₀)      = -1.5056765618988049e-01
│ norm(Q - Qe)            = 7.3188989633307638e-02
└ norm(Q - Qe) / norm(Qe) = 2.8522340698620546e-01
[ Info: 
[ Info: (CUDA.CuArray, Float64, 3, SingleColumnLU, 1, true)
┌ Info: Starting
└ norm(Q₀) = 5.6541735527969450e-01
┌ Info: time step
│   dt = 0.125
│   numberofsteps = 4
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 3.6868537487180691e-01
│ norm(Q) / norm(Q₀)      = 6.5205882244175128e-01
│ norm(Q) - norm(Q₀)      = -1.9673198040788759e-01
│ norm(Q - Qe)            = 1.0501469629884570e-01
└ norm(Q - Qe) / norm(Qe) = 2.9030952352393580e-01
[ Info: 
[ Info: (CUDA.CuArray, Float64, 3, SingleColumnLU, 1, false)
┌ Info: Starting
└ norm(Q₀) = 5.6541735527969450e-01
┌ Info: time step
│   dt = 0.125
│   numberofsteps = 4
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 3.6868537487180691e-01
│ norm(Q) / norm(Q₀)      = 6.5205882244175128e-01
│ norm(Q) - norm(Q₀)      = -1.9673198040788759e-01
│ norm(Q - Qe)            = 1.0501469629884570e-01
└ norm(Q - Qe) / norm(Qe) = 2.9030952352393580e-01
[ Info: 
[ Info: (CUDA.CuArray, Float32, 2, SingleColumnLU, 1, true)
┌ Info: Starting
└ norm(Q₀) = 4.2914789915084839e-01
┌ Info: time step
│   dt = 0.125
│   numberofsteps = 4
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 2.7857178449630737e-01
│ norm(Q) / norm(Q₀)      = 6.4912766218185425e-01
│ norm(Q) - norm(Q₀)      = -1.5057611465454102e-01
│ norm(Q - Qe)            = 7.3184572160243988e-02
└ norm(Q - Qe) / norm(Qe) = 2.8520622849464417e-01
[ Info: 
[ Info: (CUDA.CuArray, Float32, 2, SingleColumnLU, 1, false)
┌ Info: Starting
└ norm(Q₀) = 4.2914789915084839e-01
┌ Info: time step
│   dt = 0.125
│   numberofsteps = 4
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 2.7857178449630737e-01
│ norm(Q) / norm(Q₀)      = 6.4912766218185425e-01
│ norm(Q) - norm(Q₀)      = -1.5057611465454102e-01
│ norm(Q - Qe)            = 7.3184572160243988e-02
└ norm(Q - Qe) / norm(Qe) = 2.8520622849464417e-01
[ Info: 
[ Info: (CUDA.CuArray, Float32, 3, SingleColumnLU, 1, true)
┌ Info: Starting
└ norm(Q₀) = 5.6541717052459717e-01
┌ Info: time step
│   dt = 0.125
│   numberofsteps = 4
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 3.6867204308509827e-01
│ norm(Q) / norm(Q₀)      = 6.5203547477722168e-01
│ norm(Q) - norm(Q₀)      = -1.9674512743949890e-01
│ norm(Q - Qe)            = 1.0501022636890411e-01
└ norm(Q - Qe) / norm(Qe) = 2.9029724001884460e-01
[ Info: 
[ Info: (CUDA.CuArray, Float32, 3, SingleColumnLU, 1, false)
┌ Info: Starting
└ norm(Q₀) = 5.6541717052459717e-01
┌ Info: time step
│   dt = 0.125
│   numberofsteps = 4
│   dt * numberofsteps = 0.5
└   timeend = 0.5
┌ Info: Finished
│ norm(Q)                 = 3.6867204308509827e-01
│ norm(Q) / norm(Q₀)      = 6.5203547477722168e-01
│ norm(Q) - norm(Q₀)      = -1.9674512743949890e-01
│ norm(Q - Qe)            = 1.0501022636890411e-01
└ norm(Q - Qe) / norm(Qe) = 2.9029724001884460e-01
[ Info: 
Test Summary:                                                                                                                                                                | Pass  Test Summary:                                                                                                                                                                | Test Summary:                                                                                                                                                                | Pass  Total
Pass  Total
Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/pseudo1D_advection_diffusion_mrigark_implicit.jl |    8  /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/pseudo1D_advection_diffusion_mrigark_implicit.jl |     8
   8      8
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/pseudo1D_advection_diffusion_mrigark_implicit.jl |    8      8

out_10192373

+ hostname
hpc-26-15.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192373_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192373_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/advection_diffusion/pseudo1D_heat_eqn.jl --integration-testing
[ Info: (CUDA.CuArray, Float64, 2, EveryDirection)
┌ Info: time step
│   dt = 0.00024390243902439024
│   numberofsteps = 41
│   dt * numberofsteps = 0.01
└   timeend = 0.01
┌ Info: Starting
└ norm(Q₀) = 1.0525946028760076e+00
┌ Info: Finished
│ norm(Q)                 = 7.6769756052324190e-01
│ norm(Q) / norm(Q₀)      = 7.2933830215893125e-01
│ norm(Q) - norm(Q₀)      = -2.8489704235276569e-01
│ norm(Q - Qe)            = 5.1574832681275757e-03
└ norm(Q - Qe) / norm(Qe) = 6.7183283629127545e-03
[ Info: 
[ Info: (CUDA.CuArray, Float64, 2, HorizontalDirection)
┌ Info: time step
│   dt = 0.00024390243902439024
│   numberofsteps = 41
│   dt * numberofsteps = 0.01
└   timeend = 0.01
┌ Info: Starting
└ norm(Q₀) = 9.0395387652143566e-01
┌ Info: Finished
│ norm(Q)                 = 5.7963038857095928e-01
│ norm(Q) / norm(Q₀)      = 6.4121677402554333e-01
│ norm(Q) - norm(Q₀)      = -3.2432348795047639e-01
│ norm(Q - Qe)            = 2.0515449798977983e-02
└ norm(Q - Qe) / norm(Qe) = 3.5415953970101541e-02
[ Info: 
[ Info: (CUDA.CuArray, Float64, 2, VerticalDirection)
┌ Info: time step
│   dt = 0.00024390243902439024
│   numberofsteps = 41
│   dt * numberofsteps = 0.01
└   timeend = 0.01
┌ Info: Starting
└ norm(Q₀) = 9.0395387652143566e-01
┌ Info: Finished
│ norm(Q)                 = 5.7963038857095928e-01
│ norm(Q) / norm(Q₀)      = 6.4121677402554333e-01
│ norm(Q) - norm(Q₀)      = -3.2432348795047639e-01
│ norm(Q - Qe)            = 2.0515449798977920e-02
└ norm(Q - Qe) / norm(Qe) = 3.5415953970101423e-02
[ Info: 
[ Info: (CUDA.CuArray, Float64, 3, EveryDirection)
┌ Info: time step
│   dt = 0.00024390243902439024
│   numberofsteps = 41
│   dt * numberofsteps = 0.01
└   timeend = 0.01
┌ Info: Starting
└ norm(Q₀) = 1.1510565538414834e+00
┌ Info: Finished
│ norm(Q)                 = 9.1444897086428989e-01
│ norm(Q) / norm(Q₀)      = 7.9444312949911089e-01
│ norm(Q) - norm(Q₀)      = -2.3660758297719353e-01
│ norm(Q - Qe)            = 1.2605810186712559e-03
└ norm(Q - Qe) / norm(Qe) = 1.3785136194861183e-03
[ Info: 
[ Info: (CUDA.CuArray, Float64, 3, HorizontalDirection)
┌ Info: time step
│   dt = 0.00024390243902439024
│   numberofsteps = 41
│   dt * numberofsteps = 0.01
└   timeend = 0.01
┌ Info: Starting
└ norm(Q₀) = 1.0525946028760078e+00
┌ Info: Finished
│ norm(Q)                 = 7.6769756052324212e-01
│ norm(Q) / norm(Q₀)      = 7.2933830215893136e-01
│ norm(Q) - norm(Q₀)      = -2.8489704235276569e-01
│ norm(Q - Qe)            = 5.1574832681275688e-03
└ norm(Q - Qe) / norm(Qe) = 6.7183283629127424e-03
[ Info: 
[ Info: (CUDA.CuArray, Float64, 3, VerticalDirection)
┌ Info: time step
│   dt = 0.00024390243902439024
│   numberofsteps = 41
│   dt * numberofsteps = 0.01
└   timeend = 0.01
┌ Info: Starting
└ norm(Q₀) = 9.0395387652143599e-01
┌ Info: Finished
│ norm(Q)                 = 5.7963038857095939e-01
│ norm(Q) / norm(Q₀)      = 6.4121677402554322e-01
│ norm(Q) - norm(Q₀)      = -3.2432348795047661e-01
│ norm(Q - Qe)            = 2.0515449798978083e-02
└ norm(Q - Qe) / norm(Qe) = 3.5415953970101693e-02
[ Info: 
[ Info: (CUDA.CuArray, Float32, 2, EveryDirection)
┌ Info: time step
│   dt = 0.00024390243902439024
│   numberofsteps = 41
│   dt * numberofsteps = 0.01
└   timeend = 0.01
┌ Info: Starting
└ norm(Q₀) = 1.0525944232940674e+00
┌ Info: Finished
│ norm(Q)                 = 7.6769745349884033e-01
│ norm(Q) / norm(Q₀)      = 7.2933834791183472e-01
│ norm(Q) - norm(Q₀)      = -2.8489696979522705e-01
│ norm(Q - Qe)            = 5.1571349613368511e-03
└ norm(Q - Qe) / norm(Qe) = 6.7178755998611450e-03
[ Info: 
[ Info: (CUDA.CuArray, Float32, 2, HorizontalDirection)
┌ Info: time step
│   dt = 0.00024390243902439024
│   numberofsteps = 41
│   dt * numberofsteps = 0.01
└   timeend = 0.01
┌ Info: Starting
└ norm(Q₀) = 9.0395396947860718e-01
┌ Info: Finished
│ norm(Q)                 = 5.7963043451309204e-01
│ norm(Q) / norm(Q₀)      = 6.4121675491333008e-01
│ norm(Q) - norm(Q₀)      = -3.2432353496551514e-01
│ norm(Q - Qe)            = 2.0514594390988350e-02
└ norm(Q - Qe) / norm(Qe) = 3.5414483398199081e-02
[ Info: 
[ Info: (CUDA.CuArray, Float32, 2, VerticalDirection)
┌ Info: time step
│   dt = 0.00024390243902439024
│   numberofsteps = 41
│   dt * numberofsteps = 0.01
└   timeend = 0.01
┌ Info: Starting
└ norm(Q₀) = 9.0395402908325195e-01
┌ Info: Finished
│ norm(Q)                 = 5.7963049411773682e-01
│ norm(Q) / norm(Q₀)      = 6.4121675491333008e-01
│ norm(Q) - norm(Q₀)      = -3.2432353496551514e-01
│ norm(Q - Qe)            = 2.0514670759439468e-02
└ norm(Q - Qe) / norm(Qe) = 3.5414610058069229e-02
[ Info: 
[ Info: (CUDA.CuArray, Float32, 3, EveryDirection)
┌ Info: time step
│   dt = 0.00024390243902439024
│   numberofsteps = 41
│   dt * numberofsteps = 0.01
└   timeend = 0.01
┌ Info: Starting
└ norm(Q₀) = 1.1510562896728516e+00
┌ Info: Finished
│ norm(Q)                 = 9.1444879770278931e-01
│ norm(Q) / norm(Q₀)      = 7.9444319009780884e-01
│ norm(Q) - norm(Q₀)      = -2.3660749197006226e-01
│ norm(Q - Qe)            = 1.2602002825587988e-03
└ norm(Q - Qe) / norm(Qe) = 1.3780975714325905e-03
[ Info: 
[ Info: (CUDA.CuArray, Float32, 3, HorizontalDirection)
┌ Info: time step
│   dt = 0.00024390243902439024
│   numberofsteps = 41
│   dt * numberofsteps = 0.01
└   timeend = 0.01
┌ Info: Starting
└ norm(Q₀) = 1.0525944232940674e+00
┌ Info: Finished
│ norm(Q)                 = 7.6769733428955078e-01
│ norm(Q) / norm(Q₀)      = 7.2933822870254517e-01
│ norm(Q) - norm(Q₀)      = -2.8489708900451660e-01
│ norm(Q - Qe)            = 5.1570441573858261e-03
└ norm(Q - Qe) / norm(Qe) = 6.7177577875554562e-03
[ Info: 
[ Info: (CUDA.CuArray, Float32, 3, VerticalDirection)
┌ Info: time step
│   dt = 0.00024390243902439024
│   numberofsteps = 41
│   dt * numberofsteps = 0.01
└   timeend = 0.01
┌ Info: Starting
└ norm(Q₀) = 9.0395396947860718e-01
┌ Info: Finished
│ norm(Q)                 = 5.7963043451309204e-01
│ norm(Q) / norm(Q₀)      = 6.4121675491333008e-01
│ norm(Q) - norm(Q₀)      = -3.2432353496551514e-01
│ norm(Q - Qe)            = 2.0514652132987976e-02
└ norm(Q - Qe) / norm(Qe) = 3.5414583981037140e-02
[ Info: 
Test Summary:                                                                                                                                    | Pass  Test Summary:                                                                                                                                    | Test Summary:                                                                                                                                    | Pass  Pass  Total
TotalTotal

/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/pseudo1D_heat_eqn.jl |   12     12
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/pseudo1D_heat_eqn.jl/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/pseudo1D_heat_eqn.jl |  |   12    12     12
   12

out_10192374

+ hostname
hpc-26-17.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192374_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192374_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/advection_diffusion/periodic_3D_hyperdiffusion.jl --integration-testing
[ Info: (CUDA.CuArray, Float64, 2, EveryDirection)
┌ Info: time step
└   dt = 0.021649052503199043
┌ Info: Starting
└ norm(Q₀) = 4.4428829381583697e+00
┌ Info: Finished
│ norm(Q)                 = 4.2303924119636953e+00
│ norm(Q) / norm(Q₀)      = 9.5217282805953152e-01
│ norm(Q) - norm(Q₀)      = -2.1249052619467435e-01
│ norm(Q - Qe)            = 7.6772960298540248e-03
└ norm(Q - Qe) / norm(Qe) = 1.8147794217037085e-03
[ Info: 
[ Info: (CUDA.CuArray, Float64, 2, HorizontalDirection)
┌ Info: time step
└   dt = 0.021649052503199043
┌ Info: Starting
└ norm(Q₀) = 4.4428829381583697e+00
┌ Info: Finished
│ norm(Q)                 = 4.4348927223358281e+00
│ norm(Q) / norm(Q₀)      = 9.9820156958133732e-01
│ norm(Q) - norm(Q₀)      = -7.9902158225415576e-03
│ norm(Q - Qe)            = 8.4650675812551562e-04
└ norm(Q - Qe) / norm(Qe) = 1.9087422609558656e-04
[ Info: 
[ Info: (CUDA.CuArray, Float64, 2, VerticalDirection)
┌ Info: time step
└   dt = 0.021649052503199043
┌ Info: Starting
└ norm(Q₀) = 4.4428829381583697e+00
┌ Info: Finished
│ norm(Q)                 = 4.3444227732067962e+00
│ norm(Q) / norm(Q₀)      = 9.7783867675064462e-01
│ norm(Q) - norm(Q₀)      = -9.8460164951573503e-02
│ norm(Q - Qe)            = 6.1690465335714094e-03
└ norm(Q - Qe) / norm(Qe) = 1.4199772029164022e-03
[ Info: 
[ Info: (CUDA.CuArray, Float64, 3, EveryDirection)
┌ Info: time step
└   dt = 0.021649052503198762
┌ Info: Starting
└ norm(Q₀) = 1.1136655993663426e+01
┌ Info: Finished
│ norm(Q)                 = 6.0910843493796447e+00
│ norm(Q) / norm(Q₀)      = 5.4694015446336597e-01
│ norm(Q) - norm(Q₀)      = -5.0455716442837817e+00
│ norm(Q - Qe)            = 1.7363355506159683e-01
└ norm(Q - Qe) / norm(Qe) = 2.8625704237051613e-02
[ Info: 
[ Info: (CUDA.CuArray, Float64, 3, HorizontalDirection)
┌ Info: time step
└   dt = 0.021649052503198762
┌ Info: Starting
└ norm(Q₀) = 1.1136655993663426e+01
┌ Info: Finished
│ norm(Q)                 = 1.0604021232612636e+01
│ norm(Q) / norm(Q₀)      = 9.5217282805953141e-01
│ norm(Q) - norm(Q₀)      = -5.3263476105079022e-01
│ norm(Q - Qe)            = 1.9244127301144030e-02
└ norm(Q - Qe) / norm(Qe) = 1.8147794217036710e-03
[ Info: 
[ Info: (CUDA.CuArray, Float64, 3, VerticalDirection)
┌ Info: time step
└   dt = 0.021649052503198762
┌ Info: Starting
└ norm(Q₀) = 1.1136655993663426e+01
┌ Info: Finished
│ norm(Q)                 = 9.4792047633662015e+00
│ norm(Q) / norm(Q₀)      = 8.5117155174405246e-01
│ norm(Q) - norm(Q₀)      = -1.6574512302972249e+00
│ norm(Q - Qe)            = 1.4412891107361794e-01
└ norm(Q - Qe) / norm(Qe) = 1.5217804720985009e-02
[ Info: 

out_10192375

+ hostname
hpc-23-28.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192375_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192375_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project tutorials/Atmos/dry_rayleigh_benard.jl --fix-rng-seed
┌ Info: Model composition
│     param_set = EarthParameterSet()
│     orientation = FlatOrientation()
│     ref_state = HydrostaticState{ClimateMachine.TemperatureProfiles.DecayingTemperatureProfile{Float64},Float64}(ClimateMachine.TemperatureProfiles.DecayingTemperatureProfile{Float64}(290.0, 220.0, 8484.271021693852), 0.0)
│     turbulence = Vreman{Float64}(0.23)
│     hyperdiffusion = NoHyperDiffusion()
│     moisture = EquilMoist{Float64}(3, 0.1)
│     precipitation = NoPrecipitation()
│     radiation = NoRadiation()
│     source = (Gravity(),)
│     tracers = NTracers{1,Float64}([1.0])
│     boundarycondition = (AtmosBC{Impenetrable{NoSlip},PrescribedTemperature{var"#3#5"{Float64}},Impermeable,ImpermeableTracer}(Impenetrable{NoSlip}(NoSlip()), PrescribedTemperature{var"#3#5"{Float64}}(var"#3#5"{Float64}(299.0)), Impermeable(), ImpermeableTracer()), AtmosBC{Impenetrable{NoSlip},PrescribedTemperature{var"#4#6"{Float64}},Impermeable,ImpermeableTracer}(Impenetrable{NoSlip}(NoSlip()), PrescribedTemperature{var"#4#6"{Float64}}(var"#4#6"{Float64}(294.11701461178683)), Impermeable(), ImpermeableTracer()))
│     init_state_conservative = init_problem!
└     data_config = DryRayleighBenardConvectionDataConfig{Float64}(0.0, 0.0, 0.0, 250.0, 250.0, 500.0, 299.0, 0.009765970776426321, 294.11701461178683)
┌ Info: Establishing Atmos LES configuration for DryRayleighBenardConvection
│     precision        = Float64
│     polynomial order = 4
│     domain           = 250.00 m x250.00 m x500.00 m
│     resolution       = 10x10x10
│     MPI ranks        = 1
│     min(Δ_horz)      = 7.19 m
└     min(Δ_vert)      = 7.19 m
[ Info: Initializing DryRayleighBenardConvection
┌ Info: Starting DryRayleighBenardConvection
│     dt              = 4.13907e-01
│     timeend         =  1000.00
│     number of steps = 2416
└     norm(Q)         = 1.1616473315696873e+08
┌ Info: Update
│     simtime =     0.41 /  1000.00
│     runtime = 00:01:21
└     norm(Q) = 1.1618204938703574e+08
┌ Info: Update
│     simtime =   133.28 /  1000.00
│     runtime = 00:02:21
└     norm(Q) = 1.1639722830369835e+08
┌ Info: Update
│     simtime =   266.97 /  1000.00
│     runtime = 00:03:21
└     norm(Q) = 1.1657358986265256e+08
┌ Info: Update
│     simtime =   401.08 /  1000.00
│     runtime = 00:04:21
└     norm(Q) = 1.1676938733554134e+08
┌ Info: Update
│     simtime =   536.84 /  1000.00
│     runtime = 00:05:22
└     norm(Q) = 1.1696506451702549e+08
┌ Info: Update
│     simtime =   671.77 /  1000.00
│     runtime = 00:06:22
└     norm(Q) = 1.1716467569124599e+08
┌ Info: Update
│     simtime =   807.12 /  1000.00
│     runtime = 00:07:22
└     norm(Q) = 1.1737127061995643e+08
┌ Info: Update
│     simtime =   942.88 /  1000.00
│     runtime = 00:08:22
└     norm(Q) = 1.1758288949196039e+08
┌ Info: Finished
│     norm(Q)            = 1.1768770543308139e+08
│     norm(Q) / norm(Q₀) = 1.0131104530155011e+00
└     norm(Q) - norm(Q₀) = 1.5229722761126608e+06
┌ Info: Euclidean distance
│     norm(Q - Qe)            = 3.9467958084371546e+06
└     norm(Q - Qe) / norm(Qe) = 3.3970087761522279e-02
Test Summary:         | Pass  Total
DryRayleighBenardTest |    1      1

out_10192376

+ hostname
hpc-26-17.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192376_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192376_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project tutorials/Atmos/risingbubble.jl
┌ Info: Model composition
│     param_set = EarthParameterSet()
│     orientation = FlatOrientation()
│     ref_state = HydrostaticState{DryAdiabaticProfile{Float64},Float64}(DryAdiabaticProfile{Float64}(300.0, 0.0), 0.0)
│     turbulence = SmagorinskyLilly{Float64}(0.21)
│     hyperdiffusion = StandardHyperDiffusion{Int64}(60)
│     moisture = DryModel()
│     precipitation = NoPrecipitation()
│     radiation = NoRadiation()
│     source = (Gravity(),)
│     tracers = NTracers{4,Float64}([1.0, 2.0, 3.0, 4.0])
│     boundarycondition = AtmosBC{Impenetrable{FreeSlip},Insulating,Impermeable,ImpermeableTracer}(Impenetrable{FreeSlip}(FreeSlip()), Insulating(), Impermeable(), ImpermeableTracer())
│     init_state_conservative = init_risingbubble!
└     data_config = nothing
┌ Info: Establishing Atmos LES configuration for DryRisingBubble
│     precision        = Float64
│     polynomial order = 4
│     domain           = 10000.00 m x500.00 m x10000.00 m
│     resolution       = 125x125x125
│     MPI ranks        = 3
│     min(Δ_horz)      = 86.34 m
└     min(Δ_vert)      = 86.34 m
[ Info: Initializing DryRisingBubble
┌ Info: Starting DryRisingBubble
│     dt              = 4.22654e-01
│     timeend         =  1000.00
│     number of steps = 2366
└     norm(Q)         = 5.3834517091450853e+09
┌ Info: Update
│     simtime =     0.42 /  1000.00
│     runtime = 00:01:56
└     norm(Q) = 5.3834512577224026e+09
┌ Info: Update
│     simtime =   222.74 /  1000.00
│     runtime = 00:02:56
└     norm(Q) = 5.3833833034340601e+09
┌ Info: Update
│     simtime =   446.75 /  1000.00
│     runtime = 00:03:56
└     norm(Q) = 5.3835557077348614e+09
┌ Info: Update
│     simtime =   671.17 /  1000.00
│     runtime = 00:04:56
└     norm(Q) = 5.3834267706626015e+09
┌ Info: Update
│     simtime =   896.45 /  1000.00
│     runtime = 00:05:56
└     norm(Q) = 5.3832415320854464e+09
┌ Info: Finished
│     norm(Q)            = 5.3832161256496944e+09
│     norm(Q) / norm(Q₀) = 9.9995623932225663e-01
└     norm(Q) - norm(Q₀) = -2.3558349539089203e+05
┌ Info: Euclidean distance
│     norm(Q - Qe)            = 4.6065643107759334e+07
└     norm(Q - Qe) / norm(Qe) = 8.5568972467061945e-03

out_10192377

+ hostname
hpc-26-17.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192377_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192377_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project experiments/AtmosGCM/heldsuarez.jl --diagnostics=default --fix-rng-seed
┌ Info: Model composition
│     param_set = EarthParameterSet()
│     orientation = SphericalOrientation()
│     ref_state = HydrostaticState{DecayingTemperatureProfile{Float32},Float32}(DecayingTemperatureProfile{Float32}(290.0f0, 220.0f0, 8484.2705f0), 0.0f0)
│     turbulence = SmagorinskyLilly{Float32}(0.21f0)
│     hyperdiffusion = StandardHyperDiffusion{Float32}(14400.0f0)
│     moisture = DryModel()
│     precipitation = NoPrecipitation()
│     radiation = NoRadiation()
│     source = (Gravity(), Coriolis(), held_suarez_forcing!, RayleighSponge{Float32}(30000.0f0, 12000.0f0, 0.0011111111f0, Float32[0.0, 0.0, 0.0], 2.0f0))
│     tracers = NoTracers()
│     boundarycondition = AtmosBC{Impenetrable{FreeSlip},Insulating,Impermeable,ImpermeableTracer}(Impenetrable{FreeSlip}(FreeSlip()), Insulating(), Impermeable(), ImpermeableTracer())
│     init_state_conservative = init_heldsuarez!
└     data_config = HeldSuarezDataConfig{Float32}(255.0f0)
┌ Info: Establishing Atmos GCM configuration for HeldSuarez
│     precision        = Float32
│     polynomial order = 5
│     #horiz elems     = 5
│     #vert elems      = 5
│     domain height    = 3.00e+04 m
│     MPI ranks        = 1
│     min(Δ_horz)      = 167863.59 m
└     min(Δ_vert)      = 703.00 m
[ Info: Initializing HeldSuarez
┌ Info: Starting HeldSuarez
│     dt              = 9.83472e+01
│     timeend         = 4320000.00
│     number of steps = 43926
└     norm(Q)         = 7.7109628239872000e+13
┌ Info: Diagnostics: AtmosGCMDefault
└     initializing at     0.00
┌ Info: Diagnostics: AtmosRefStatePerturbations
└     initializing at     0.00
┌ Info: Update
│     simtime =    98.35 / 4320000.00
│     runtime = 00:01:25
└     norm(Q) = 7.7109670182912000e+13
┌ Info: Update
│     simtime = 641142.56 / 4320000.00
│     runtime = 00:02:25
└     norm(Q) = 7.7314964586496000e+13
┌ Info: Update
│     simtime = 1285793.88 / 4320000.00
│     runtime = 00:03:25
└     norm(Q) = 7.7022369939456000e+13
┌ Info: Update
│     simtime = 1931035.50 / 4320000.00
│     runtime = 00:04:25
└     norm(Q) = 7.6565509570560000e+13
┌ Info: Update
│     simtime = 2575177.00 / 4320000.00
│     runtime = 00:05:25
└     norm(Q) = 7.6085488254976000e+13
┌ Info: Update
│     simtime = 3218026.75 / 4320000.00
│     runtime = 00:06:25
└     norm(Q) = 7.5674404519936000e+13
┌ Info: Update
│     simtime = 3861269.50 / 4320000.00
│     runtime = 00:07:25
└     norm(Q) = 7.5343482322944000e+13
┌ Info: Diagnostics: AtmosGCMDefault
└     collecting at 3932500.75
┌ Info: Diagnostics: AtmosRefStatePerturbations
└     collecting at 3932500.75
┌ Info: Finished
│     norm(Q)            = 7.5177597599744000e+13
│     norm(Q) / norm(Q₀) = 9.7494435310363770e-01
└     norm(Q) - norm(Q₀) = -1.9320306401280000e+12
┌ Info: Euclidean distance
│     norm(Q - Qe)            = 1.3744651370496000e+13
└     norm(Q - Qe) / norm(Qe) = 1.7824810743331909e-01

out_10192378

+ hostname
hpc-26-17.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192378_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192378_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project experiments/AtmosLES/surfacebubble.jl
┌ Info: Model composition
│     param_set = EarthParameterSet()
│     orientation = FlatOrientation()
│     ref_state = HydrostaticState{ClimateMachine.TemperatureProfiles.DecayingTemperatureProfile{Float64},Float64}(ClimateMachine.TemperatureProfiles.DecayingTemperatureProfile{Float64}(290.0, 220.0, 8484.271021693852), 0.0)
│     turbulence = SmagorinskyLilly{Float64}(0.23)
│     hyperdiffusion = NoHyperDiffusion()
│     moisture = EquilMoist{Float64}(3, 0.1)
│     precipitation = NoPrecipitation()
│     radiation = NoRadiation()
│     source = (Gravity(),)
│     tracers = NoTracers()
│     boundarycondition = (AtmosBC{Impenetrable{FreeSlip},PrescribedEnergyFlux{var"#energyflux#3"{Float64,Float64,Float64}},Impermeable,ImpermeableTracer}(Impenetrable{FreeSlip}(FreeSlip()), PrescribedEnergyFlux{var"#energyflux#3"{Float64,Float64,Float64}}(var"#energyflux#3"{Float64,Float64,Float64}(100.0, 500.0, 2000.0)), Impermeable(), ImpermeableTracer()), AtmosBC{Impenetrable{FreeSlip},Insulating,Impermeable,ImpermeableTracer}(Impenetrable{FreeSlip}(FreeSlip()), Insulating(), Impermeable(), ImpermeableTracer()))
│     init_state_conservative = init_surfacebubble!
└     data_config = nothing
┌ Info: Establishing Atmos LES configuration for SurfaceDrivenBubble
│     precision        = Float64
│     polynomial order = 4
│     domain           = 2000.00 m x2000.00 m x2000.00 m
│     resolution       = 50x50x50
│     MPI ranks        = 3
│     min(Δ_horz)      = 34.53 m
└     min(Δ_vert)      = 34.53 m
[ Info: Initializing SurfaceDrivenBubble
┌ Info: Starting SurfaceDrivenBubble
│     dt              = 1.69090e-01
│     timeend         =  2000.00
│     number of steps = 11828
└     norm(Q)         = 2.1322438724310231e+09
┌ Info: Update
│     simtime =     0.17 /  2000.00
│     runtime = 00:01:22
└     norm(Q) = 2.1322440522654135e+09
┌ Info: Update
│     simtime =   145.59 /  2000.00
│     runtime = 00:02:16
└     norm(Q) = 2.1324025164797907e+09
┌ Info: Update
│     simtime =   306.05 /  2000.00
│     runtime = 00:03:17
└     norm(Q) = 2.1325797428294134e+09
┌ Info: Update
│     simtime =   466.01 /  2000.00
│     runtime = 00:04:17
└     norm(Q) = 2.1327562491547997e+09
┌ Info: Update
│     simtime =   626.82 /  2000.00
│     runtime = 00:05:17
└     norm(Q) = 2.1327885613798244e+09
┌ Info: Update
│     simtime =   787.28 /  2000.00
│     runtime = 00:06:17
└     norm(Q) = 2.1327878918894100e+09
┌ Info: Update
│     simtime =   947.24 /  2000.00
│     runtime = 00:07:17
└     norm(Q) = 2.1327886115414832e+09
┌ Info: Update
│     simtime =  1107.20 /  2000.00
│     runtime = 00:08:17
└     norm(Q) = 2.1327900342847409e+09
┌ Info: Update
│     simtime =  1267.16 /  2000.00
│     runtime = 00:09:17
└     norm(Q) = 2.1327917133057196e+09
┌ Info: Update
│     simtime =  1426.78 /  2000.00
│     runtime = 00:10:17
└     norm(Q) = 2.1327933610664892e+09
┌ Info: Update
│     simtime =  1586.91 /  2000.00
│     runtime = 00:11:17
└     norm(Q) = 2.1327946216983054e+09
┌ Info: Update
│     simtime =  1747.38 /  2000.00
│     runtime = 00:12:17
└     norm(Q) = 2.1327956783707702e+09
┌ Info: Update
│     simtime =  1907.85 /  2000.00
│     runtime = 00:13:17
└     norm(Q) = 2.1327965908799269e+09
┌ Info: Finished
│     norm(Q)            = 2.1327970037534018e+09
│     norm(Q) / norm(Q₀) = 1.0002594127855309e+00
└     norm(Q) - norm(Q₀) = 5.5313132237863541e+05
┌ Info: Euclidean distance
│     norm(Q - Qe)            = 8.8329683488067647e+05
└     norm(Q - Qe) / norm(Qe) = 4.1425694607512612e-04

out_10192379

+ hostname
hpc-26-18.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192379_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192379_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project experiments/AtmosLES/dycoms.jl --fix-rng-seed
┌ Info: Model composition
│     param_set = EarthParameterSet()
│     orientation = FlatOrientation()
│     ref_state = HydrostaticState{DecayingTemperatureProfile{Float64},Float64}(DecayingTemperatureProfile{Float64}(290.0, 220.0, 8484.271021693852), 0.0)
│     turbulence = Vreman{Float64}(0.21)
│     hyperdiffusion = NoHyperDiffusion()
│     moisture = EquilMoist{Float64}(4, 1.0)
│     precipitation = NoPrecipitation()
│     radiation = DYCOMSRadiation{Float64}(85.0, 1.0, 840.0, 1.13, 3.75e-6, 70.0, 22.0)
│     source = (Gravity(), RayleighSponge{Float64}(1500.0, 1000.0, 1.0, [7.0, -5.5, 0.0], 2.0), Subsidence{Float64}(3.75e-6), GeostrophicForcing{Float64}(7.62e-5, 7.0, -5.5))
│     tracers = NoTracers()
│     boundarycondition = (AtmosBC{Impenetrable{DragLaw{var"#3#6"{Float64}}},PrescribedEnergyFlux{var"#4#7"{Float64,Float64}},PrescribedMoistureFlux{var"#5#8"{Float64}},ImpermeableTracer}(Impenetrable{DragLaw{var"#3#6"{Float64}}}(DragLaw{var"#3#6"{Float64}}(var"#3#6"{Float64}(0.0011))), PrescribedEnergyFlux{var"#4#7"{Float64,Float64}}(var"#4#7"{Float64,Float64}(115.0, 15.0)), PrescribedMoistureFlux{var"#5#8"{Float64}}(var"#5#8"{Float64}(4.5985284708893154e-5)), ImpermeableTracer()), AtmosBC{Impenetrable{FreeSlip},Insulating,Impermeable,ImpermeableTracer}(Impenetrable{FreeSlip}(FreeSlip()), Insulating(), Impermeable(), ImpermeableTracer()))
│     init_state_conservative = init_dycoms!
└     data_config = nothing
┌ Info: Establishing Atmos LES configuration for DYCOMS
│     precision        = Float64
│     polynomial order = 4
│     domain           = 1000.00 m x1000.00 m x1500.00 m
│     resolution       = 40x40x20
│     MPI ranks        = 3
│     min(Δ_horz)      = 28.78 m
└     min(Δ_vert)      = 13.63 m
[ Info: Initializing DYCOMS
┌ Info: Starting DYCOMS
│     dt              = 6.58328e-02
│     timeend         =   100.00
│     number of steps = 1519
└     norm(Q)         = 1.3882828058036425e+09
┌ Info: Update
│     simtime =     0.07 /   100.00
│     runtime = 00:01:27
└     norm(Q) = 1.3882922559466660e+09
┌ Info: Update
│     simtime =    45.75 /   100.00
│     runtime = 00:02:22
└     norm(Q) = 1.3887393776786017e+09
┌ Info: Update
│     simtime =    95.92 /   100.00
│     runtime = 00:03:22
└     norm(Q) = 1.3889493995617003e+09
┌ Info: Finished
│     norm(Q)            = 1.3890005332065828e+09
│     norm(Q) / norm(Q₀) = 1.0005169893338302e+00
└     norm(Q) - norm(Q₀) = 7.1772740294027328e+05
┌ Info: Euclidean distance
│     norm(Q - Qe)            = 4.5348763400610955e+06
└     norm(Q - Qe) / norm(Qe) = 3.2665412212834681e-03

out_10192380

+ hostname
hpc-26-18.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192380_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192380_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project experiments/AtmosLES/bomex.jl --diagnostics=default --fix-rng-seed
┌ Info: Model composition
│     param_set = EarthParameterSet()
│     orientation = FlatOrientation()
│     ref_state = HydrostaticState{ClimateMachine.TemperatureProfiles.DecayingTemperatureProfile{Float32},Float32}(ClimateMachine.TemperatureProfiles.DecayingTemperatureProfile{Float32}(290.0f0, 220.0f0, 8484.2705f0), 0.0f0)
│     turbulence = SmagorinskyLilly{Float32}(0.23f0)
│     hyperdiffusion = NoHyperDiffusion()
│     moisture = EquilMoist{Float32}(5, 0.1f0)
│     precipitation = NoPrecipitation()
│     radiation = NoRadiation()
│     source = (Gravity(), BomexTendencies{Float32}(-1.2f-8, 300.0f0, 500.0f0, -2.3148148f-5, 1500.0f0, 2100.0f0, -0.0065f0, 3000.0f0), BomexSponge{Float32}(3000.0f0, 2400.0f0, 0.75f0, 2.0f0, -10.0f0, 0.0018f0, 0.0f0), BomexGeostrophic{Float32}(3.76f-5, -10.0f0, 0.0018f0, 0.0f0))
│     tracers = NoTracers()
│     boundarycondition = (AtmosBC{Impenetrable{DragLaw{var"#3#6"{Float32}}},PrescribedEnergyFlux{var"#4#7"{Float32,Float32}},PrescribedMoistureFlux{var"#5#8"{Float32}},ImpermeableTracer}(Impenetrable{DragLaw{var"#3#6"{Float32}}}(DragLaw{var"#3#6"{Float32}}(var"#3#6"{Float32}(0.28f0))), PrescribedEnergyFlux{var"#4#7"{Float32,Float32}}(var"#4#7"{Float32,Float32}(147.2f0, 9.5f0)), PrescribedMoistureFlux{var"#5#8"{Float32}}(var"#5#8"{Float32}(6.0388535f-5)), ImpermeableTracer()), AtmosBC{Impenetrable{FreeSlip},Insulating,Impermeable,ImpermeableTracer}(Impenetrable{FreeSlip}(FreeSlip()), Insulating(), Impermeable(), ImpermeableTracer()))
│     init_state_conservative = init_bomex!
└     data_config = nothing
┌ Info: Establishing Atmos LES configuration for BOMEX
│     precision        = Float32
│     polynomial order = 4
│     domain           = 6400.00 m x6400.00 m x3000.00 m
│     resolution       = 100x100x40
│     MPI ranks        = 1
│     min(Δ_horz)      = 69.07 m
└     min(Δ_vert)      = 27.26 m
[ Info: Initializing BOMEX
┌ Info: Starting BOMEX
│     dt              = 6.86866e-02
│     timeend         =  1800.00
│     number of steps = 26206
└     norm(Q)         = 1.8651301888000000e+10
┌ Info: Diagnostics: AtmosLESDefault
└     initializing at     0.00
┌ Info: Diagnostics: AtmosLESCore
└     initializing at     0.00
┌ Info: Update
│     simtime =     0.07 /  1800.00
│     runtime = 00:01:17
└     norm(Q) = 1.8651303936000000e+10
┌ Info: Diagnostics: AtmosLESDefault
└     collecting at   171.71
┌ Info: Diagnostics: AtmosLESCore
└     collecting at   171.71
┌ Info: Update
│     simtime =   171.78 /  1800.00
│     runtime = 00:02:25
└     norm(Q) = 1.8676748288000000e+10
abs(δρ) = 7.101638f-6
abs(δρe) = 9.288385f-5
┌ Info: Update
│     simtime =   342.13 /  1800.00
│     runtime = 00:03:25
└     norm(Q) = 1.8691690496000000e+10
┌ Info: Diagnostics: AtmosLESDefault
└     collecting at   343.43
┌ Info: Diagnostics: AtmosLESCore
└     collecting at   343.43
abs(δρ) = 2.4036312f-6
abs(δρe) = 8.616339f-5
┌ Info: Update
│     simtime =   489.20 /  1800.00
│     runtime = 00:04:25
└     norm(Q) = 1.8679865344000000e+10
┌ Info: Diagnostics: AtmosLESDefault
└     collecting at   515.17
┌ Info: Diagnostics: AtmosLESCore
└     collecting at   515.17
abs(δρ) = 6.664614f-6
abs(δρe) = 0.00010101915f0
┌ Info: Update
│     simtime =   635.74 /  1800.00
│     runtime = 00:05:25
└     norm(Q) = 1.8685325312000000e+10
┌ Info: Diagnostics: AtmosLESDefault
└     collecting at   686.83
┌ Info: Diagnostics: AtmosLESCore
└     collecting at   686.83
┌ Info: Update
│     simtime =   786.60 /  1800.00
│     runtime = 00:06:25
└     norm(Q) = 1.8683992064000000e+10
abs(δρ) = 3.5398934f-5
abs(δρe) = 0.00013525813f0
┌ Info: Diagnostics: AtmosLESDefault
└     collecting at   858.49
┌ Info: Diagnostics: AtmosLESCore
└     collecting at   858.49
┌ Info: Update
│     simtime =   933.13 /  1800.00
│     runtime = 00:07:25
└     norm(Q) = 1.8684899328000000e+10
abs(δρ) = 3.070093f-5
abs(δρe) = 0.00016334259f0
┌ Info: Diagnostics: AtmosLESDefault
└     collecting at  1030.16
┌ Info: Diagnostics: AtmosLESCore
└     collecting at  1030.16
┌ Info: Update
│     simtime =  1079.37 /  1800.00
│     runtime = 00:08:25
└     norm(Q) = 1.8684882944000000e+10
┌ Info: Diagnostics: AtmosLESDefault
└     collecting at  1201.97
┌ Info: Diagnostics: AtmosLESCore
└     collecting at  1201.97
┌ Info: Update
│     simtime =  1229.19 /  1800.00
│     runtime = 00:09:25
└     norm(Q) = 1.8682902528000000e+10
abs(δρ) = 3.7802565f-5
abs(δρe) = 0.0001840699f0
┌ Info: Diagnostics: AtmosLESDefault
└     collecting at  1373.79
┌ Info: Diagnostics: AtmosLESCore
└     collecting at  1373.79
┌ Info: Update
│     simtime =  1375.23 /  1800.00
│     runtime = 00:10:25
└     norm(Q) = 1.8681876480000000e+10
abs(δρ) = 4.5013458f-5
abs(δρe) = 0.00018972924f0
┌ Info: Diagnostics: AtmosLESDefault
└     collecting at  1545.60
┌ Info: Diagnostics: AtmosLESCore
└     collecting at  1545.60
┌ Info: Update
│     simtime =  1545.67 /  1800.00
│     runtime = 00:11:33
└     norm(Q) = 1.8680188928000000e+10
abs(δρ) = 4.905593f-5
abs(δρe) = 0.00020140162f0
┌ Info: Diagnostics: AtmosLESDefault
└     collecting at  1717.41
┌ Info: Diagnostics: AtmosLESCore
└     collecting at  1717.41
┌ Info: Update
│     simtime =  1717.48 /  1800.00
│     runtime = 00:12:42
└     norm(Q) = 1.8679459840000000e+10
┌ Info: Finished
│     norm(Q)            = 1.8679347200000000e+10
│     norm(Q) / norm(Q₀) = 1.0015037059783936e+00
└     norm(Q) - norm(Q₀) = 2.8045312000000000e+07
┌ Info: Euclidean distance
│     norm(Q - Qe)            = 2.8356508800000000e+08
└     norm(Q - Qe) / norm(Qe) = 1.5203412622213364e-02

out_10192381

+ hostname
hpc-26-18.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192381_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192381_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project experiments/OceanBoxGCM/homogeneous_box.jl
[ Info: Initializing homogeneous_box
┌ Info: Calculating timestep
│ Advective Constraint    = Inf seconds
│ Nondiffusive Constraint = 55.1 seconds
│ Viscous Constraint      = 16866.5 seconds
│ Diffusive Constrait     = 84332.4 seconds
└ Timestep                = 55.1 seconds
┌ Info: Starting homogeneous_box
│     dt              = 5.51020e+01
│     timeend         = 21600.00
│     number of steps = 392
└     norm(Q)         = 1.6012272062093482e+09
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/experiments/OceanBoxGCM/homogeneous_box.jl: Error During Test at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/experiments/OceanBoxGCM/homogeneous_box.jl:106
  Got exception outside of a @test
  DimensionMismatch("parent has 1630000 elements, which is incompatible with size (25, 5, 4, 20, 133)")
  Stacktrace:
   [1] _reshape at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu/packages/CUDA/42B9G/src/array.jl:228 [inlined]
   [2] reshape(::CUDA.CuArray{Float64,3,Nothing}, ::NTuple{5,Int64}) at ./reshapedarray.jl:112
   [3] reshape(::CUDA.CuArray{Float64,3,Nothing}, ::Int64, ::Int64, ::Vararg{Int64,N} where N) at ./reshapedarray.jl:116
   [4] update_auxiliary_state_gradient!(::ClimateMachine.DGMethods.DGModel{HydrostaticBoussinesqModel{EarthParameterSet,HomogeneousBox{Float64,Tuple{OceanBC{Impenetrable{NoSlip},Insulating},OceanBC{Impenetrable{NoSlip},Insulating},OceanBC{Penetrable{KinematicStress},Insulating}}},Float64},ClimateMachine.Mesh.Grids.DiscontinuousSpectralElementGrid{Float64,3,4,125,CUDA.CuArray,CUDA.CuArray{Float64,1,Nothing},CUDA.CuArray{Float64,2,Nothing},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Float64,4,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Int64,2,Nothing},CUDA.CuArray{Int64,3,Nothing},ClimateMachine.Mesh.Topologies.StackedBrickTopology{3,Float64}},ClimateMachine.DGMethods.NumericalFluxes.RusanovNumericalFlux,ClimateMachine.DGMethods.NumericalFluxes.CentralNumericalFluxSecondOrder,ClimateMachine.DGMethods.NumericalFluxes.CentralNumericalFluxGradient,ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:y, :w, :pkin, :wz0),NTuple{4,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:ν∇u, :κ∇θ),Tuple{StaticArrays.SArray{Tuple{3,2},Float64,2,6},StaticArrays.SArray{Tuple{3},Float64,1,3}}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},Tuple{ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(),Tuple{}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(),Tuple{}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}}},ClimateMachine.Mesh.Grids.EveryDirection,ClimateMachine.Mesh.Grids.EveryDirection,NamedTuple{(:vert_filter, :exp_filter),Tuple{CutoffFilter,ExponentialFilter}}}, ::HydrostaticBoussinesqModel{EarthParameterSet,HomogeneousBox{Float64,Tuple{OceanBC{Impenetrable{NoSlip},Insulating},OceanBC{Impenetrable{NoSlip},Insulating},OceanBC{Penetrable{KinematicStress},Insulating}}},Float64}, ::ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:u, :η, :θ),Tuple{StaticArrays.SArray{Tuple{2},Float64,1,2},Float64,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}}, ::Float64, ::UnitRange{Int64}) at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/src/Ocean/HydrostaticBoussinesq/HydrostaticBoussinesqModel.jl:644
   [5] (::ClimateMachine.DGMethods.DGModel{HydrostaticBoussinesqModel{EarthParameterSet,HomogeneousBox{Float64,Tuple{OceanBC{Impenetrable{NoSlip},Insulating},OceanBC{Impenetrable{NoSlip},Insulating},OceanBC{Penetrable{KinematicStress},Insulating}}},Float64},ClimateMachine.Mesh.Grids.DiscontinuousSpectralElementGrid{Float64,3,4,125,CUDA.CuArray,CUDA.CuArray{Float64,1,Nothing},CUDA.CuArray{Float64,2,Nothing},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Float64,4,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Int64,2,Nothing},CUDA.CuArray{Int64,3,Nothing},ClimateMachine.Mesh.Topologies.StackedBrickTopology{3,Float64}},ClimateMachine.DGMethods.NumericalFluxes.RusanovNumericalFlux,ClimateMachine.DGMethods.NumericalFluxes.CentralNumericalFluxSecondOrder,ClimateMachine.DGMethods.NumericalFluxes.CentralNumericalFluxGradient,ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:y, :w, :pkin, :wz0),NTuple{4,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:ν∇u, :κ∇θ),Tuple{StaticArrays.SArray{Tuple{3,2},Float64,2,6},StaticArrays.SArray{Tuple{3},Float64,1,3}}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},Tuple{ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(),Tuple{}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(),Tuple{}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}}},ClimateMachine.Mesh.Grids.EveryDirection,ClimateMachine.Mesh.Grids.EveryDirection,NamedTuple{(:vert_filter, :exp_filter),Tuple{CutoffFilter,ExponentialFilter}}})(::ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:u, :η, :θ),Tuple{StaticArrays.SArray{Tuple{2},Float64,1,2},Float64,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}}, ::ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:u, :η, :θ),Tuple{StaticArrays.SArray{Tuple{2},Float64,1,2},Float64,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}}, ::Nothing, ::Float64, ::Bool, ::Bool) at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/src/Numerics/DGMethods/DGModel.jl:281
   [6] #_#16 at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/src/Numerics/DGMethods/DGModel.jl:99 [inlined]
   [7] dostep!(::ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:u, :η, :θ),Tuple{StaticArrays.SArray{Tuple{2},Float64,1,2},Float64,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}}, ::AdditiveRungeKutta{Float64,Float64,ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:u, :η, :θ),Tuple{StaticArrays.SArray{Tuple{2},Float64,1,2},Float64,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},ClimateMachine.ODESolvers.LinBESolver{Float64,ClimateMachine.SystemSolvers.ColumnwiseLU{ClimateMachine.SystemSolvers.DGColumnBandedMatrix{3,4,4,133,20,2,true,CUDA.CuArray{Float64,2,Nothing}}},ClimateMachine.SystemSolvers.SingleColumnLU,ClimateMachine.DGMethods.DGModel{LinearHBModel{HydrostaticBoussinesqModel{EarthParameterSet,HomogeneousBox{Float64,Tuple{OceanBC{Impenetrable{NoSlip},Insulating},OceanBC{Impenetrable{NoSlip},Insulating},OceanBC{Penetrable{KinematicStress},Insulating}}},Float64}},ClimateMachine.Mesh.Grids.DiscontinuousSpectralElementGrid{Float64,3,4,125,CUDA.CuArray,CUDA.CuArray{Float64,1,Nothing},CUDA.CuArray{Float64,2,Nothing},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Float64,4,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Int64,2,Nothing},CUDA.CuArray{Int64,3,Nothing},ClimateMachine.Mesh.Topologies.StackedBrickTopology{3,Float64}},ClimateMachine.DGMethods.NumericalFluxes.RusanovNumericalFlux,ClimateMachine.DGMethods.NumericalFluxes.CentralNumericalFluxSecondOrder,ClimateMachine.DGMethods.NumericalFluxes.CentralNumericalFluxGradient,ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:y, :w, :pkin, :wz0),NTuple{4,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:ν∇u, :κ∇θ),Tuple{StaticArrays.SArray{Tuple{3,2},Float64,2,6},StaticArrays.SArray{Tuple{3},Float64,1,3}}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},Tuple{ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(),Tuple{}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(),Tuple{}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}}},ClimateMachine.Mesh.Grids.VerticalDirection,ClimateMachine.Mesh.Grids.VerticalDirection,Nothing}},LowStorageVariant,NamedTuple{(:Qtt,),Tuple{ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:u, :η, :θ),Tuple{StaticArrays.SArray{Tuple{2},Float64,1,2},Float64,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}}}},3,9,2}, ::LowStorageVariant, ::Nothing, ::Float64, ::Nothing, ::Nothing, ::Nothing) at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/src/Numerics/ODESolvers/AdditiveRungeKuttaMethod.jl:328
   [8] dostep! at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/src/Numerics/ODESolvers/AdditiveRungeKuttaMethod.jl:190 [inlined] (repeats 2 times)
   [9] general_dostep!(::ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:u, :η, :θ),Tuple{StaticArrays.SArray{Tuple{2},Float64,1,2},Float64,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}}, ::AdditiveRungeKutta{Float64,Float64,ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:u, :η, :θ),Tuple{StaticArrays.SArray{Tuple{2},Float64,1,2},Float64,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},ClimateMachine.ODESolvers.LinBESolver{Float64,ClimateMachine.SystemSolvers.ColumnwiseLU{ClimateMachine.SystemSolvers.DGColumnBandedMatrix{3,4,4,133,20,2,true,CUDA.CuArray{Float64,2,Nothing}}},ClimateMachine.SystemSolvers.SingleColumnLU,ClimateMachine.DGMethods.DGModel{LinearHBModel{HydrostaticBoussinesqModel{EarthParameterSet,HomogeneousBox{Float64,Tuple{OceanBC{Impenetrable{NoSlip},Insulating},OceanBC{Impenetrable{NoSlip},Insulating},OceanBC{Penetrable{KinematicStress},Insulating}}},Float64}},ClimateMachine.Mesh.Grids.DiscontinuousSpectralElementGrid{Float64,3,4,125,CUDA.CuArray,CUDA.CuArray{Float64,1,Nothing},CUDA.CuArray{Float64,2,Nothing},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Float64,4,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Int64,2,Nothing},CUDA.CuArray{Int64,3,Nothing},ClimateMachine.Mesh.Topologies.StackedBrickTopology{3,Float64}},ClimateMachine.DGMethods.NumericalFluxes.RusanovNumericalFlux,ClimateMachine.DGMethods.NumericalFluxes.CentralNumericalFluxSecondOrder,ClimateMachine.DGMethods.NumericalFluxes.CentralNumericalFluxGradient,ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:y, :w, :pkin, :wz0),NTuple{4,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:ν∇u, :κ∇θ),Tuple{StaticArrays.SArray{Tuple{3,2},Float64,2,6},StaticArrays.SArray{Tuple{3},Float64,1,3}}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},Tuple{ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(),Tuple{}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(),Tuple{}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}}},ClimateMachine.Mesh.Grids.VerticalDirection,ClimateMachine.Mesh.Grids.VerticalDirection,Nothing}},LowStorageVariant,NamedTuple{(:Qtt,),Tuple{ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:u, :η, :θ),Tuple{StaticArrays.SArray{Tuple{2},Float64,1,2},Float64,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}}}},3,9,2}, ::Nothing, ::Float64; adjustfinalstep::Bool) at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/src/Numerics/ODESolvers/ODESolvers.jl:57
   [10] solve!(::ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:u, :η, :θ),Tuple{StaticArrays.SArray{Tuple{2},Float64,1,2},Float64,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}}, ::AdditiveRungeKutta{Float64,Float64,ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:u, :η, :θ),Tuple{StaticArrays.SArray{Tuple{2},Float64,1,2},Float64,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},ClimateMachine.ODESolvers.LinBESolver{Float64,ClimateMachine.SystemSolvers.ColumnwiseLU{ClimateMachine.SystemSolvers.DGColumnBandedMatrix{3,4,4,133,20,2,true,CUDA.CuArray{Float64,2,Nothing}}},ClimateMachine.SystemSolvers.SingleColumnLU,ClimateMachine.DGMethods.DGModel{LinearHBModel{HydrostaticBoussinesqModel{EarthParameterSet,HomogeneousBox{Float64,Tuple{OceanBC{Impenetrable{NoSlip},Insulating},OceanBC{Impenetrable{NoSlip},Insulating},OceanBC{Penetrable{KinematicStress},Insulating}}},Float64}},ClimateMachine.Mesh.Grids.DiscontinuousSpectralElementGrid{Float64,3,4,125,CUDA.CuArray,CUDA.CuArray{Float64,1,Nothing},CUDA.CuArray{Float64,2,Nothing},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Float64,4,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Int64,2,Nothing},CUDA.CuArray{Int64,3,Nothing},ClimateMachine.Mesh.Topologies.StackedBrickTopology{3,Float64}},ClimateMachine.DGMethods.NumericalFluxes.RusanovNumericalFlux,ClimateMachine.DGMethods.NumericalFluxes.CentralNumericalFluxSecondOrder,ClimateMachine.DGMethods.NumericalFluxes.CentralNumericalFluxGradient,ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:y, :w, :pkin, :wz0),NTuple{4,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:ν∇u, :κ∇θ),Tuple{StaticArrays.SArray{Tuple{3,2},Float64,2,6},StaticArrays.SArray{Tuple{3},Float64,1,3}}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},Tuple{ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(),Tuple{}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(),Tuple{}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}}},ClimateMachine.Mesh.Grids.VerticalDirection,ClimateMachine.Mesh.Grids.VerticalDirection,Nothing}},LowStorageVariant,NamedTuple{(:Qtt,),Tuple{ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:u, :η, :θ),Tuple{StaticArrays.SArray{Tuple{2},Float64,1,2},Float64,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}}}},3,9,2}, ::Nothing; timeend::Float64, adjustfinalstep::Bool, numberofsteps::Int64, callbacks::Tuple{EveryXWallTimeSeconds}) at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/src/Numerics/ODESolvers/ODESolvers.jl:118
   [11] invoke!(::ClimateMachine.SolverConfiguration{Float64}; diagnostics_config::Nothing, user_callbacks::Tuple{}, check_euclidean_distance::Bool, adjustfinalstep::Bool, user_info_callback::Function) at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/src/Driver/Driver.jl:615
   [12] invoke! at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/src/Driver/Driver.jl:514 [inlined]
   [13] run_homogeneous_box(; imex::Bool, BC::Tuple{OceanBC{Impenetrable{NoSlip},Insulating},OceanBC{Impenetrable{NoSlip},Insulating},OceanBC{Penetrable{KinematicStress},Insulating}}) at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/experiments/OceanBoxGCM/homogeneous_box.jl:92
   [14] macro expansion at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/experiments/OceanBoxGCM/homogeneous_box.jl:131 [inlined]
   [15] macro expansion at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Test/src/Test.jl:1113 [inlined]
   [16] top-level scope at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/experiments/OceanBoxGCM/homogeneous_box.jl:107
   [17] include(::Module, ::String) at ./Base.jl:377
   [18] exec_options(::Base.JLOptions) at ./client.jl:288
   [19] _start() at ./client.jl:484
  
Test Summary:                                                                                                              | Error  Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/experiments/OceanBoxGCM/homogeneous_box.jl |     1      1
ERROR: LoadError: Some tests did not pass: 0 passed, 0 failed, 1 errored, 0 broken.
in expression starting at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/experiments/OceanBoxGCM/homogeneous_box.jl:106
--------------------------------------------------------------------------
Primary job  terminated normally, but 1 process returned
a non-zero exit code. Per user-direction, the job has been aborted.
--------------------------------------------------------------------------
--------------------------------------------------------------------------
mpiexec detected that one or more processes exited with non-zero status, thus causing
the job to be terminated. The first process to do so was:

  Process name: [[3980,1],1]
  Exit code:    1
--------------------------------------------------------------------------

out_10192382

+ hostname
hpc-26-18.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192382_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192382_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project experiments/OceanBoxGCM/ocean_gyre.jl
[ Info: Initializing ocean_gyre
┌ Info: Calculating timestep
│ Advective Constraint    = Inf seconds
│ Nondiffusive Constraint = 139.5 seconds
│ Viscous Constraint      = 5963.2 seconds
│ Diffusive Constrait     = 298.2 seconds
└ Timestep                = 139.5 seconds
# SC Start: creating state check callback
# SC Creating state check callback labeled "Q" for symbols
# SC u
# SC η
# SC θ
# SC Creating state check callback labeled "s_aux" for symbols
# SC y
# SC w
# SC pkin
# SC wz0
# SC Creating state check callback labeled "s_gflux" for symbols
# SC ν∇u
# SC κ∇θ
# SC Finish: creating state check callback
┌ Info: Starting ocean_gyre
│     dt              = 1.39355e+02
│     timeend         = 86400.00
│     number of steps = 620
└     norm(Q)         = 4.1952353926806128e+08
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/experiments/OceanBoxGCM/ocean_gyre.jl: Error During Test at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/experiments/OceanBoxGCM/ocean_gyre.jl:139
  Got exception outside of a @test
  DimensionMismatch("parent has 1590000 elements, which is incompatible with size (25, 5, 4, 20, 134)")
  Stacktrace:
   [1] _reshape at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu/packages/CUDA/42B9G/src/array.jl:228 [inlined]
   [2] reshape(::CUDA.CuArray{Float64,3,Nothing}, ::NTuple{5,Int64}) at ./reshapedarray.jl:112
   [3] reshape(::CUDA.CuArray{Float64,3,Nothing}, ::Int64, ::Int64, ::Vararg{Int64,N} where N) at ./reshapedarray.jl:116
   [4] update_auxiliary_state_gradient!(::ClimateMachine.DGMethods.DGModel{HydrostaticBoussinesqModel{EarthParameterSet,OceanGyre{Float64,Tuple{OceanBC{Impenetrable{NoSlip},Insulating},OceanBC{Impenetrable{NoSlip},Insulating},OceanBC{Penetrable{KinematicStress},TemperatureFlux}}},Float64},ClimateMachine.Mesh.Grids.DiscontinuousSpectralElementGrid{Float64,3,4,125,CUDA.CuArray,CUDA.CuArray{Float64,1,Nothing},CUDA.CuArray{Float64,2,Nothing},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Float64,4,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Int64,2,Nothing},CUDA.CuArray{Int64,3,Nothing},ClimateMachine.Mesh.Topologies.StackedBrickTopology{3,Float64}},ClimateMachine.DGMethods.NumericalFluxes.RusanovNumericalFlux,ClimateMachine.DGMethods.NumericalFluxes.CentralNumericalFluxSecondOrder,ClimateMachine.DGMethods.NumericalFluxes.CentralNumericalFluxGradient,ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:y, :w, :pkin, :wz0),NTuple{4,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:ν∇u, :κ∇θ),Tuple{StaticArrays.SArray{Tuple{3,2},Float64,2,6},StaticArrays.SArray{Tuple{3},Float64,1,3}}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},Tuple{ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(),Tuple{}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(),Tuple{}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}}},ClimateMachine.Mesh.Grids.EveryDirection,ClimateMachine.Mesh.Grids.EveryDirection,NamedTuple{(:vert_filter, :exp_filter),Tuple{CutoffFilter,ExponentialFilter}}}, ::HydrostaticBoussinesqModel{EarthParameterSet,OceanGyre{Float64,Tuple{OceanBC{Impenetrable{NoSlip},Insulating},OceanBC{Impenetrable{NoSlip},Insulating},OceanBC{Penetrable{KinematicStress},TemperatureFlux}}},Float64}, ::ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:u, :η, :θ),Tuple{StaticArrays.SArray{Tuple{2},Float64,1,2},Float64,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}}, ::Float64, ::UnitRange{Int64}) at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/src/Ocean/HydrostaticBoussinesq/HydrostaticBoussinesqModel.jl:644
   [5] (::ClimateMachine.DGMethods.DGModel{HydrostaticBoussinesqModel{EarthParameterSet,OceanGyre{Float64,Tuple{OceanBC{Impenetrable{NoSlip},Insulating},OceanBC{Impenetrable{NoSlip},Insulating},OceanBC{Penetrable{KinematicStress},TemperatureFlux}}},Float64},ClimateMachine.Mesh.Grids.DiscontinuousSpectralElementGrid{Float64,3,4,125,CUDA.CuArray,CUDA.CuArray{Float64,1,Nothing},CUDA.CuArray{Float64,2,Nothing},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Float64,4,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Int64,2,Nothing},CUDA.CuArray{Int64,3,Nothing},ClimateMachine.Mesh.Topologies.StackedBrickTopology{3,Float64}},ClimateMachine.DGMethods.NumericalFluxes.RusanovNumericalFlux,ClimateMachine.DGMethods.NumericalFluxes.CentralNumericalFluxSecondOrder,ClimateMachine.DGMethods.NumericalFluxes.CentralNumericalFluxGradient,ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:y, :w, :pkin, :wz0),NTuple{4,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:ν∇u, :κ∇θ),Tuple{StaticArrays.SArray{Tuple{3,2},Float64,2,6},StaticArrays.SArray{Tuple{3},Float64,1,3}}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},Tuple{ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(),Tuple{}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(),Tuple{}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}}},ClimateMachine.Mesh.Grids.EveryDirection,ClimateMachine.Mesh.Grids.EveryDirection,NamedTuple{(:vert_filter, :exp_filter),Tuple{CutoffFilter,ExponentialFilter}}})(::ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:u, :η, :θ),Tuple{StaticArrays.SArray{Tuple{2},Float64,1,2},Float64,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}}, ::ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:u, :η, :θ),Tuple{StaticArrays.SArray{Tuple{2},Float64,1,2},Float64,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}}, ::Nothing, ::Float64, ::Bool, ::Bool) at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/src/Numerics/DGMethods/DGModel.jl:281
   [6] #_#16 at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/src/Numerics/DGMethods/DGModel.jl:99 [inlined]
   [7] dostep!(::ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:u, :η, :θ),Tuple{StaticArrays.SArray{Tuple{2},Float64,1,2},Float64,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}}, ::LowStorageRungeKutta2N{Float64,Float64,ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:u, :η, :θ),Tuple{StaticArrays.SArray{Tuple{2},Float64,1,2},Float64,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},14}, ::Nothing, ::Float64, ::Nothing, ::Nothing, ::Nothing) at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/src/Numerics/ODESolvers/LowStorageRungeKuttaMethod.jl:94
   [8] dostep! at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/src/Numerics/ODESolvers/LowStorageRungeKuttaMethod.jl:83 [inlined]
   [9] #general_dostep!#1 at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/src/Numerics/ODESolvers/ODESolvers.jl:57 [inlined]
   [10] solve!(::ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:u, :η, :θ),Tuple{StaticArrays.SArray{Tuple{2},Float64,1,2},Float64,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}}, ::LowStorageRungeKutta2N{Float64,Float64,ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:u, :η, :θ),Tuple{StaticArrays.SArray{Tuple{2},Float64,1,2},Float64,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},14}, ::Nothing; timeend::Float64, adjustfinalstep::Bool, numberofsteps::Int64, callbacks::Tuple{EveryXSimulationSteps,EveryXWallTimeSeconds}) at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/src/Numerics/ODESolvers/ODESolvers.jl:118
   [11] invoke!(::ClimateMachine.SolverConfiguration{Float64}; diagnostics_config::Nothing, user_callbacks::Array{EveryXSimulationSteps,1}, check_euclidean_distance::Bool, adjustfinalstep::Bool, user_info_callback::Function) at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/src/Driver/Driver.jl:615
   [12] run_ocean_gyre(; imex::Bool, BC::Tuple{OceanBC{Impenetrable{NoSlip},Insulating},OceanBC{Impenetrable{NoSlip},Insulating},OceanBC{Penetrable{KinematicStress},TemperatureFlux}}, nt::Int64, refDat::Tuple{Array{Array{Any,1},1},Array{Array{Any,1},1}}) at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/experiments/OceanBoxGCM/ocean_gyre.jl:109
   [13] macro expansion at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/experiments/OceanBoxGCM/ocean_gyre.jl:210 [inlined]
   [14] macro expansion at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Test/src/Test.jl:1113 [inlined]
   [15] top-level scope at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/experiments/OceanBoxGCM/ocean_gyre.jl:141
   [16] include(::Module, ::String) at ./Base.jl:377
   [17] exec_options(::Base.JLOptions) at ./client.jl:288
   [18] _start() at ./client.jl:484
  
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/experiments/OceanBoxGCM/ocean_gyre.jl: Test Summary:                                                                                                         | Error  Total
Error During Test at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/experiments/OceanBoxGCM/ocean_gyre.jl |     1      1
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/experiments/OceanBoxGCM/ocean_gyre.jl:139
  Got exception outside of a @test
  DimensionMismatch("parent has 1580000 elements, which is incompatible with size (25, 5, 4, 20, 133)")
  Stacktrace:
   [1] _reshape at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu/packages/CUDA/42B9G/src/array.jl:228 [inlined]
   [2] reshape(::CUDA.CuArray{Float64,3,Nothing}, ::NTuple{5,Int64}) at ./reshapedarray.jl:112
   [3] reshape(::CUDA.CuArray{Float64,3,Nothing}, ::Int64, ::Int64, ::Vararg{Int64,N} where N) at ./reshapedarray.jl:116
   [4] update_auxiliary_state_gradient!(::ClimateMachine.DGMethods.DGModel{HydrostaticBoussinesqModel{EarthParameterSet,OceanGyre{Float64,Tuple{OceanBC{Impenetrable{NoSlip},Insulating},OceanBC{Impenetrable{NoSlip},Insulating},OceanBC{Penetrable{KinematicStress},TemperatureFlux}}},Float64},ClimateMachine.Mesh.Grids.DiscontinuousSpectralElementGrid{Float64,3,4,125,CUDA.CuArray,CUDA.CuArray{Float64,1,Nothing},CUDA.CuArray{Float64,2,Nothing},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Float64,4,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Int64,2,Nothing},CUDA.CuArray{Int64,3,Nothing},ClimateMachine.Mesh.Topologies.StackedBrickTopology{3,Float64}},ClimateMachine.DGMethods.NumericalFluxes.RusanovNumericalFlux,ClimateMachine.DGMethods.NumericalFluxes.CentralNumericalFluxSecondOrder,ClimateMachine.DGMethods.NumericalFluxes.CentralNumericalFluxGradient,ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:y, :w, :pkin, :wz0),NTuple{4,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:ν∇u, :κ∇θ),Tuple{StaticArrays.SArray{Tuple{3,2},Float64,2,6},StaticArrays.SArray{Tuple{3},Float64,1,3}}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},Tuple{ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(),Tuple{}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(),Tuple{}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}}},ClimateMachine.Mesh.Grids.EveryDirection,ClimateMachine.Mesh.Grids.EveryDirection,NamedTuple{(:vert_filter, :exp_filter),Tuple{CutoffFilter,ExponentialFilter}}}, ::HydrostaticBoussinesqModel{EarthParameterSet,OceanGyre{Float64,Tuple{OceanBC{Impenetrable{NoSlip},Insulating},OceanBC{Impenetrable{NoSlip},Insulating},OceanBC{Penetrable{KinematicStress},TemperatureFlux}}},Float64}, ::ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:u, :η, :θ),Tuple{StaticArrays.SArray{Tuple{2},Float64,1,2},Float64,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}}, ::Float64, ::UnitRange{Int64}) at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/src/Ocean/HydrostaticBoussinesq/HydrostaticBoussinesqModel.jl:644
   [5] (::ClimateMachine.DGMethods.DGModel{HydrostaticBoussinesqModel{EarthParameterSet,OceanGyre{Float64,Tuple{OceanBC{Impenetrable{NoSlip},Insulating},OceanBC{Impenetrable{NoSlip},Insulating},OceanBC{Penetrable{KinematicStress},TemperatureFlux}}},Float64},ClimateMachine.Mesh.Grids.DiscontinuousSpectralElementGrid{Float64,3,4,125,CUDA.CuArray,CUDA.CuArray{Float64,1,Nothing},CUDA.CuArray{Float64,2,Nothing},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Float64,4,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Int64,2,Nothing},CUDA.CuArray{Int64,3,Nothing},ClimateMachine.Mesh.Topologies.StackedBrickTopology{3,Float64}},ClimateMachine.DGMethods.NumericalFluxes.RusanovNumericalFlux,ClimateMachine.DGMethods.NumericalFluxes.CentralNumericalFluxSecondOrder,ClimateMachine.DGMethods.NumericalFluxes.CentralNumericalFluxGradient,ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:y, :w, :pkin, :wz0),NTuple{4,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:ν∇u, :κ∇θ),Tuple{StaticArrays.SArray{Tuple{3,2},Float64,2,6},StaticArrays.SArray{Tuple{3},Float64,1,3}}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},Tuple{ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(),Tuple{}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(),Tuple{}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}}},ClimateMachine.Mesh.Grids.EveryDirection,ClimateMachine.Mesh.Grids.EveryDirection,NamedTuple{(:vert_filter, :exp_filter),Tuple{CutoffFilter,ExponentialFilter}}})(::ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:u, :η, :θ),Tuple{StaticArrays.SArray{Tuple{2},Float64,1,2},Float64,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}}, ::ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:u, :η, :θ),Tuple{StaticArrays.SArray{Tuple{2},Float64,1,2},Float64,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}}, ::Nothing, ::Float64, ::Bool, ::Bool) at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/src/Numerics/DGMethods/DGModel.jl:281
   [6] #_#16 at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/src/Numerics/DGMethods/DGModel.jl:99 [inlined]
   [7] dostep!(::ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:u, :η, :θ),Tuple{StaticArrays.SArray{Tuple{2},Float64,1,2},Float64,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}}, ::LowStorageRungeKutta2N{Float64,Float64,ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:u, :η, :θ),Tuple{StaticArrays.SArray{Tuple{2},Float64,1,2},Float64,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},14}, ::Nothing, ::Float64, ::Nothing, ::Nothing, ::Nothing) at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/src/Numerics/ODESolvers/LowStorageRungeKuttaMethod.jl:94
   [8] dostep! at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/src/Numerics/ODESolvers/LowStorageRungeKuttaMethod.jl:83 [inlined]
   [9] #general_dostep!#1 at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/src/Numerics/ODESolvers/ODESolvers.jl:57 [inlined]
   [10] solve!(::ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:u, :η, :θ),Tuple{StaticArrays.SArray{Tuple{2},Float64,1,2},Float64,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}}, ::LowStorageRungeKutta2N{Float64,Float64,ClimateMachine.MPIStateArrays.MPIStateArray{Float64,NamedTuple{(:u, :η, :θ),Tuple{StaticArrays.SArray{Tuple{2},Float64,1,2},Float64,Float64}},CUDA.CuArray{Float64,3,Nothing},CUDA.CuArray{Int64,1,Nothing},CUDA.CuArray{Float64,3,CUDA.CuArray{Float64,3,Nothing}},ClimateMachine.MPIStateArrays.CMBuffers.CMBuffer{Float64,CUDA.CuArray{Float64,2,Nothing},Nothing}},14}, ::Nothing; timeend::Float64, adjustfinalstep::Bool, numberofsteps::Int64, callbacks::Tuple{EveryXSimulationSteps,EveryXWallTimeSeconds}) at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/src/Numerics/ODESolvers/ODESolvers.jl:118
   [11] invoke!(::ClimateMachine.SolverConfiguration{Float64}; diagnostics_config::Nothing, user_callbacks::Array{EveryXSimulationSteps,1}, check_euclidean_distance::Bool, adjustfinalstep::Bool, user_info_callback::Function) at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/src/Driver/Driver.jl:615
   [12] run_ocean_gyre(; imex::Bool, BC::Tuple{OceanBC{Impenetrable{NoSlip},Insulating},OceanBC{Impenetrable{NoSlip},Insulating},OceanBC{Penetrable{KinematicStress},TemperatureFlux}}, nt::Int64, refDat::Tuple{Array{Array{Any,1},1},Array{Array{Any,1},1}}) at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/experiments/OceanBoxGCM/ocean_gyre.jl:109
   [13] macro expansion at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/experiments/OceanBoxGCM/ocean_gyre.jl:210 [inlined]
   [14] macro expansion at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Test/src/Test.jl:1113 [inlined]
   [15] top-level scope at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/experiments/OceanBoxGCM/ocean_gyre.jl:141
   [16] include(::Module, ::String) at ./Base.jl:377
   [17] exec_options(::Base.JLOptions) at ./client.jl:288
   [18] _start() at ./client.jl:484
  
ERROR: Test Summary:                                                                                                         | Error  LoadError: Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/experiments/OceanBoxGCM/ocean_gyre.jl |     1      1
Some tests did not pass: 0 passed, 0 failed, 1 errored, 0 broken.
in expression starting at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/experiments/OceanBoxGCM/ocean_gyre.jl:139
ERROR: LoadError: Some tests did not pass: 0 passed, 0 failed, 1 errored, 0 broken.
in expression starting at /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/experiments/OceanBoxGCM/ocean_gyre.jl:139
--------------------------------------------------------------------------
Primary job  terminated normally, but 1 process returned
a non-zero exit code. Per user-direction, the job has been aborted.
--------------------------------------------------------------------------
--------------------------------------------------------------------------
mpiexec detected that one or more processes exited with non-zero status, thus causing
the job to be terminated. The first process to do so was:

  Process name: [[3968,1],2]
  Exit code:    1
--------------------------------------------------------------------------

out_10192383

+ hostname
hpc-89-37.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192383_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192383_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Arrays/varsindex.jl
Test Summary:           | Pass  Total
MPIStateArray varsindex |    9      9

out_10192384

+ hostname
hpc-89-37.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192384_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192384_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Diagnostics/diagnostic_fields_test.jl
┌ Info: Model composition
│     param_set = EarthParameterSet()
│     orientation = FlatOrientation()
│     ref_state = HydrostaticState{DryAdiabaticProfile{Float32},Float32}(DryAdiabaticProfile{Float32}(290.0f0, 220.0f0), 0.0f0)
│     turbulence = SmagorinskyLilly{Float32}(0.23f0)
│     hyperdiffusion = NoHyperDiffusion()
│     moisture = EquilMoist{Float32}(3, 0.1f0)
│     precipitation = NoPrecipitation()
│     radiation = NoRadiation()
│     source = (Gravity(),)
│     tracers = NoTracers()
│     boundarycondition = AtmosBC{Impenetrable{FreeSlip},Insulating,Impermeable,ImpermeableTracer}(Impenetrable{FreeSlip}(FreeSlip()), Insulating(), Impermeable(), ImpermeableTracer())
│     init_state_conservative = init_risingbubble!
└     data_config = nothing
┌ Info: Establishing Atmos LES configuration for DryRisingBubble
│     precision        = Float32
│     polynomial order = 4
│     domain           = 2500.00 m x2500.00 m x2500.00 m
│     resolution       = 50x50x50
│     MPI ranks        = 3
│     min(Δ_horz)      = 35.97 m
└     min(Δ_vert)      = 35.97 m
[ Info: Initializing DryRisingBubble
┌ Info: Model composition
│     param_set = EarthParameterSet()
│     orientation = FlatOrientation()
│     ref_state = HydrostaticState{DryAdiabaticProfile{Float64},Float64}(DryAdiabaticProfile{Float64}(290.0, 220.0), 0.0)
│     turbulence = SmagorinskyLilly{Float64}(0.23)
│     hyperdiffusion = NoHyperDiffusion()
│     moisture = EquilMoist{Float64}(3, 0.1)
│     precipitation = NoPrecipitation()
│     radiation = NoRadiation()
│     source = (Gravity(),)
│     tracers = NoTracers()
│     boundarycondition = AtmosBC{Impenetrable{FreeSlip},Insulating,Impermeable,ImpermeableTracer}(Impenetrable{FreeSlip}(FreeSlip()), Insulating(), Impermeable(), ImpermeableTracer())
│     init_state_conservative = init_risingbubble!
└     data_config = nothing
┌ Info: Establishing Atmos LES configuration for DryRisingBubble
│     precision        = Float64
│     polynomial order = 4
│     domain           = 2500.00 m x2500.00 m x2500.00 m
│     resolution       = 50x50x50
│     MPI ranks        = 3
│     min(Δ_horz)      = 35.97 m
└     min(Δ_vert)      = 35.97 m
[ Info: Initializing DryRisingBubble
Test Summary:            | Test Summary:            | Test Summary:            | Pass  Pass  TotalTotal

Pass  Total
Diagnostics Fields testsDiagnostics Fields tests |  |    2     2      2
    2
Diagnostics Fields tests |    2      2

out_10192385

+ hostname
hpc-89-37.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192385_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192385_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/vars_test.jl
[ Info: (CUDA.CuArray, Float64, 2)
[ Info: (CUDA.CuArray, Float64, 3)

out_10192386

+ hostname
hpc-89-37.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192386_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192386_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/remainder_model.jl
┌ Warning: `a::StaticArray - b::Number` is deprecated, use `a .- b` instead.
│   caller = macro expansion at remainder_model.jl:215 [inlined]
└ @ Core /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/remainder_model.jl:215
Test Summary:   | Pass  Total
remainder model |   24     24
Test Summary:   | Pass  Test Summary:   | Total
Pass  Total
remainder model |   24     24
remainder model |   24     24

out_10192387

+ hostname
hpc-89-37.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192387_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192387_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/Euler/isentropicvortex.jl
┌ Info: Configuration
│ ArrayType     = CUDA.CuArray
│ FT        = Float64
│ NumericalFlux = RusanovNumericalFlux
└ dims          = 2
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 0)
│ dt        = 3.5087719298245615e-06
└ norm(Q₀)  = 3.5435113910177142e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 3.5434920766920145e+03
│ norm(Q) / norm(Q₀)      = 9.9999454938235877e-01
│ norm(Q) - norm(Q₀)      = -1.9314325699724577e-02
│ norm(Q - Qe)            = 1.1990999506536756e+01
└ norm(Q - Qe) / norm(Qe) = 3.3837129375777690e-03
┌ Info: Configuration
│ ArrayType     = CUDA.CuArray
│ FT        = Float64
│ NumericalFlux = CentralNumericalFluxFirstOrder
└ dims          = 2
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 0)
│ dt        = 3.5087719298245615e-06
└ norm(Q₀)  = 3.5435113910177142e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 3.5435098602154872e+03
│ norm(Q) / norm(Q₀)      = 9.9999956799850254e-01
│ norm(Q) - norm(Q₀)      = -1.5308022270801303e-03
│ norm(Q - Qe)            = 2.0840574601661469e+01
└ norm(Q - Qe) / norm(Qe) = 5.8809544498566787e-03
┌ Info: Configuration
│ ArrayType     = CUDA.CuArray
│ FT        = Float64
│ NumericalFlux = RusanovNumericalFlux
└ dims          = 3
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 1)
│ dt        = 3.5087719298245615e-06
└ norm(Q₀)  = 1.1205566910367500e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 1.1205505833106820e+03
│ norm(Q) / norm(Q₀)      = 9.9999454938235899e-01
│ norm(Q) - norm(Q₀)      = -6.1077260679667233e-03
│ norm(Q - Qe)            = 3.7918869862611517e+00
└ norm(Q - Qe) / norm(Qe) = 3.3837129375777928e-03
┌ Info: Configuration
│ ArrayType     = CUDA.CuArray
│ FT        = Float64
│ NumericalFlux = CentralNumericalFluxFirstOrder
└ dims          = 3
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 1)
│ dt        = 3.5087719298245615e-06
└ norm(Q₀)  = 1.1205566910367500e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 1.1205562069545817e+03
│ norm(Q) / norm(Q₀)      = 9.9999956799850276e-01
│ norm(Q) - norm(Q₀)      = -4.8408216821371752e-04
│ norm(Q - Qe)            = 6.5903683487905829e+00
└ norm(Q - Qe) / norm(Qe) = 5.8809544498566102e-03
┌ Info: Configuration
│ ArrayType     = CUDA.CuArray
│ FT        = Float32
│ NumericalFlux = RusanovNumericalFlux
└ dims          = 2
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 0)
│ dt        = 3.5087718970316928e-06
└ norm(Q₀)  = 3.5435100097656250e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 3.5434895019531250e+03
│ norm(Q) / norm(Q₀)      = 9.9999421834945679e-01
│ norm(Q) - norm(Q₀)      = -2.0507812500000000e-02
│ norm(Q - Qe)            = 1.1990933418273926e+01
└ norm(Q - Qe) / norm(Qe) = 3.3836953807622194e-03
┌ Info: Configuration
│ ArrayType     = CUDA.CuArray
│ FT        = Float32
│ NumericalFlux = CentralNumericalFluxFirstOrder
└ dims          = 2
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 0)
│ dt        = 3.5087718970316928e-06
└ norm(Q₀)  = 3.5435100097656250e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 3.5435073242187500e+03
│ norm(Q) / norm(Q₀)      = 9.9999922513961792e-01
│ norm(Q) - norm(Q₀)      = -2.6855468750000000e-03
│ norm(Q - Qe)            = 2.0840526580810547e+01
└ norm(Q - Qe) / norm(Qe) = 5.8809430338442326e-03
┌ Info: Configuration
│ ArrayType     = CUDA.CuArray
│ FT        = Float32
│ NumericalFlux = RusanovNumericalFlux
└ dims          = 3
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 1)
│ dt        = 3.5087718970316928e-06
└ norm(Q₀)  = 1.1205562744140625e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 1.1205500488281250e+03
│ norm(Q) / norm(Q₀)      = 9.9999445676803589e-01
│ norm(Q) - norm(Q₀)      = -6.2255859375000000e-03
│ norm(Q - Qe)            = 3.7918405532836914e+00
└ norm(Q - Qe) / norm(Qe) = 3.3836725633591413e-03
┌ Info: Configuration
│ ArrayType     = CUDA.CuArray
│ FT        = Float32
│ NumericalFlux = CentralNumericalFluxFirstOrder
└ dims          = 3
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 1)
│ dt        = 3.5087718970316928e-06
└ norm(Q₀)  = 1.1205562744140625e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 1.1205556640625000e+03
│ norm(Q) / norm(Q₀)      = 9.9999946355819702e-01
│ norm(Q) - norm(Q₀)      = -6.1035156250000000e-04
│ norm(Q - Qe)            = 6.5903930664062500e+00
└ norm(Q - Qe) / norm(Qe) = 5.8809784241020679e-03
Test Summary:                                                                                                                     | Pass  Test Summary:                                                                                                                     | Test Summary:                                                                                                                     | Pass  Pass  Total
TotalTotal

/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/Euler/isentropicvortex.jl |    8  /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/Euler/isentropicvortex.jl | /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/Euler/isentropicvortex.jl |     8
   8     8      8
    8

out_10192388

+ hostname
hpc-89-37.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192388_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192388_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/Euler/isentropicvortex_imex.jl
┌ Info: Configuration
│ ArrayType = CUDA.CuArray
│ FT    = Float64
│ dims      = 2
└ splitting = (Full, Linear)
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 0)
│ dt        = 3.5842293906810039e-06
└ norm(Q₀)  = 3.5435113910177142e+03
┌ Info: Update
│ simtime = 2.5089605734767022e-05
│ runtime = 00:01:02
└ norm(Q) = 3.5435007042115940e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 3.5433940589518693e+03
│ norm(Q) / norm(Q₀)      = 9.9996688819284107e-01
│ norm(Q) - norm(Q₀)      = -1.1733206584494837e-01
│ norm(Q - Qe)            = 2.3225467541888278e+01
└ norm(Q - Qe) / norm(Qe) = 6.5540110824175339e-03
┌ Info: Configuration
│ ArrayType = CUDA.CuArray
│ FT    = Float64
│ dims      = 2
└ splitting = (Nonlinear, Linear)
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 0)
│ dt        = 3.5842293906810039e-06
└ norm(Q₀)  = 3.5435113910177142e+03
┌ Info: Update
│ simtime = 5.9139784946236375e-04
│ runtime = 00:01:00
└ norm(Q) = 3.5434030990373981e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 3.5433940589518702e+03
│ norm(Q) / norm(Q₀)      = 9.9996688819284130e-01
│ norm(Q) - norm(Q₀)      = -1.1733206584403888e-01
│ norm(Q - Qe)            = 2.3225467541886843e+01
└ norm(Q - Qe) / norm(Qe) = 6.5540110824171289e-03
Test Summary:                                                                                                                          | Pass  Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/Euler/isentropicvortex_imex.jl |    2      2
Test Summary:                                                                                                                          | Pass  Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/Euler/isentropicvortex_imex.jl |    2      2
Test Summary:                                                                                                                          | Pass  Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/Euler/isentropicvortex_imex.jl |    2      2

out_10192389

+ hostname
hpc-26-18.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192389_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192389_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/Euler/isentropicvortex_multirate.jl
┌ Info: Configuration
│ ArrayType  = CUDA.CuArray
│ FastMethod = SSPRK33ShuOsher
│ FT     = Float64
└ dims       = 2
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 0)
│ slow_dt   = 2.7777777777777782e-05
│ fast_dt   = 9.2592592592592608e-06
└ norm(Q₀)  = 3.5435113910177142e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 3.5433927502798711e+03
│ norm(Q) / norm(Q₀)      = 9.9996651887781596e-01
│ norm(Q) - norm(Q₀)      = -1.1864073784317952e-01
│ norm(Q - Qe)            = 2.3222373077773376e+01
└ norm(Q - Qe) / norm(Qe) = 6.5531378534043122e-03
┌ Info: Configuration
│ ArrayType  = CUDA.CuArray
│ FastMethod = ARK2GiraldoKellyConstantinescu
│ FT     = Float64
└ dims       = 2
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 0)
│ slow_dt   = 2.7777777777777782e-05
│ fast_dt   = 9.2592592592592608e-06
└ norm(Q₀)  = 3.5435113910177142e+03
┌ Info: Update
│ simtime = 3.0555555555555555e-04
│ runtime = 00:01:02
└ norm(Q) = 3.5434426895952515e+03
┌ Info: Update
│ simtime = 6.6666666666666675e-04
│ runtime = 00:02:05
└ norm(Q) = 3.5433931034199973e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 3.5433931034199973e+03
│ norm(Q) / norm(Q₀)      = 9.9996661853606095e-01
│ norm(Q) - norm(Q₀)      = -1.1828759771697150e-01
│ norm(Q - Qe)            = 2.3245216640100868e+01
└ norm(Q - Qe) / norm(Qe) = 6.5595840943846609e-03
Test Summary:                                                                                                                               | Pass  Test Summary:                                                                                                                               | Test Summary:                                                                                                                               | Pass  Pass  Total
Total
Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/Euler/isentropicvortex_multirate.jl |    2      2
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/Euler/isentropicvortex_multirate.jl | /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/Euler/isentropicvortex_multirate.jl |    2     2      2
    2

out_10192390

+ hostname
hpc-26-18.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192390_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192390_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/Euler/isentropicvortex_mrigark.jl
┌ Info: Configuration
│ ArrayType  = CUDA.CuArray
│ mrigark_method = MRIGARKERK33aSandu
│ FT     = Float64
└ dims       = 2
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 0)
│ slow_dt   = 7.1684587813620078e-06
│ fast_dt   = 2.3894862604540028e-06
└ norm(Q₀)  = 3.5435113910177142e+03
┌ Info: Update
│ simtime = 3.4408602150537665e-04
│ runtime = 00:01:00
└ norm(Q) = 3.5434375147833189e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 3.5433917654007209e+03
│ norm(Q) / norm(Q₀)      = 9.9996624093905939e-01
│ norm(Q) - norm(Q₀)      = -1.1962561699328944e-01
│ norm(Q - Qe)            = 2.3357934866484523e+01
└ norm(Q - Qe) / norm(Qe) = 6.5913921302649522e-03
┌ Info: Configuration
│ ArrayType  = CUDA.CuArray
│ mrigark_method = MRIGARKERK45aSandu
│ FT     = Float64
└ dims       = 2
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 0)
│ slow_dt   = 7.1684587813620078e-06
│ fast_dt   = 2.3894862604540028e-06
└ norm(Q₀)  = 3.5435113910177142e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 3.5433940921715789e+03
│ norm(Q) / norm(Q₀)      = 9.9996689756764079e-01
│ norm(Q) - norm(Q₀)      = -1.1729884613532704e-01
│ norm(Q - Qe)            = 2.3207510164249381e+01
└ norm(Q - Qe) / norm(Qe) = 6.5489436773440201e-03
Test Summary:                                                                                                                             | Pass  Test Summary:                                                                                                                             | Pass  Total
Test Summary:                                                                                                                             | Pass  Total
Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/Euler/isentropicvortex_mrigark.jl |    2      2
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/Euler/isentropicvortex_mrigark.jl |    2      2
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/Euler/isentropicvortex_mrigark.jl |    2      2

out_10192391

+ hostname
hpc-89-37.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192391_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192391_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/Euler/isentropicvortex_mrigark_implicit.jl
┌ Info: Configuration
│ ArrayType      = CUDA.CuArray
│ mrigark_method = MRIGARKIRK21aSandu
│ FT             = Float64
└ dims           = 2
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 0)
│ dt        = 7.1994240460763143e-07
└ norm(Q₀)  = 3.5435113910177142e+03
┌ Info: Update
│ simtime = 7.1994240460763143e-07
│ runtime = 00:01:31
└ norm(Q) = 3.5435114340963905e+03
┌ Info: Update
│ simtime = 6.2994960403167622e-04
│ runtime = 00:02:31
└ norm(Q) = 3.5433992536744290e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 3.5433941432216752e+03
│ norm(Q) / norm(Q₀)      = 9.9996691197428167e-01
│ norm(Q) - norm(Q₀)      = -1.1724779603900970e-01
│ norm(Q - Qe)            = 2.3236071337680961e+01
└ norm(Q - Qe) / norm(Qe) = 6.5570033750383646e-03
┌ Info: Configuration
│ ArrayType      = CUDA.CuArray
│ mrigark_method = MRIGARKESDIRK34aSandu
│ FT             = Float64
└ dims           = 2
┌ Info: Starting refinement level 1
│ numelems  = (5, 5, 0)
│ dt        = 7.1994240460763143e-07
└ norm(Q₀)  = 3.5435113910177142e+03
┌ Info: Update
│ simtime = 1.9942404607631352e-04
│ runtime = 00:01:00
└ norm(Q) = 3.5434605749760690e+03
┌ Info: Update
│ simtime = 4.1036717062634906e-04
│ runtime = 00:02:00
└ norm(Q) = 3.5434283881537640e+03
┌ Info: Update
│ simtime = 6.2131029517638466e-04
│ runtime = 00:03:00
└ norm(Q) = 3.5434004626044675e+03
┌ Info: Finished refinement level 1
│ norm(Q)                 = 3.5433941177190109e+03
│ norm(Q) / norm(Q₀)      = 9.9996690477727812e-01
│ norm(Q) - norm(Q₀)      = -1.1727329870336689e-01
│ norm(Q - Qe)            = 2.3235626679097365e+01
└ norm(Q - Qe) / norm(Qe) = 6.5568778965187386e-03
Test Summary:                                                                                                                                      | Test Summary:                                                                                                                                      | Test Summary:                                                                                                                                      | Pass  Pass  Pass  Total
Total
Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/Euler/isentropicvortex_mrigark_implicit.jl | /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/Euler/isentropicvortex_mrigark_implicit.jl | /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/Euler/isentropicvortex_mrigark_implicit.jl |    2     2     2      2
    2
    2

out_10192392

+ hostname
hpc-26-17.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192392_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192392_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/Euler/acousticwave_1d_imex.jl
┌ Info: Starting
│ ArrayType       = CUDA.CuArray
│ FT              = Float32
│ polynomialorder = 5
│ numelem_horz    = 10
│ numelem_vert    = 5
│ dt              = 1.0000000000000000e+02
└ norm(Q₀)        = 9.5070778818560000e+13
┌ Info: Update
│ simtime = 1.0000000000000000e+02
│ runtime = 00:01:18
└ norm(Q) = 9.5070652989440000e+13
┌ Info: Finished
│ norm(Q)                 = 9.5070938202112000e+13
│ norm(Q) / norm(Q₀)      = 1.0000016689300537e+00
└ norm(Q) - norm(Q₀)      = 1.5938355200000000e+08
┌ Info: Starting
│ ArrayType       = CUDA.CuArray
│ FT              = Float32
│ polynomialorder = 5
│ numelem_horz    = 10
│ numelem_vert    = 5
│ dt              = 1.0000000000000000e+02
└ norm(Q₀)        = 9.5070778818560000e+13
┌ Info: Finished
│ norm(Q)                 = 9.5070938202112000e+13
│ norm(Q) / norm(Q₀)      = 1.0000016689300537e+00
└ norm(Q) - norm(Q₀)      = 1.5938355200000000e+08
┌ Info: Starting
│ ArrayType       = CUDA.CuArray
│ FT              = Float64
│ polynomialorder = 5
│ numelem_horz    = 10
│ numelem_vert    = 5
│ dt              = 1.0000000000000000e+02
└ norm(Q₀)        = 9.5073561639231469e+13
┌ Info: Update
│ simtime = 1.0000000000000000e+02
│ runtime = 00:01:14
└ norm(Q) = 9.5073433073024078e+13
┌ Info: Finished
│ norm(Q)                 = 9.5073452874031266e+13
│ norm(Q) / norm(Q₀)      = 9.9999885598900129e-01
└ norm(Q) - norm(Q₀)      = -1.0876520020312500e+08
┌ Info: Starting
│ ArrayType       = CUDA.CuArray
│ FT              = Float64
│ polynomialorder = 5
│ numelem_horz    = 10
│ numelem_vert    = 5
│ dt              = 1.0000000000000000e+02
└ norm(Q₀)        = 9.5073561639231469e+13
┌ Info: Finished
│ norm(Q)                 = 9.5073452874031266e+13
│ norm(Q) / norm(Q₀)      = 9.9999885598900129e-01
└ norm(Q) - norm(Q₀)      = -1.0876520020312500e+08

out_10192393

+ hostname
hpc-23-28.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192393_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192393_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/Euler/acousticwave_mrigark.jl
┌ Info: Starting
│ ArrayType       = CUDA.CuArray
│ FT              = Float64
│ polynomialorder = 5
│ numelem_horz    = 10
│ numelem_vert    = 5
│ acoustic dt     = 6.7670313998798237e-01
│ remainder_dt    = 3.3834586466165413e+00
└ norm(Q₀)        = 9.5073561639231469e+13
┌ Info: Update
│ simtime = 3.3834586466165413e+00
│ runtime = 00:01:44
└ norm(Q) = 9.5073541244901906e+13
┌ Info: Update
│ simtime = 3.1736842105263040e+03
│ runtime = 00:02:43
└ norm(Q) = 9.5073461117755609e+13
┌ Info: Finished
│ norm(Q)                 = 9.5073337896189500e+13
│ norm(Q) / norm(Q₀)      = 9.9999764663237489e-01
└ norm(Q) - norm(Q₀)      = -2.2374304196875000e+08
┌ Info: Starting
│ ArrayType       = CUDA.CuArray
│ FT              = Float64
│ polynomialorder = 5
│ numelem_horz    = 10
│ numelem_vert    = 5
│ acoustic dt     = 1.3333333333333334e+02
│ remainder_dt    = 2.3494467606676028e+02
└ norm(Q₀)        = 9.5073561639231469e+13
┌ Info: Finished
│ norm(Q)                 = 9.5073455097540781e+13
│ norm(Q) / norm(Q₀)      = 9.9999887937625509e-01
└ norm(Q) - norm(Q₀)      = -1.0654169068750000e+08
Test Summary: | Pass  Test Summary: | Pass  Total
Total
Test Summary: | Pass  Total
acoustic wave |    2      2
acoustic wave |    2      2
acoustic wave |    2      2

out_10192394

+ hostname
hpc-26-18.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192394_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192394_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/compressible_Navier_Stokes/mms_bc_atmos.jl
[ Info: (CUDA.CuArray, Float64, 2, 400, 0.0025)
┌ Info: Starting
└ norm(Q₀) = 1.4718488046151106e+02
┌ Info: Finished
│ norm(Q)                 = 1.4749573751578205e+02
│ norm(Q) / norm(Q₀)      = 1.0021120175747420e+00
│ norm(Q) - norm(Q₀)      = 3.1085705427099697e-01
│ norm(Q - Qe)            = 1.6931876910307139e-01
└ norm(Q - Qe) / norm(Qe) = 1.1479958671721405e-03
[ Info: (CUDA.CuArray, Float64, 3, 800, 0.00125)
┌ Info: Starting
└ norm(Q₀) = 1.1341641833287574e+02
┌ Info: Update
│ simtime = 1.2500000000000000e-03
│ runtime = 00:01:09
└ norm(Q) = 1.1341642292274383e+02
┌ Info: Finished
│ norm(Q)                 = 1.1338036285141953e+02
│ norm(Q) / norm(Q₀)      = 9.9968209645493844e-01
│ norm(Q) - norm(Q₀)      = -3.6055481456202187e-02
│ norm(Q - Qe)            = 3.3983777728925434e-02
└ norm(Q - Qe) / norm(Qe) = 2.9973197445392301e-04
Test Summary: | Pass  Test Summary: | Total
Pass  Test Summary: | Pass  Total
Total
mms_bc_atmos  |    4      4
mms_bc_atmos  |    4  mms_bc_atmos  |     4
   4      4

out_10192395

+ hostname
hpc-89-37.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192395_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192395_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/compressible_Navier_Stokes/mms_bc_dgmodel.jl
[ Info: (CUDA.CuArray, Float64, 2)
┌ Info: Starting
└ norm(Q₀) = 1.4718488046151109e+02
┌ Info: Finished
│ norm(Q)                 = 1.4749698909866964e+02
│ norm(Q) / norm(Q₀)      = 1.0021205210493083e+00
│ norm(Q) - norm(Q₀)      = 3.1210863715855908e-01
│ norm(Q - Qe)            = 1.6694721292986217e-01
└ norm(Q - Qe) / norm(Qe) = 1.1319165116462717e-03
[ Info: (CUDA.CuArray, Float64, 3)
┌ Info: Starting
└ norm(Q₀) = 1.1341641833287574e+02
┌ Info: Update
│ simtime = 1.2500000000000000e-03
│ runtime = 00:01:00
└ norm(Q) = 1.1341642290285348e+02
┌ Info: Finished
│ norm(Q)                 = 1.1338058691327824e+02
│ norm(Q) / norm(Q₀)      = 9.9968407202304399e-01
│ norm(Q) - norm(Q₀)      = -3.5831419597499803e-02
│ norm(Q - Qe)            = 3.3672443923201095e-02
└ norm(Q - Qe) / norm(Qe) = 2.9698605559085961e-04
Test Summary:  | Pass  Test Summary:  | Test Summary:  | Pass  Pass  Total
Total
Total
mms_bc_dgmodel |    2  mms_bc_dgmodel | mms_bc_dgmodel |     2
   2     2      2
    2

out_10192396

+ hostname
hpc-89-37.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192396_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192396_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/compressible_Navier_Stokes/density_current_model.jl
┌ Info: Starting
│ norm(Q₀) = 4.3981214720000000e+09
│ ArrayType = CUDA.CuArray
└ FloatType = Float32
┌ Info: Update
│ simtime = 9.9999997764825821e-03
│ runtime = 00:01:32
└ norm(Q) = 4.3981214720000000e+09
┌ Info: Finished
│ norm(Q)                 = 4.3981199360000000e+09
│ norm(Q) / norm(Q₀)      = 9.9999964237213135e-01
│ norm(Q) - norm(Q₀)      = -1.5360000000000000e+03
│ norm(Q - Qe)            = 3.5941828125000000e+04
└ norm(Q - Qe) / norm(Qe) = 8.1720863818190992e-06
┌ Info: Starting
│ norm(Q₀) = 4.3981206422010155e+09
│ ArrayType = CUDA.CuArray
└ FloatType = Float64
┌ Info: Update
│ simtime = 1.0000000000000000e-02
│ runtime = 00:01:13
└ norm(Q) = 4.3981206294035711e+09
┌ Info: Finished
│ norm(Q)                 = 4.3981193634268847e+09
│ norm(Q) / norm(Q₀)      = 9.9999970924532666e-01
│ norm(Q) - norm(Q₀)      = -1.2787741308212280e+03
│ norm(Q - Qe)            = 3.5863022189968789e+04
└ norm(Q - Qe) / norm(Qe) = 8.1541697255538071e-06

out_10192397

+ hostname
hpc-26-18.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192397_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192397_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/advection_diffusion/direction_splitting_advection_diffusion.jl --fix-rng-seed
Test Summary:                                                                                                                                                          | Test Summary:                                                                                                                                                          | Pass  Pass  Test Summary:                                                                                                                                                          | Total
Total
Pass  Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/direction_splitting_advection_diffusion.jl | /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/direction_splitting_advection_diffusion.jl |  162   162    162
  162
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/direction_splitting_advection_diffusion.jl |  162    162

out_10192398

+ hostname
hpc-89-37.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192398_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192398_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/conservation/sphere.jl --fix-rng-seed
[ Info: (CUDA.CuArray, Float64)
┌ Info: Starting
│ norm(Q₀) = 4.4181152030340352e+00
└ sum(Q₀) = 2.9255250415427088e+01
┌ Info: Finished
│ norm(Q)            = 4.4304590011362341e+00
│ norm(Q) / norm(Q₀) = 1.0027939058931108e+00
│ norm(Q) - norm(Q₀) = 1.2343798102198988e-02
│ max mass loss      = 0.0000000000000000e+00
│ max mass gain      = 0.0000000000000000e+00
└ initial mass       = 2.9255250415427088e+01
Test Summary:                                                                                                                  | Pass  Total
Test Summary:                                                                                                                  | Pass  Test Summary:                                                                                                                  | Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/conservation/sphere.jl | Pass     1      1
Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/conservation/sphere.jl |    1      1
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/conservation/sphere.jl |    1      1
[ Info: (CUDA.CuArray, Float32)
┌ Info: Starting
│ norm(Q₀) = 4.4181141853332520e+00
└ sum(Q₀) = 2.9255237579345703e+01
┌ Info: Finished
│ norm(Q)            = 4.4304580688476563e+00
│ norm(Q) / norm(Q₀) = 1.0027939081192017e+00
│ norm(Q) - norm(Q₀) = 1.2343883514404297e-02
│ max mass loss      = 0.0000000000000000e+00
│ max mass gain      = 0.0000000000000000e+00
└ initial mass       = 2.9255237579345703e+01
Test Summary:                                                                                                                  | Pass  Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/conservation/sphere.jl |    1      1
Test Summary:                                                                                                                  | Pass  Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/conservation/sphere.jl |    1      1
Test Summary:                                                                                                                  | Pass  Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/conservation/sphere.jl |    1      1

out_10192399

+ hostname
hpc-26-15.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192399_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192399_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/DGMethods/advection_diffusion/advection_sphere.jl
[ Info: (CUDA.CuArray, Float64)
┌ Info: Starting
│ problem   = SolidBodyRotation
│ method    = LSRK144NiegemannDiehlBusch
│ time step = 7.1428571428571425e-02
└ norm(Q₀)  = 6.3810679373927837e-01
┌ Info: Finished
│ Δmass                   = 0.0000000000000000e+00
│ norm(Q)                 = 6.0927759884214927e-01
│ norm(Q) / norm(Q₀)      = 9.5482073662279754e-01
│ norm(Q) - norm(Q₀)      = -2.8829194897129096e-02
│ norm(Q - Qe)            = 1.3199024557832689e-01
└ norm(Q - Qe) / norm(Qe) = 2.0684663895344182e-01
[ Info: 
[ Info: (CUDA.CuArray, Float64)
┌ Info: Starting
│ problem   = SolidBodyRotation
│ method    = SSPRK33ShuOsher
│ time step = 1.5151515151515152e-02
└ norm(Q₀)  = 6.3810679373927837e-01
┌ Info: Finished
│ Δmass                   = 2.6639292720309465e-15
│ norm(Q)                 = 6.0758654980256122e-01
│ norm(Q) / norm(Q₀)      = 9.5217063313513739e-01
│ norm(Q) - norm(Q₀)      = -3.0520243936717151e-02
│ norm(Q - Qe)            = 1.1055145388897815e-01
└ norm(Q - Qe) / norm(Qe) = 1.7324914101157171e-01
[ Info: 
[ Info: (CUDA.CuArray, Float64)
┌ Info: Starting
│ problem   = SolidBodyRotation
│ method    = SSPRK34SpiteriRuuth
│ time step = 2.2727272727272728e-02
└ norm(Q₀)  = 6.3810679373927837e-01
┌ Info: Finished
│ Δmass                   = 2.3835156644487414e-15
│ norm(Q)                 = 5.9932132891912626e-01
│ norm(Q) / norm(Q₀)      = 9.3921790960276264e-01
│ norm(Q) - norm(Q₀)      = -3.8785464820152105e-02
│ norm(Q - Qe)            = 1.2641959922001456e-01
└ norm(Q - Qe) / norm(Qe) = 1.9811667962223242e-01
[ Info: 
[ Info: (CUDA.CuArray, Float64)
┌ Info: Starting
│ problem   = ReversingDeformationalFlow
│ method    = LSRK144NiegemannDiehlBusch
│ time step = 1.6129032258064516e-01
└ norm(Q₀)  = 7.5338057383963186e-01
┌ Info: Finished
│ Δmass                   = 1.8614343182663523e-15
│ norm(Q)                 = 5.1909887102432950e-01
│ norm(Q) / norm(Q₀)      = 6.8902608993317016e-01
│ norm(Q) - norm(Q₀)      = -2.3428170281530236e-01
│ norm(Q - Qe)            = 5.5387951598735385e-01
└ norm(Q - Qe) / norm(Qe) = 7.3519219265833535e-01
[ Info: 
[ Info: (CUDA.CuArray, Float64)
┌ Info: Starting
│ problem   = ReversingDeformationalFlow
│ method    = SSPRK33ShuOsher
│ time step = 3.3112582781456956e-02
└ norm(Q₀)  = 7.5338057383963186e-01
┌ Info: Finished
│ Δmass                   = 5.5843029547990565e-15
│ norm(Q)                 = 5.1716060651384754e-01
│ norm(Q) / norm(Q₀)      = 6.8645333377541107e-01
│ norm(Q) - norm(Q₀)      = -2.3621996732578432e-01
│ norm(Q - Qe)            = 5.5353962032596282e-01
└ norm(Q - Qe) / norm(Qe) = 7.3474103201895391e-01
[ Info: 
[ Info: (CUDA.CuArray, Float64)
┌ Info: Starting
│ problem   = ReversingDeformationalFlow
│ method    = SSPRK34SpiteriRuuth
│ time step = 4.9504950495049507e-02
└ norm(Q₀)  = 7.5338057383963186e-01
┌ Info: Finished
│ Δmass                   = 5.0258726593191508e-15
│ norm(Q)                 = 5.1534369870022134e-01
│ norm(Q) / norm(Q₀)      = 6.8404166047679349e-01
│ norm(Q) - norm(Q₀)      = -2.3803687513941052e-01
│ norm(Q - Qe)            = 5.5404045660832890e-01
└ norm(Q - Qe) / norm(Qe) = 7.3540581725467291e-01
[ Info: 
Test Summary:                                                                                                                                   | Test Summary:                                                                                                                                   | Pass  Pass  Total
Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/advection_sphere.jl | /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/DGMethods/advection_diffusion/advection_sphere.jl |   12    12     12
   12

out_10192400

+ hostname
hpc-26-18.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192400_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192400_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Driver/gcm_driver_test.jl
┌ Info: Model composition
│     param_set = EarthParameterSet()
│     orientation = SphericalOrientation()
│     ref_state = HydrostaticState{DecayingTemperatureProfile{Float64},Float64}(DecayingTemperatureProfile{Float64}(300.0, 300.0, 8776.832091407436), 0.0)
│     turbulence = ConstantViscosityWithDivergence{Float64}(0.0)
│     hyperdiffusion = NoHyperDiffusion()
│     moisture = DryModel()
│     precipitation = NoPrecipitation()
│     radiation = NoRadiation()
│     source = Gravity()
│     tracers = NoTracers()
│     boundarycondition = AtmosBC{Impenetrable{FreeSlip},Insulating,Impermeable,ImpermeableTracer}(Impenetrable{FreeSlip}(FreeSlip()), Insulating(), Impermeable(), ImpermeableTracer())
│     init_state_conservative = AcousticWaveSetup{Float64}(10000.0, 300.0, 3.0, 100.0, 1)
└     data_config = nothing
┌ Info: Establishing Atmos GCM configuration for GCM Driver test
│     precision        = Float64
│     polynomial order = 4
│     #horiz elems     = 4
│     #vert elems      = 6
│     domain height    = 1.00e+04 m
│     MPI ranks        = 1
│     min(Δ_horz)      = 305681.05 m
└     min(Δ_vert)      = 287.79 m
[ Info: Initializing GCM Driver test
┌ Info: Starting GCM Driver test
│     dt              = 6.00000e+02
│     timeend         =  1800.00
│     number of steps = 3
└     norm(Q)         = 9.5073564673383188e+13
┌ Info: Finished
│     norm(Q)            = 9.5073454833172516e+13
│     norm(Q) / norm(Q₀) = 9.9999884468189393e-01
└     norm(Q) - norm(Q₀) = -1.0984021067187500e+08

out_10192401

+ hostname
hpc-26-15.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192401_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192401_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/SystemSolvers/poisson.jl
[ Info: (CUDA.CuArray, Float64, 1, 2)
┌ Info: Finished
│ error = 5.0540243605694528e-02
└ iters = 49
[ Info: (CUDA.CuArray, Float64, 1, 3)
┌ Info: Finished
│ error = 1.4957957659447893e-02
└ iters = 39
[ Info: (CUDA.CuArray, Float64, 2, 2)
┌ Info: Finished
│ error = 5.0540243614350278e-02
└ iters = 38
[ Info: (CUDA.CuArray, Float64, 2, 3)
┌ Info: Finished
│ error = 1.4957957667511243e-02
└ iters = 35

out_10192402

+ hostname
hpc-89-38.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192402_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192402_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/SystemSolvers/columnwiselu.jl

out_10192403

+ hostname
hpc-26-17.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192403_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192403_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/SystemSolvers/bandedsystem.jl
Test Summary:                                                                                                                              | Pass  Total
Test Summary:                                                                                                                              | Test Summary:                                                                                                                              | Pass  Pass  /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/SystemSolvers/bandedsystem.jl DGModel matrix | TotalTotal

  72     72
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/SystemSolvers/bandedsystem.jl DGModel matrix | /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Numerics/SystemSolvers/bandedsystem.jl DGModel matrix |   72    72     72
   72

out_10192404

+ hostname
hpc-89-37.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192404_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192404_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Numerics/Mesh/interpolation.jl
Test Summary:       | Pass  Total
Test Summary:       | Test Summary:       | Pass  Pass  Total
Interpolation tests |    4  Total
    4
Interpolation tests |    4  Interpolation tests |     4
   4      4

out_10192405

+ hostname
hpc-89-38.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192405_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192405_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Ocean/ShallowWater/GyreDriver.jl
[ Info: running Ne 10 and N 4 with
┌ Info: 
└  dt = 63.102347
┌ Info: Finished
└ error = 1.6068814534512799e-03

out_10192406

+ hostname
hpc-26-15.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192406_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192406_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ unset CPATH
++ unset LD_LIBRARY_PATH
++ unset LOADEDMODULES
++ unset _LMFILES_
+ module load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
++ /software/Modules/bin/modulecmd sh load julia/1.4.2 cuda/10.0 openmpi/4.0.3_cuda-10.0 hdf5/1.10.1 netcdf-c/4.6.1
+ eval CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include ';export' 'CPATH;CUDA_HOME=/central/software/CUDA/10.0' ';export' 'CUDA_HOME;CUDA_VERSION=10.0' ';export' 'CUDA_VERSION;HDF5_DIR=/software/hdf5/1.10.1' ';export' 'HDF5_DIR;LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib' ';export' 'LD_LIBRARY_PATH;LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LD_RUN_PATH;LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64' ';export' 'LIBRARY_PATH;LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1' ';export' 'LOADEDMODULES;MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man' ';export' 'MANPATH;OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/' ';export' 'OPENMPI_HOME;OPENMPI_VERSION=4.0.3' ';export' 'OPENMPI_VERSION;PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;_LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1' ';export' '_LMFILES_;'
++ CPATH=/central/software/netcdf-c/4.6.1//include:/software/hdf5/1.10.1/include:/central/software/OpenMPI/4.0.3_cuda-10.0//include:/central/software/julia/1.4.2//include
++ export CPATH
++ CUDA_HOME=/central/software/CUDA/10.0
++ export CUDA_HOME
++ CUDA_VERSION=10.0
++ export CUDA_VERSION
++ HDF5_DIR=/software/hdf5/1.10.1
++ export HDF5_DIR
++ LD_LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64:/central/software/julia/1.4.2//lib
++ export LD_LIBRARY_PATH
++ LD_RUN_PATH=/central/software/netcdf-c/4.6.1//lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LD_RUN_PATH
++ LIBRARY_PATH=/central/software/netcdf-c/4.6.1//lib:/software/hdf5/1.10.1/lib:/central/software/OpenMPI/4.0.3_cuda-10.0//lib:/central/software/CUDA/10.0/lib64
++ export LIBRARY_PATH
++ LOADEDMODULES=julia/1.4.2:cuda/10.0:openmpi/4.0.3_cuda-10.0:hdf5/1.10.1:netcdf-c/4.6.1
++ export LOADEDMODULES
++ MANPATH=/central/software/netcdf-c/4.6.1//share/man:/software/hdf5/1.10.1/share/man:/central/software/OpenMPI/4.0.3_cuda-10.0//share/man:/central/software/CUDA/10.0/doc/man:/central/software/julia/1.4.2//share/man:/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man
++ export MANPATH
++ OPENMPI_HOME=/central/software/OpenMPI/4.0.3_cuda-10.0/
++ export OPENMPI_HOME
++ OPENMPI_VERSION=4.0.3
++ export OPENMPI_VERSION
++ PATH=/central/software/netcdf-c/4.6.1//bin:/software/hdf5/1.10.1/bin:/central/software/OpenMPI/4.0.3_cuda-10.0//bin:/central/software/CUDA/10.0/bin:/central/software/julia/1.4.2//bin:/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin
++ export PATH
++ _LMFILES_=/software/Modules/modulefiles/developement/julia/1.4.2:/software/Modules/modulefiles/libraries/cuda/10.0:/software/Modules/modulefiles/mpi/openmpi/4.0.3_cuda-10.0:/software/Modules/modulefiles/data/hdf5/1.10.1:/software/Modules/modulefiles/data/netcdf-c/4.6.1
++ export _LMFILES_
+ mpiexec julia --color=no --project test/Ocean/ShallowWater/2D_hydrostatic_spindown.jl
┌ Info: doing VTK output
└   outprefix = "vtk_shallow_spindown/fast/mpirank0000_step0000"
# SC Start: creating state check callback
# SC Creating state check callback labeled "2D state" for symbols
# SC η
# SC U
# SC Finish: creating state check callback
┌ Info: Starting
│ norm(Q₀) = 1.0000000000000008e+06
└ ArrayType = CUDA.CuArray
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000000|    2D state|        η | -9.570004283478e-01|  9.572452611952e-01| -1.989519660128e-16|  6.772836616366e-01|
# SC 0000000|    2D state|     U[1] | -1.792532800489e+01|  1.792532800489e+01|  2.819433575496e-15|  1.274390739934e+01|
# SC 0000000|    2D state|     U[2] | -4.857737804991e-13|  4.921405210505e-13|  8.749782554455e-15|  8.768407748740e-14|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000024|    2D state|        η | -8.244119230668e-01|  8.244339910020e-01| -2.273736754432e-17|  5.834075469111e-01|
# SC 0000024|    2D state|     U[1] | -3.512801331247e+01|  3.512801331247e+01| -8.003553375602e-15|  2.497982254591e+01|
# SC 0000024|    2D state|     U[2] | -7.196144996179e-13|  7.094554236801e-13| -1.093728586298e-14|  1.809087237818e-13|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000048|    2D state|        η | -6.144793034641e-01|  6.144946933273e-01| -3.296918293927e-16|  4.348410147654e-01|
# SC 0000048|    2D state|     U[1] | -4.901995498250e+01|  4.901995498250e+01|  1.309672370553e-14|  3.485484383011e+01|
# SC 0000048|    2D state|     U[2] | -1.025011127645e-12|  1.272012870611e-12|  2.132190873677e-14|  2.580982296554e-13|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000072|    2D state|        η | -3.472229442154e-01|  3.471049320739e-01| -9.663381206337e-17|  2.456489143135e-01|
# SC 0000072|    2D state|     U[1] | -5.827752340806e+01|  5.827752340806e+01| -1.764419721439e-14|  4.144038160451e+01|
# SC 0000072|    2D state|     U[2] | -1.794368116858e-12|  1.335260626555e-12| -3.025856471840e-14|  3.618240353169e-13|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000096|    2D state|        η | -4.763924420977e-02|  4.752491380495e-02| -1.065814103640e-16|  3.369680773001e-02|
# SC 0000096|    2D state|     U[1] | -6.205731222939e+01|  6.205731222939e+01|  2.128217602149e-14|  4.412697895910e+01|
# SC 0000096|    2D state|     U[2] | -1.611920578275e-12|  2.149574890313e-12|  2.686473302846e-14|  4.084146214073e-13|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000120|    2D state|        η | -2.560178592636e-01|  2.559630305845e-01| -8.952838470577e-17|  1.811621769315e-01|
# SC 0000120|    2D state|     U[1] | -6.000444094081e+01|  6.000444094081e+01| -1.855369191617e-14|  4.266661476095e+01|
# SC 0000120|    2D state|     U[2] | -2.243969800589e-12|  1.711949702521e-12| -2.668293592676e-14|  4.300567537813e-13|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000144|    2D state|        η | -5.351443527053e-01|  5.351153344761e-01| -5.968558980385e-17|  3.786780380728e-01|
# SC 0000144|    2D state|     U[1] | -5.231521203845e+01|  5.231521203845e+01|  1.982698449865e-14|  3.720030613681e+01|
# SC 0000144|    2D state|     U[2] | -1.622645129088e-12|  2.264555798432e-12|  2.814943581281e-14|  4.027824152123e-13|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000168|    2D state|        η | -7.635599856548e-01|  7.636623115743e-01| -2.842170943040e-16|  5.403552154437e-01|
# SC 0000168|    2D state|     U[1] | -3.973138903400e+01|  3.973138903400e+01| -9.822542779148e-15|  2.825071776819e+01|
# SC 0000168|    2D state|     U[2] | -1.987870739099e-12|  1.394432649644e-12| -1.651143732209e-14|  3.518085910816e-13|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000192|    2D state|        η | -9.200822132533e-01|  9.200384241042e-01| -1.136868377216e-17|  6.510808852866e-01|
# SC 0000192|    2D state|     U[1] | -2.342978002845e+01|  2.342978002845e+01|  1.045918907039e-14|  1.666038251575e+01|
# SC 0000192|    2D state|     U[2] | -1.115241347696e-12|  1.500422620096e-12|  1.195301397724e-14|  2.561261283700e-13|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000216|    2D state|        η | -9.898734969061e-01|  9.899710495454e-01| -2.557953848736e-16|  7.005041344611e-01|
# SC 0000216|    2D state|     U[1] | -4.953768384280e+00|  4.953768384280e+00|  1.296029950026e-15|  3.522268485124e+00|
# SC 0000216|    2D state|     U[2] | -9.362170117848e-13|  7.739809666306e-13| -6.602004208646e-15|  1.531085602784e-13|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000240|    2D state|        η | -9.667308999352e-01|  9.666829448740e-01|  8.526512829121e-17|  6.840902369194e-01|
# SC 0000240|    2D state|     U[1] | -1.396139302125e+01|  1.396139302125e+01|  6.366462912410e-16|  9.927994563815e+00|
# SC 0000240|    2D state|     U[2] | -4.385462785720e-13|  2.962033460984e-13| -6.900065614521e-15|  6.918344176260e-14|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
# SC +++++++++++ClimateMachine StateCheck call-back start+++++++++++++++++
# SC  Step  |   Label    |  Field   |                            Stats                       
# SC -------|------------|----------|              min() |              max() |             mean() |              std() |
# SC 0000264|    2D state|        η | -8.527228636775e-01|  8.528281863846e-01| -1.477928890381e-16|  6.034549743954e-01|
# SC 0000264|    2D state|     U[1] | -3.154026150718e+01|  3.154026150718e+01|  1.009539118968e-14|  2.242694051910e+01|
# SC 0000264|    2D state|     U[2] | -4.260780388789e-13|  5.551128204908e-13|  1.401883177828e-14|  1.356838320527e-13|
# SC +++++++++++ClimateMachine StateCheck call-back end+++++++++++++++++++
2D error = 0.00011327920483873622
# BEGIN SCPRINT
# varr - reference values (from reference run)    
# parr - digits match precision (hand edit as needed) 
#
# [
#  [ MPIStateArray Name, Field Name, Maximum, Minimum, Mean, Standard Deviation ],
#  [         :                :          :        :      :          :           ],
# ]
varr = [
 [ "2D state",     :η, -8.52722863677451425879e-01,  8.52828186384614772564e-01, -1.47792889038100831877e-16,  6.03454974395413956678e-01 ],
 [ "2D state", "U[1]", -3.15402615071803431590e+01,  3.15402615071801193380e+01,  1.00953911896795038877e-14,  2.24269405191037947134e+01 ],
 [ "2D state", "U[2]", -4.26078038878888084931e-13,  5.55112820490818215379e-13,  1.40188317782751228245e-14,  1.35683832052743391766e-13 ],
]
parr = [
 [ "2D state",     :η,    16,    16,    16,    16 ],
 [ "2D state", "U[1]",    16,    16,    16,    16 ],
 [ "2D state", "U[2]",    16,    16,    16,    16 ],
]
# END SCPRINT
# SC +++++++++++ClimateMachine StateCheck ref val check start+++++++++++++++++
# SC "N( )" bracketing indicates field failed to match      
# SC "P="  row pass count      
# SC "F="  row pass count      
# SC "NA=" row not checked count      
# SC 
# SC        Label         Field      min()      max()     mean()      std() 
# SC     2D state,            η,        15,        15,         0,        15 :: P=5, F=0, NA=1
# SC     2D state,         U[1],        15,        15,         0,        15 :: P=5, F=0, NA=1
# SC     2D state,         U[2],         0,         0,         0,         0 :: P=2, F=0, NA=4
# SC +++++++++++ClimateMachine StateCheck ref val check end+++++++++++++++++
Test Summary:                                                                                                                      | Pass  Total
/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/test/Ocean/ShallowWater/2D_hydrostatic_spindown.jl |    2      2

out_10192407

+ hostname
hpc-89-37.cm.cluster
+ cd /central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c
++ pwd
+ export JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ JULIA_DEPOT_PATH=/central/scratchio/esm/slurmci/sources/bac79e6b787f70f7de32d4ea15a23033e124176c/.slurmdepot_gpu
+ export JULIA_CUDA_USE_BINARYBUILDER=false
+ JULIA_CUDA_USE_BINARYBUILDER=false
+ export JULIA_MPI_BINARY=system
+ JULIA_MPI_BINARY=system
+ export OPENBLAS_NUM_THREADS=1
+ OPENBLAS_NUM_THREADS=1
+ export CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED=true
+ export CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192407_output
+ CLIMATEMACHINE_SETTINGS_OUTPUT_DIR=/central/scratchio/esm/slurmci/logs/bac79e6b787f70f7de32d4ea15a23033e124176c/10192407_output
+ module purge
++ /software/Modules/bin/modulecmd sh purge
+ eval MANPATH=/central/slurm/install/current/share/man/:/usr/share/man/overrides:/usr/share/man:/opt/ibutils/share/man:/usr/local/share/man:/cm/local/apps/environment-modules/current/share/man ';export' 'MANPATH;PATH=/central/slurm/install/current/bin/:/usr/lib64/qt-3.3/bin:/usr/bin:/bin:/usr/lpp/mmfs/bin:/opt/ibutils/bin:/opt/dell/srvadmin/bin' ';export' 'PATH;unset' 'CPATH;unset' 'LD_LIBRARY_PATH;unset' 'LOADEDMODULES;unset' '_LMFILES_;'
++ MANPATH=/central/slurm

out_10192408

out_10192409

out_10192410

out_10192411

out_10192412