The comptuational cost of ocean large eddy simulation on CPU and GPU

computational_costs_les.jl

using Oceananigans.Units
using GLMakie

# Wall time per time step in a large eddy simulation with 256^3 resolution.
# 
# Numbers provided by Qing Li (HKUST Guangzhou, qingli411.github.io)
#
# Details for the "LESMIP" contained in
# https://github.com/qingli411/A2022_LESMIP/blob/main/notebook/Compare_LES-BF5hWD05WV00.ipynb
# 
# "_a5000" refers to the Nvidia RTX A5000.
# "_128" refers to a simulation on 128 Intel Xeon Gold 6348 CPUs
oceananigans_a100 = 1.074hour / 16650
palm_a5000 = 13.37hour / 14238 # note this is not used in the plots below
ncarles_128 = 3.47hour / 28880

# Cost estimates for other computational configurations
ϵ_palm = 0.83 # estimated cost of PALM vs NCAR LES on CPU
ncarles_256 = ncarles_128 / 2
ncarles_512 = ncarles_256 / 2
palm_128 = ϵ_palm * ncarles_128
palm_256 = ϵ_palm * ncarles_256
palm_512 = ϵ_palm * ncarles_512
oceananigans_4a100 = oceananigans_a100 / 4

data = [ncarles_128,
        ncarles_256,
        ncarles_512,
        palm_128,
        palm_256,
        palm_512,
        oceananigans_a100,
        oceananigans_4a100]

colors = Makie.wong_colors()
x = [1, 1, 1, 2, 2, 2, 3, 3]
dodge = [1, 2, 3, 1, 2, 3, 1, 2]
color_id = [1, 2, 3, 1, 2, 3, 4, 5]
color = [colors[id] for id in color_id]

xticks = (1:3, ["NCAR LES", "PALM", "Oceananigans"])

fig = Figure(size=(800, 300))
ax = Axis(fig[1, 1]; xticks, ylabel = "Wall time per time-step (s)")
barplot!(ax, x, data; dodge, color)

labels = ["128 CPUs (2 nodes at 64 CPU/node)",
          "256 CPUs (4 nodes, est)",
          "512 CPUs (8 nodes, est)",
          "1 A100 GPU",
          "4 A100 GPUs (1 node at 4 GPU/node, est)"]

elements = [PolyElement(polycolor = colors[i]) for i in 1:length(labels)]
title = "Computational cost"
Legend(fig[1, 2], elements, labels, title)

Label(fig[0, 1:2], "Computational cost of large eddy simulation with 256³ resolution")

# Single node estimate
ncarles_64 = 2ncarles_128
relative_cost_1node = ncarles_64 / oceananigans_4a100 # ~147x

@info "Oceananigans is $relative_cost_1node times faster than NCAR LES"

save("cost_analysis.png", fig)

display(fig)

This script produces this plot: