Sixzero

using CUDA

function distributed_matmul(C, A::CuMatrix{Float32}, B::CuMatrix{Float32}, num_gpus::Int)
    m, n = size(A)
    n, k = size(B)
    
    # Ensure we have the correct number of GPUs
    @assert CUDA.ndevices() >= num_gpus "Not enough GPUs available"

    # Calculate rows per GPU

Sixzero

#%%
# Copyright 2020 DeepMind Technologies Limited. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software

Sixzero

using BenchmarkTools
using CUDA
BenchmarkTools.DEFAULT_PARAMETERS.seconds = 1.00


test_sum(a, b, c) = begin
  I = (blockIdx().x - 1) * blockDim().x + threadIdx().x
  if I > 1000
    return
  end

Sixzero

X, Y = (Float32[-0.31125240132442067; 0.8163067649323273;;;], 
        Float32[5.7064323; 2.599511;;;])

# w = randn(1,1,1) .* ones(2,1,1) 
b = Float32[25.510088, ]
# b = randn(1,1,1) .* ones(2,1,1) 
w = Float32[0.15980364, ]
modl(X,w,b) = begin

Sixzero

a = randn(1024)
b = randn(1024)
c = randn(1024)

@time c .= a .+ b
@time c .= a .+ b
;

Sixzero

#%%
using BenchmarkTools
BenchmarkTools.DEFAULT_PARAMETERS.seconds = 2.50
using Base: @opaque

add_opaq = @opaque (a::Vector, b::Vector) -> a .+ b
@noinline add(a::Vector, b::Vector) = a .+ b
@noinline add!(c::Vector, a::Vector, b::Vector) = c .= a .+ b
@inline add_inline(a::Vector, b::Vector) = a .+ b
#%%