Merge pull request #411 from JuliaHealth/buildkite

Add Buildkite GPU pipeline

.buildkite/pipeline.yml

@@ -1,5 +1,122 @@
 steps:
-  - label: "Example Test"
-    command: echo "Hello!"
+  - label: "AMDGPU: Run tests on v{{matrix.version}}"
+    matrix:
+      setup:
+        version:
+          - "1.10"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "{{matrix.version}}"
+      - JuliaCI/julia-coverage#v1:
+          codecov: true
+          dirs:
+            - KomaMRICore/src
+            - KomaMRICore/ext
+    command: |
+      julia -e 'println("--- :julia: Instantiating project")
+          using Pkg
+          Pkg.develop([
+              PackageSpec(path=pwd(), subdir="."),
+              PackageSpec(path=pwd(), subdir="KomaMRIBase"),
+              PackageSpec(path=pwd(), subdir="KomaMRICore"),
+          ])'
+      
+      julia -e 'println("--- :julia: Running tests")
+          using Pkg
+          Pkg.test("KomaMRICore"; coverage=true, test_args=["AMDGPU"])'
     agents:
       queue: "juliagpu"
+      rocm: "*"
+    timeout_in_minutes: 120
+
+  - label: "CUDA: Run tests on v{{matrix.version}}"
+    matrix:
+      setup:
+        version:
+          - "1.9"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "{{matrix.version}}"
+      - JuliaCI/julia-coverage#v1:
+          codecov: true
+          dirs:
+            - KomaMRICore/src
+            - KomaMRICore/ext
+    command: |
+      julia -e 'println("--- :julia: Instantiating project")
+          using Pkg
+          Pkg.develop([
+              PackageSpec(path=pwd(), subdir="."),
+              PackageSpec(path=pwd(), subdir="KomaMRIBase"),
+              PackageSpec(path=pwd(), subdir="KomaMRICore"),
+          ])'
+      
+      julia -e 'println("--- :julia: Running tests")
+          using Pkg
+          Pkg.test("KomaMRICore"; coverage=true, test_args=["CUDA"])'
+    agents:
+      queue: "juliagpu"
+      cuda: "*"
+    timeout_in_minutes: 120
+
+  - label: "Metal: Run tests on v{{matrix.version}}"
+    matrix:
+      setup:
+        version:
+          - "1.9"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "{{matrix.version}}"
+    command: |
+      julia -e 'println("--- :julia: Instantiating project")
+          using Pkg
+          Pkg.develop([
+              PackageSpec(path=pwd(), subdir="."),
+              PackageSpec(path=pwd(), subdir="KomaMRIBase"),
+              PackageSpec(path=pwd(), subdir="KomaMRICore"),
+          ])'
+      
+      julia -e 'println("--- :julia: Running tests")
+          using Pkg
+          Pkg.test("KomaMRICore"; test_args=["Metal"])'
+    agents:
+      queue: "juliaecosystem"
+      os: "macos"
+      arch: "aarch64"
+    timeout_in_minutes: 120
+
+  - label: "oneAPI: Run tests on v{{matrix.version}}"
+    matrix:
+      setup:
+        version:
+          - "1.9"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "{{matrix.version}}"
+      - JuliaCI/julia-coverage#v1:
+          codecov: true
+          dirs:
+            - KomaMRICore/src
+            - KomaMRICore/ext
+    command: |
+      julia -e 'println("--- :julia: Instantiating project")
+          using Pkg
+          Pkg.develop([
+              PackageSpec(path=pwd(), subdir="."),
+              PackageSpec(path=pwd(), subdir="KomaMRIBase"),
+              PackageSpec(path=pwd(), subdir="KomaMRICore"),
+          ])'
+      
+      julia -e 'println("--- :julia: Running tests")
+          using Pkg
+          Pkg.test("KomaMRICore"; coverage=true, test_args=["oneAPI"])'
+    agents:
+      queue: "juliagpu"
+      intel: "*"
+    timeout_in_minutes: 120
+
+env:
+  CI: BUILDKITE
+  CODECOV_FLAGS: core
+  JULIA_PKG_SERVER: ""
+  SECRET_CODECOV_TOKEN: "lUmUVYkTlE8u0mR/ymv5rtE1A59wXZmQ3miRkmKciC/4+xHPNjpeIJ03FupuC1ElGBeX0m6DDFavZ9burLosGxbBYIPziBQZ5P9NdPDZjBdo7NM3QSSBeUfDsDYbHsYglfJZ35UL6Pd2YTAkJG0ePrTpfUaBb9rcll926NdqUP0vE2hbR2leKFFgBVNtK9Zf+NE7hO3meZQEZ+sN5tA7xGr24A3Ay7ckPg5HbPPD3KII2/fLtW+w0fQUfJdXNFrajJ0FyNE0kNLDlIzoYN6XM4yUeLBXoyXFHnqT5dSu7pqrByEk/ptYpjUzKFoRSnDQy+p8vrx9e/iAVf3lwwgwWA==;U2FsdGVkX1+IZLGTj2FNEy4XvUQbzkZFEwUpwIdRDCKrEea5O/OKGV5vYJufty3m9yniE+av4937HabS9dO1RA=="

KomaMRICore/ext/KomaAMDGPUExt.jl

@@ -8,7 +8,7 @@ KomaMRICore.name(::ROCBackend) = "AMDGPU"
 KomaMRICore.isfunctional(::ROCBackend) = AMDGPU.functional()
 KomaMRICore.set_device!(::ROCBackend, dev_idx::Integer) = AMDGPU.device_id!(dev_idx)
 KomaMRICore.set_device!(::ROCBackend, dev::AMDGPU.HIPDevice) = AMDGPU.device!(dev)
-KomaMRICore.device_name(::ROCBackend) = AMDGPU.device().name
+KomaMRICore.device_name(::ROCBackend) = AMDGPU.HIP.name(AMDGPU.device())
 
 function Adapt.adapt_storage(
     ::ROCBackend, x::Vector{KomaMRICore.LinearInterpolator{T,V}}

KomaMRICore/ext/KomaMetalExt.jl

@@ -23,13 +23,13 @@ end
 #Temporary workaround for https://github.com/JuliaGPU/Metal.jl/issues/348
 #Once run_spin_excitation! and run_spin_precession! are kernel-based, this code
 #can be removed
-Base.cumsum(x::MtlVector) = convert(MtlVector, cumsum(KomaMRICore.cpu(x)))
+Base.cumsum(x::MtlVector{T}) where T = convert(MtlVector{T}, cumsum(KomaMRICore.cpu(x)))
 Base.cumsum(x::MtlArray{T}; dims) where T = convert(MtlArray{T}, cumsum(KomaMRICore.cpu(x), dims=dims))
 Base.findall(x::MtlVector{Bool}) = convert(MtlVector, findall(KomaMRICore.cpu(x)))
 
 function __init__()
     push!(KomaMRICore.LOADED_BACKENDS[], MetalBackend())
-    @warn "Due to https://github.com/JuliaGPU/Metal.jl/issues/348, some functions may need to run on the CPU. Performance may be impacted as a result."
+    @warn "Metal does not support all array operations used by KomaMRI (https://github.com/JuliaGPU/Metal.jl/issues/348). GPU performance may be slower than expected"
 end
 
 end

KomaMRICore/ext/KomaoneAPIExt.jl

@@ -23,8 +23,15 @@ function KomaMRICore._print_devices(::oneAPIBackend)
     @info "$(length(oneAPI.devices())) oneAPI capable device(s)." devices...
 end
 
+#Temporary workaround since oneAPI.jl (similar to Metal) does not support some array operations
+#Once run_spin_excitation! and run_spin_precession! are kernel-based, this code can be removed
+Base.cumsum(x::oneVector{T}) where T = convert(oneVector{T}, cumsum(KomaMRICore.cpu(x)))
+Base.cumsum(x::oneArray{T}; dims) where T = convert(oneArray{T}, cumsum(KomaMRICore.cpu(x), dims=dims))
+Base.findall(x::oneVector{Bool}) = convert(oneVector, findall(KomaMRICore.cpu(x)))
+
 function __init__()
     push!(KomaMRICore.LOADED_BACKENDS[], oneAPIBackend())
+    @warn "oneAPI does not support all array operations used by KomaMRI. GPU performance may be slower than expected"
 end
 
 end

KomaMRICore/src/KomaMRICore.jl

@@ -17,8 +17,8 @@ include("rawdata/ISMRMRD.jl")
 include("datatypes/Spinor.jl")
 include("other/DiffusionModel.jl")
 # Simulator
-include("simulation/Functors.jl")
 include("simulation/GPUFunctions.jl")
+include("simulation/Functors.jl")
 include("simulation/SimulatorCore.jl")
 
 # ISMRMRD

KomaMRICore/src/simulation/Bloch/BlochDictSimulationMethod.jl

@@ -50,7 +50,7 @@ function run_spin_precession!(
     #Mxy precession and relaxation, and Mz relaxation
     tp = cumsum(seq.Δt) # t' = t - t0
     dur = sum(seq.Δt)   # Total length, used for signal relaxation
-    Mxy = [M.xy M.xy .* exp.(1im .* ϕ .- tp' ./ p.T2)] #This assumes Δw and T2 are constant in time
+    Mxy = [M.xy M.xy .* exp.(-tp' ./ p.T2) .* (cos.(ϕ) .+ im * sin.(ϕ))] #This assumes Δw and T2 are constant in time
     M.xy .= Mxy[:, end]
     #Acquired signal
     sig[:, :, 1] .= transpose(Mxy[:, findall(seq.ADC)])

KomaMRICore/src/simulation/Bloch/BlochSimulationMethod.jl

@@ -59,7 +59,7 @@ function run_spin_precession!(
     #Mxy precession and relaxation, and Mz relaxation
     tp   = cumsum(seq.Δt) # t' = t - t0
     dur  = sum(seq.Δt)   # Total length, used for signal relaxation
-    Mxy  = [M.xy M.xy .* exp.(1im .* ϕ .- tp' ./ p.T2)] #This assumes Δw and T2 are constant in time
+    Mxy = [M.xy M.xy .* exp.(-tp' ./ p.T2) .* (cos.(ϕ) .+ im * sin.(ϕ))] #This assumes Δw and T2 are constant in time
     M.xy .= Mxy[:, end]
     M.z  .= M.z .* exp.(-dur ./ p.T1) .+ p.ρ .* (1 .- exp.(-dur ./ p.T1))
     #Acquired signal

KomaMRICore/src/simulation/Functors.jl

@@ -7,6 +7,34 @@ _isleaf(::AbstractArray{<:Number}) = true
 _isleaf(::AbstractArray{T}) where T = isbitstype(T)
 _isleaf(::AbstractRange) = true
 
+"""
+    gpu(x)
+
+Moves 'x' to the GPU. For this function to work, a GPU backend will need to be
+loaded with 'using AMDGPU / CUDA / Metal / oneAPI.
+
+This works for functions, and any struct marked with `@functor`.
+
+Use [`cpu`](@ref) to copy back to ordinary `Array`s.
+
+See also [`f32`](@ref) and [`f64`](@ref) to change element type only.
+
+# Examples
+```julia
+using CUDA
+x = gpu(x)
+```
+"""
+function gpu(x)
+    get_backend(true)
+
+    if (BACKEND[] isa KA.GPU)
+        return gpu(x, BACKEND[])
+    else
+        @error "function 'gpu' called with no functional backends available. Add 'using CUDA / Metal / AMDGPU / oneAPI' to your code and try again"
+    end
+end
+
 """
 	gpu(x, backend)
 

KomaMRICore/test/Project.toml

@@ -1,11 +1,8 @@
 [deps]
-AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e"
-CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
 HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f"
 KomaMRIBase = "d0bc0b20-b151-4d03-b2a4-6ca51751cb9c"
-Metal = "dde4c033-4e86-420c-a63e-0dd931031962"
-oneAPI = "8f75cd03-7ff8-4ecb-9b8f-daf728133b1b"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a"
-TestItems = "1c621080-faea-4a02-84b6-bbd5e436b8fe"
+TestItems = "1c621080-faea-4a02-84b6-bbd5e436b8fe"

KomaMRICore/test/initialize.jl

@@ -0,0 +1,29 @@
+using Pkg
+using Suppressor
+
+const USE_GPU = if "AMDGPU" in ARGS
+    @suppress Pkg.add("AMDGPU")
+    using AMDGPU
+    @info "Testing AMD" maxlog=1
+    true
+elseif "CUDA" in ARGS
+    @suppress Pkg.add("CUDA")
+    using CUDA
+    @info "Testing CUDA" maxlog=1
+    true
+elseif "Metal" in ARGS
+    @suppress Pkg.add("Metal")
+    using Metal
+    @info "Testing Metal" maxlog=1
+    true
+elseif "oneAPI" in ARGS
+    @suppress Pkg.add("oneAPI")
+    using oneAPI
+    @info "Testing oneAPI" maxlog=1
+    true
+else
+    @info "Testing on the CPU with $(Threads.nthreads()) thread(s)" maxlog=1
+    false
+end
+
+using KomaMRICore