Skip to content

Commit

Permalink
add script, which set QED dependencies of QED dependencies to dev dep…
Browse files Browse the repository at this point in the history
…ending on the target branch

- if the target branch is main, use released QED dependencies
- if the target branch is not main, use the dev branch versions of the QED dependencies
  • Loading branch information
SimeonEhrig committed Aug 19, 2024
1 parent 7588afe commit 8ecbf6a
Show file tree
Hide file tree
Showing 5 changed files with 207 additions and 3 deletions.
2 changes: 2 additions & 0 deletions .ci/integTestGen/Project.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ authors = ["Simeon Ehrig <s.ehrig@hzdr.de>"]
version = "0.1.0"

[deps]
HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
PkgDependency = "9eb5382b-762c-48ca-8139-e736883fe800"
YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6"
Expand Down
2 changes: 2 additions & 0 deletions .ci/integTestGen/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,5 @@ You can set the environment variables in two different ways:
## Optional Environment Variables

By default, if an integration test is generated it clones the develop branch of the upstream project. The clone can be overwritten by the environment variable `CI_INTG_PKG_URL_<dep_name>=https://url/to/the/repository#<commit_hash>`. You can find all available environment variables in the dictionary `package_infos` in the `integTestGen.jl`.

Set the environment variable `CI_COMMIT_REF_NAME` to determine the target branch of a GitHub pull request. The form must be `CI_COMMIT_REF_NAME=pr-<PR number>/<repo owner of source branch>/<project name>/<source branch name>`. Here is an example: `CI_COMMIT_REF_NAME=pr-41/SimeonEhrig/QED.jl/setDevDepDeps`. If the environment variable is not set, the default target branch `dev` is used.
71 changes: 71 additions & 0 deletions .ci/integTestGen/src/get_target_branch.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
module getTargetBranch

using HTTP
using JSON

"""
get_target_branch()::AbstractString
Returns the name of the target branch of the pull request. The function is required for our special
setup where we mirror a PR from GitHub to GitLab CI. No merge request will be open on GitLab.
Instead, a feature branch will be created and the commit will be pushed. As a result, we lose
information about the original PR. So we need to use the GitHub Rest API to get the information
depending on the repository name and PR number.
"""
function get_target_branch()::AbstractString
# GitLab CI provides the environemnt variable with the following pattern
# # pr-<PR number>/<repo owner of the source branch>/<project name>/<source branch name>
# e.g. pr-41/SimeonEhrig/QED.jl/setDevDepDeps
if !haskey(ENV, "CI_COMMIT_REF_NAME")
error("Environment variable CI_COMMIT_REF_NAME is not set.")
end

splited_commit_ref_name = split(ENV["CI_COMMIT_REF_NAME"], "/")

if (!startswith(splited_commit_ref_name[1], "pr-"))
error("CI_COMMIT_REF_NAME does not start with pr-")
end

# parse to Int only to check if it is a number
pr_number = parse(Int, splited_commit_ref_name[1][(length("pr-") + 1):end])
if (pr_number <= 0)
error(
"a PR number always needs to be a positiv integer number bigger than 0: $pr_number",
)
end

repository_name = splited_commit_ref_name[3]

try
headers = (
("Accept", "application/vnd.github+json"),
("X-GitHub-Api-Version", "2022-11-28"),
)
# in all cases, we assume that the PR targets the repositories in QEDjl-project
# there is no environment variable with the information, if the target repository is
# the upstream repository or a fork.
url = "https://api.github.com/repos/QEDjl-project/$repository_name/pulls/$pr_number"
response = HTTP.get(url, headers)
response_text = String(response.body)
repository_data = JSON.parse(response_text)
return repository_data["base"]["ref"]
catch e
# if for unknown reason, the PR does not exist, use fallback the dev branch
if isa(e, HTTP.Exceptions.StatusError) && e.status == 404
return "dev"
else
# Only the HTML code 404, page does not exist is handled. All other error will abort
# the script.
throw(e)
end
end

return "dev"
end

if abspath(PROGRAM_FILE) == @__FILE__
target_branch = get_target_branch()
println(target_branch)
end

end
33 changes: 30 additions & 3 deletions .ci/integTestGen/src/integTestGen.jl
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
module integTestGen

include("get_target_branch.jl")

using Pkg: Pkg
using PkgDependency: PkgDependency
using YAML: YAML
using Logging

"""
Contains all git-related information about a package.
Expand Down Expand Up @@ -207,10 +210,14 @@ Generate GitLab CI job yaml for integration testing of a given package.
# Args
- `package_name::String`: Name of the package to test.
- `target_branch::AbstractString`: Name of the target branch of the pull request.
- `job_yaml::Dict`: Add generated job to this dict.
"""
function generate_job_yaml!(
package_name::String, job_yaml::Dict, package_infos::AbstractDict{String,PackageInfo}
package_name::String,
target_branch::AbstractString,
job_yaml::Dict,
package_infos::AbstractDict{String,PackageInfo},
)
package_info = package_infos[package_name]
# if modified_url is empty, use original url
Expand All @@ -227,9 +234,17 @@ function generate_job_yaml!(
error("Ill formed url: $(url)")
end

push!(script, "git clone $(split_url[1]) integration_test")
push!(script, "git clone -b $target_branch $(split_url[1]) integration_test")
if (target_branch != "main")
push!(
script,
"git clone -b dev https://github.com/QEDjl-project/QED.jl.git /integration_test_tools",
)
end
push!(script, "cd integration_test")

# checkout specfic branch given by the environemnt variable
# CI_INTG_PKG_URL_<dep_name>=https://url/to/the/repository#<commit_hash>
if length(split_url) == 2
push!(script, "git checkout $(split_url[2])")
end
Expand All @@ -246,6 +261,11 @@ function generate_job_yaml!(
push!(
script, "julia --project=. -e 'import Pkg; Pkg.develop(path=\"$ci_project_dir\");'"
)
if (target_branch != "main")
push!(
script, "julia --project=. /integration_test_tools/.ci/set_dev_dependencies.jl"
)
end
push!(script, "julia --project=. -e 'import Pkg; Pkg.test(; coverage = true)'")

return job_yaml["IntegrationTest$package_name"] = Dict(
Expand Down Expand Up @@ -273,6 +293,13 @@ function generate_dummy_job_yaml!(job_yaml::Dict)
end

if abspath(PROGRAM_FILE) == @__FILE__
if !haskey(ENV, "CI_COMMIT_REF_NAME")
@warn "Environemnt variable CI_COMMIT_REF_NAME not defined. Use default branch `dev`."
target_branch = "dev"
else
target_branch = getTargetBranch.get_target_branch()
end

package_infos = Dict(
"QED" => PackageInfo(
"https://github.com/QEDjl-project/QED.jl.git", "CI_INTG_PKG_URL_QED"
Expand Down Expand Up @@ -316,7 +343,7 @@ if abspath(PROGRAM_FILE) == @__FILE__

if !isempty(depending_pkg)
for p in depending_pkg
generate_job_yaml!(p, job_yaml, package_infos)
generate_job_yaml!(p, target_branch, job_yaml, package_infos)
end
else
generate_dummy_job_yaml!(job_yaml)
Expand Down
102 changes: 102 additions & 0 deletions .ci/set_dev_dependencies.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
"""
The script sets all QED dependencies of QED dependencies to the version of the current
development branch. For our example we use the project QEDprocess which has a dependency
to QEDfields and QEDfields has a dependency to QEDcore (I haven't checked if this is the
case, it's just hypothetical). If we install the dev-branch version of QEDfields, the last
registered version of QEDcore is still installed. If QEDfields uses a function which only
exist in dev branch of QEDcore and is not released yet, the integration test will fail.
The script needs to be executed the project space, which should be modified.
"""

using Pkg

# TODO(SimeonEhrig): is copied from integTestGen.jl
"""
_match_package_filter(
package_filter::Union{<:AbstractString,Regex},
package::AbstractString
)::Bool
Check if `package_filter` contains `package`. Wrapper function for `contains()` and `in()`.
# Returns
- `true` if it matches.
"""
function _match_package_filter(
package_filter::Union{<:AbstractString,Regex}, package::AbstractString
)::Bool
return contains(package, package_filter)
end

"""
_match_package_filter(
package_filter::AbstractVector{<:AbstractString},
package::AbstractString
)::Bool
"""
function _match_package_filter(
package_filter::AbstractVector{<:AbstractString}, package::AbstractString
)::Bool
return package in package_filter
end

"""
get_filtered_dependencies(
name_filter::Union{<:AbstractString,Regex}=r".*",
project_source=Pkg.dependencies()
)::AbstractVector{Pkg.API.PackageInfo}
Takes the project_dependencies and filter it by the name_filter. Removes also the UUID as
dict key.
# Returns
- `Vector` of filtered dependencies.
"""
function get_filtered_dependencies(
name_filter::Union{<:AbstractString,Regex}=r".*",
project_dependencies=Pkg.dependencies(),
)::AbstractVector{Pkg.API.PackageInfo}
deps = Vector{Pkg.API.PackageInfo}(undef, 0)
for (uuid, dep) in project_dependencies
if _match_package_filter(name_filter, dep.name)
push!(deps, dep)
end
end
return deps
end

"""
set_dev_dependencies(
dependencies::AbstractVector{Pkg.API.PackageInfo},
custom_urls::AbstractDict{String,String}=Dict{String,String}(),
)
Set all dependencies to the development version, if they are not already development versions.
The dict custom_urls takes as key a dependency name and a URL as value. If a dependency is in
custom_urls, it will use the URL as development version. If the dependency does not exist in
custom_urls, it will set the URL https://github.com/QEDjl-project/<dependency_name>.jl
"""
function set_dev_dependencies(
dependencies::AbstractVector{Pkg.API.PackageInfo},
custom_urls::AbstractDict{String,String}=Dict{String,String}(),
)
for dep in dependencies
# if tree_hash is nothing, it is already a dev version
if !isnothing(dep.tree_hash)
if haskey(custom_urls, dep.name)
Pkg.develop(; url=custom_urls[dep.name])
else
Pkg.develop(; url="https://github.com/QEDjl-project/$(dep.name).jl")
end
end
end
end

if abspath(PROGRAM_FILE) == @__FILE__
deps = get_filtered_dependencies(r"^QED*")
set_dev_dependencies(deps)
end

0 comments on commit 8ecbf6a

Please sign in to comment.