diff --git a/.ci/integTestGen/Project.toml b/.ci/integTestGen/Project.toml index 5bd11dd..ffae16d 100644 --- a/.ci/integTestGen/Project.toml +++ b/.ci/integTestGen/Project.toml @@ -4,6 +4,8 @@ authors = ["Simeon Ehrig "] version = "0.1.0" [deps] +HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" PkgDependency = "9eb5382b-762c-48ca-8139-e736883fe800" YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6" diff --git a/.ci/integTestGen/README.md b/.ci/integTestGen/README.md index 022556e..ee335f6 100644 --- a/.ci/integTestGen/README.md +++ b/.ci/integTestGen/README.md @@ -15,3 +15,5 @@ You can set the environment variables in two different ways: ## Optional Environment Variables By default, if an integration test is generated it clones the develop branch of the upstream project. The clone can be overwritten by the environment variable `CI_INTG_PKG_URL_=https://url/to/the/repository#`. You can find all available environment variables in the dictionary `package_infos` in the `integTestGen.jl`. + +Set the environment variable `CI_COMMIT_REF_NAME` to determine the target branch of a GitHub pull request. The form must be `CI_COMMIT_REF_NAME=pr-///`. Here is an example: `CI_COMMIT_REF_NAME=pr-41/SimeonEhrig/QED.jl/setDevDepDeps`. If the environment variable is not set, the default target branch `dev` is used. diff --git a/.ci/integTestGen/src/get_target_branch.jl b/.ci/integTestGen/src/get_target_branch.jl new file mode 100644 index 0000000..c125a29 --- /dev/null +++ b/.ci/integTestGen/src/get_target_branch.jl @@ -0,0 +1,71 @@ +module getTargetBranch + +using HTTP +using JSON + +""" + get_target_branch()::AbstractString + +Returns the name of the target branch of the pull request. The function is required for our special +setup where we mirror a PR from GitHub to GitLab CI. No merge request will be open on GitLab. +Instead, a feature branch will be created and the commit will be pushed. As a result, we lose +information about the original PR. So we need to use the GitHub Rest API to get the information +depending on the repository name and PR number. +""" +function get_target_branch()::AbstractString + # GitLab CI provides the environemnt variable with the following pattern + # # pr-/// + # e.g. pr-41/SimeonEhrig/QED.jl/setDevDepDeps + if !haskey(ENV, "CI_COMMIT_REF_NAME") + error("Environment variable CI_COMMIT_REF_NAME is not set.") + end + + splited_commit_ref_name = split(ENV["CI_COMMIT_REF_NAME"], "/") + + if (!startswith(splited_commit_ref_name[1], "pr-")) + error("CI_COMMIT_REF_NAME does not start with pr-") + end + + # parse to Int only to check if it is a number + pr_number = parse(Int, splited_commit_ref_name[1][(length("pr-") + 1):end]) + if (pr_number <= 0) + error( + "a PR number always needs to be a positiv integer number bigger than 0: $pr_number", + ) + end + + repository_name = splited_commit_ref_name[3] + + try + headers = ( + ("Accept", "application/vnd.github+json"), + ("X-GitHub-Api-Version", "2022-11-28"), + ) + # in all cases, we assume that the PR targets the repositories in QEDjl-project + # there is no environment variable with the information, if the target repository is + # the upstream repository or a fork. + url = "https://api.github.com/repos/QEDjl-project/$repository_name/pulls/$pr_number" + response = HTTP.get(url, headers) + response_text = String(response.body) + repository_data = JSON.parse(response_text) + return repository_data["base"]["ref"] + catch e + # if for unknown reason, the PR does not exist, use fallback the dev branch + if isa(e, HTTP.Exceptions.StatusError) && e.status == 404 + return "dev" + else + # Only the HTML code 404, page does not exist is handled. All other error will abort + # the script. + throw(e) + end + end + + return "dev" +end + +if abspath(PROGRAM_FILE) == @__FILE__ + target_branch = get_target_branch() + println(target_branch) +end + +end diff --git a/.ci/integTestGen/src/integTestGen.jl b/.ci/integTestGen/src/integTestGen.jl index a15c731..eaeb6bc 100644 --- a/.ci/integTestGen/src/integTestGen.jl +++ b/.ci/integTestGen/src/integTestGen.jl @@ -1,8 +1,11 @@ module integTestGen +include("get_target_branch.jl") + using Pkg: Pkg using PkgDependency: PkgDependency using YAML: YAML +using Logging """ Contains all git-related information about a package. @@ -207,10 +210,14 @@ Generate GitLab CI job yaml for integration testing of a given package. # Args - `package_name::String`: Name of the package to test. +- `target_branch::AbstractString`: Name of the target branch of the pull request. - `job_yaml::Dict`: Add generated job to this dict. """ function generate_job_yaml!( - package_name::String, job_yaml::Dict, package_infos::AbstractDict{String,PackageInfo} + package_name::String, + target_branch::AbstractString, + job_yaml::Dict, + package_infos::AbstractDict{String,PackageInfo}, ) package_info = package_infos[package_name] # if modified_url is empty, use original url @@ -227,9 +234,17 @@ function generate_job_yaml!( error("Ill formed url: $(url)") end - push!(script, "git clone $(split_url[1]) integration_test") + push!(script, "git clone -b $target_branch $(split_url[1]) integration_test") + if (target_branch != "main") + push!( + script, + "git clone -b dev https://github.com/QEDjl-project/QED.jl.git /integration_test_tools", + ) + end push!(script, "cd integration_test") + # checkout specfic branch given by the environemnt variable + # CI_INTG_PKG_URL_=https://url/to/the/repository# if length(split_url) == 2 push!(script, "git checkout $(split_url[2])") end @@ -246,6 +261,11 @@ function generate_job_yaml!( push!( script, "julia --project=. -e 'import Pkg; Pkg.develop(path=\"$ci_project_dir\");'" ) + if (target_branch != "main") + push!( + script, "julia --project=. /integration_test_tools/.ci/set_dev_dependencies.jl" + ) + end push!(script, "julia --project=. -e 'import Pkg; Pkg.test(; coverage = true)'") return job_yaml["IntegrationTest$package_name"] = Dict( @@ -273,6 +293,13 @@ function generate_dummy_job_yaml!(job_yaml::Dict) end if abspath(PROGRAM_FILE) == @__FILE__ + if !haskey(ENV, "CI_COMMIT_REF_NAME") + @warn "Environemnt variable CI_COMMIT_REF_NAME not defined. Use default branch `dev`." + target_branch = "dev" + else + target_branch = getTargetBranch.get_target_branch() + end + package_infos = Dict( "QED" => PackageInfo( "https://github.com/QEDjl-project/QED.jl.git", "CI_INTG_PKG_URL_QED" @@ -316,7 +343,7 @@ if abspath(PROGRAM_FILE) == @__FILE__ if !isempty(depending_pkg) for p in depending_pkg - generate_job_yaml!(p, job_yaml, package_infos) + generate_job_yaml!(p, target_branch, job_yaml, package_infos) end else generate_dummy_job_yaml!(job_yaml) diff --git a/.ci/set_dev_dependencies.jl b/.ci/set_dev_dependencies.jl new file mode 100644 index 0000000..33447dd --- /dev/null +++ b/.ci/set_dev_dependencies.jl @@ -0,0 +1,102 @@ +""" +The script sets all QED dependencies of QED dependencies to the version of the current +development branch. For our example we use the project QEDprocess which has a dependency +to QEDfields and QEDfields has a dependency to QEDcore (I haven't checked if this is the +case, it's just hypothetical). If we install the dev-branch version of QEDfields, the last +registered version of QEDcore is still installed. If QEDfields uses a function which only +exist in dev branch of QEDcore and is not released yet, the integration test will fail. + +The script needs to be executed the project space, which should be modified. +""" + +using Pkg + +# TODO(SimeonEhrig): is copied from integTestGen.jl +""" + _match_package_filter( + package_filter::Union{<:AbstractString,Regex}, + package::AbstractString + )::Bool + +Check if `package_filter` contains `package`. Wrapper function for `contains()` and `in()`. + +# Returns + +- `true` if it matches. + +""" +function _match_package_filter( + package_filter::Union{<:AbstractString,Regex}, package::AbstractString +)::Bool + return contains(package, package_filter) +end + +""" + _match_package_filter( + package_filter::AbstractVector{<:AbstractString}, + package::AbstractString + )::Bool +""" +function _match_package_filter( + package_filter::AbstractVector{<:AbstractString}, package::AbstractString +)::Bool + return package in package_filter +end + +""" + get_filtered_dependencies( + name_filter::Union{<:AbstractString,Regex}=r".*", + project_source=Pkg.dependencies() + )::AbstractVector{Pkg.API.PackageInfo} + +Takes the project_dependencies and filter it by the name_filter. Removes also the UUID as +dict key. + +# Returns + +- `Vector` of filtered dependencies. +""" +function get_filtered_dependencies( + name_filter::Union{<:AbstractString,Regex}=r".*", + project_dependencies=Pkg.dependencies(), +)::AbstractVector{Pkg.API.PackageInfo} + deps = Vector{Pkg.API.PackageInfo}(undef, 0) + for (uuid, dep) in project_dependencies + if _match_package_filter(name_filter, dep.name) + push!(deps, dep) + end + end + return deps +end + +""" + set_dev_dependencies( + dependencies::AbstractVector{Pkg.API.PackageInfo}, + custom_urls::AbstractDict{String,String}=Dict{String,String}(), + ) + +Set all dependencies to the development version, if they are not already development versions. +The dict custom_urls takes as key a dependency name and a URL as value. If a dependency is in +custom_urls, it will use the URL as development version. If the dependency does not exist in +custom_urls, it will set the URL https://github.com/QEDjl-project/.jl +""" +function set_dev_dependencies( + dependencies::AbstractVector{Pkg.API.PackageInfo}, + custom_urls::AbstractDict{String,String}=Dict{String,String}(), +) + for dep in dependencies + # if tree_hash is nothing, it is already a dev version + if !isnothing(dep.tree_hash) + if haskey(custom_urls, dep.name) + Pkg.develop(; url=custom_urls[dep.name]) + else + Pkg.develop(; url="https://github.com/QEDjl-project/$(dep.name).jl") + end + end + end +end + +if abspath(PROGRAM_FILE) == @__FILE__ + deps = get_filtered_dependencies(r"^QED*") + set_dev_dependencies(deps) +end