diff --git a/.github/workflows/short-tagging.yml b/.github/workflows/short-tagging.yml deleted file mode 100644 index 28e2e4c..0000000 --- a/.github/workflows/short-tagging.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: Keep the versions up-to-date ☕️ - -on: - release: - types: [published, edited] - -jobs: - actions-tagger: - runs-on: ubuntu-latest - steps: - - uses: Actions-R-Us/actions-tagger@latest - with: - publish_latest_tag: true diff --git a/.github/workflows/test-val-pipeline.yaml b/.github/workflows/test-val-pipeline.yaml new file mode 100644 index 0000000..807d000 --- /dev/null +++ b/.github/workflows/test-val-pipeline.yaml @@ -0,0 +1,14 @@ +--- +name: Test the validation pipeline reusable workflow + +on: + pull_request: + branches: + - main + +jobs: + test-val-pipeline-aragog: + uses: ./.github/workflows/val-pipeline.yaml + with: + target: aragog + report_pkg_dir: ./tests/packages/aragog diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml deleted file mode 100644 index b85e79f..0000000 --- a/.github/workflows/test.yaml +++ /dev/null @@ -1,53 +0,0 @@ ---- -name: Test This Action - -on: - pull_request: - branches: - - main - -jobs: - test: - runs-on: ubuntu-latest - name: ${{ matrix.package.name }} - container: - image: rocker/verse:4.1.2 - env: - GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - strategy: - fail-fast: false - max-parallel: 2 - matrix: - package: - - name: aragog - report_pkg_dir: ./tests/packages/aragog - report_template_path: ./template.Rmd - report_rmarkdown_format: pdf_document - additional_tlmgr_packages: "courier ec fancyhdr" - - name: buckbeak - report_pkg_dir: ./tests/packages/buckbeak - report_template_path: ./tests/packages/buckbeak/validation-template.Rmd - report_rmarkdown_format: html_document - additional_tlmgr_packages: "courier ec" - - steps: - - name: Checkout ${{ matrix.package.name }} - uses: actions/checkout@v3 - - - name: Generate validation report for ${{ matrix.package.name }} - id: validation - uses: ./ - with: - report_pkg_dir: ${{ matrix.package.report_pkg_dir }} - report_template_path: ${{ matrix.package.report_template_path }} - report_output_prefix: validation_report - report_rmarkdown_format: ${{ matrix.package.report_rmarkdown_format }} - additional_tlmgr_packages: ${{ matrix.package.additional_tlmgr_packages }} - - - name: Upload ${{ matrix.package.name }} validation report - uses: actions/upload-artifact@v2 - if: success() - with: - name: ${{ matrix.package.name }} validation report - path: ${{ steps.validation.outputs.report_output_filename }} - if-no-files-found: error diff --git a/.github/workflows/test_no_cache.yaml b/.github/workflows/test_no_cache.yaml deleted file mode 100644 index ab12849..0000000 --- a/.github/workflows/test_no_cache.yaml +++ /dev/null @@ -1,47 +0,0 @@ ---- -name: Test This Action without cache - -on: - pull_request: - branches: - - main - -jobs: - test: - runs-on: ubuntu-latest - name: ${{ matrix.package.name }} - container: - image: rocker/verse:4.1.2 - env: - GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - strategy: - fail-fast: false - max-parallel: 2 - matrix: - package: - - name: buckbeak - report_pkg_dir: ./tests/packages/buckbeak - report_template_path: ./tests/packages/buckbeak/validation-template.Rmd - report_rmarkdown_format: html_document - - steps: - - name: Checkout ${{ matrix.package.name }} - uses: actions/checkout@v3 - - - name: Generate validation report for ${{ matrix.package.name }} - uses: ./ - id: validation - with: - report_pkg_dir: ${{ matrix.package.report_pkg_dir }} - report_template_path: ${{ matrix.package.report_template_path }} - report_output_prefix: validation_report - report_rmarkdown_format: ${{ matrix.package.report_rmarkdown_format }} - no_cache: true - - - name: Upload ${{ matrix.package.name }} validation report - uses: actions/upload-artifact@v2 - if: success() - with: - name: ${{ matrix.package.name }} validation report - path: ${{ steps.validation.outputs.report_output_filename }} - if-no-files-found: error diff --git a/.github/workflows/val-pipeline.yaml b/.github/workflows/val-pipeline.yaml new file mode 100644 index 0000000..d23ff44 --- /dev/null +++ b/.github/workflows/val-pipeline.yaml @@ -0,0 +1,138 @@ +--- +name: Reusable workflow for the validation pipeline + +on: + workflow_call: + inputs: + cran: + description: 'CRAN repository' + required: false + default: 'https://packagemanager.rstudio.com/cran/__linux__/focal/latest' + type: string + target: + description: 'Name of the target package' + required: true + default: 'buckbeak' + type: string + report_pkg_dir: + description: "Path to target package's root" + required: false + default: '.' + type: string + +jobs: + dependencies-matrix: + runs-on: ubuntu-latest + name: Create the matrix of dependencies + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: Checkout ${{ inputs.target }} + uses: actions/checkout@v3 + + - id: set-matrix + run: | + { + echo 'matrix<> "$GITHUB_OUTPUT" + cat $GITHUB_OUTPUT + + val-pipeline-dependencies: + needs: dependencies-matrix + runs-on: ubuntu-latest + name: Validation outputs for dependencies + container: + image: rocker/verse:4.1.2 + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTION_PATH: . + strategy: + fail-fast: false + max-parallel: 2 + matrix: + dependency: ${{ fromJson(needs.dependencies-matrix.outputs.matrix) }} + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set R options + run: | + echo 'options( + repos = "${{ inputs.cran }}", Ncpus = 2, crayon.enabled = TRUE + )' >> ~/.Rprofile + + - name: Set up R library + run: | + mkdir -p ./R_LIBS_USER + echo '.libPaths("./R_LIBS_USER")' >> ~/.Rprofile + + - name: Download dependency + run: | + R -q -e 'remotes_url <- "${{ inputs.cran }}/src/contrib/Archive/remotes/remotes_2.4.2.tar.gz"; + install.packages(remotes_url, repos = NULL, type = "source", + dependencies = c("Depends", "Imports"), + INSTALL_opts = c("--no-docs", "--clean", "--no-multiarch", + "--no-data", "--no-help", "--no-demo") + ); + library(remotes); + archive_dep <- download_version( + "${{ matrix.dependency.name }}", + version = "${{ matrix.dependency.version }}", + type = "source" + ); + untar(archive_dep, exdir = "./_dep/")' + + - name: Get path to dependency + id: get-dep-path + run: echo "dep-path=$(echo $PWD/_dep/${{ matrix.dependency.name }})\n" >> $GITHUB_OUTPUT + + - name: Generate validation report for ${{ matrix.dependency.name }} + id: validation + uses: ./ + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + with: + report_pkg_dir: ${{ steps.get-dep-path.outputs.dep-path }} + report_output_prefix: ${{ inputs.target }}-${{ matrix.dependency.name }}-${{ matrix.dependency.version }}-validation-report + + - name: Upload ${{ matrix.dependency.name }} validation report + uses: actions/upload-artifact@v2 + if: success() + with: + name: ${{ matrix.dependency.name }} validation report + path: ${{ steps.validation.outputs.report_output_filename }} + if-no-files-found: error + + val-pipeline-target: + runs-on: ubuntu-latest + name: Validation output for target package + container: + image: rocker/verse:4.1.2 + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTION_PATH: . + strategy: + fail-fast: false + max-parallel: 2 + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Generate validation report for ${{ inputs.target }} + id: validation + uses: ./ + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + with: + report_pkg_dir: ${{ inputs.report_pkg_dir }} + report_output_prefix: validation_report + + - name: Upload ${{ inputs.target }} validation report + uses: actions/upload-artifact@v2 + if: success() + with: + name: ${{ inputs.target }} validation report + path: ${{ steps.validation.outputs.report_output_filename }} + if-no-files-found: error diff --git a/README.md b/README.md index adff48f..9094540 100644 --- a/README.md +++ b/README.md @@ -1,187 +1,10 @@ -# thevalidatoR +# Description - +Repository of the R Validation Hub's Regulatory R Repository Working Group for a proof-of-concept of a validation pipeline for R packages based on [insightsengineering/thevalidatoR](https://github.com/insightsengineering/thevalidatoR). This validation pipeline is made for use in our future public package repository. -[](https://pharmaverse.org) -[![SuperLinter](https://github.com/insightsengineering/thevalidatoR/actions/workflows/lint.yaml/badge.svg)](https://github.com/insightsengineering/thevalidatoR/actions/workflows/lint.yaml) -[![Test This Action](https://github.com/insightsengineering/thevalidatoR/actions/workflows/test.yaml/badge.svg)](https://github.com/insightsengineering/thevalidatoR/actions/workflows/test.yaml) +For more information on the Working Group, see [pharmaR/regulatory-r-repo-wg](https://github.com/pharmaR/regulatory-r-repo-wg). -- [An R Package Validation Report](#an-r-package-validation-report) - - [Description](#description) - - [Action Type](#action-type) - - [Author](#author) - - [Inputs](#inputs) - - [Outputs](#outputs) - - [How to use](#how-to-use) - - [Quickstart](#quickstart) - - [V1.0 Examples](#v10-examples) - - [rbmi](#rbmi) - - [admiral](#admiral) - - -# An R Package Validation Report - -### Description - -A Github Action that generates a validation report for an R package. The four main steps are: - -- Run `R CMD check` (check installation) -- Run `covr::package_coverage()` (check unit test coverage) -- Run `covtracer` (link documentation to unit tests) -- Place results into report -- _If valtools present - run valtools and also publish report?_ (to discuss) -- Attach report as object to release - -### Action Type - -Composite - -### Author +## Author Roche - -### Inputs - -- `report_pkg_dir`: - - _Description_: Path to package's root - - _Required_: `false` - - _Default_: `.` - -- `report_template_path`: - - _Description_: File path of the R markdown template to use for the report. The default template is available [here.](./template.Rmd) - - _Required_: `false` - - _Default_: `template.Rmd` - -- `report_rmarkdown_format`: - - _Description_: The output format to use when rendering the report. Value is used by `rmarkdown::render`'s `output_format` parameter. - - _Required_: `false` - - _Default_: `pdf_document` - -- `report_output_prefix`: - - _Description_: The output filename prefix for the validation report. If left blank, it defaults to the following convention: `--validation-report`. - - _Required_: `false` - - _Default_: `""` - -- `additional_tlmgr_packages`: - - _Description_: Additional tex packages to install with tlmgr. - - _Required_: `false` - - _Default_: `courier ec` - -- `no_cache`: - - _Description_: Disable github action R dependency caching. - - _Required_: `false` - - _Default_: `false` - -- `cache_version`: - - _Description_: Version of the cache. To clean cache bump this version. - - _Required_: `false` - - _Default_: `v1` - -- `disable_install_dev_deps`: - - _Description_: Disable installation of dev dependencies while building the report. - - _Required_: `false` - - _Default_: `false` - -### Outputs - -- `report_output_filename`: - - _Description_: Filename of the generated report. - - - -## How to use - -To use this GitHub Action you will need to complete the following: - -- Create a new file in your repository called `.github/workflows/r-pkg-validation.yml` -- Copy the template over (and edit if you wish to modify it) - -### Quickstart - -In your repository you should have a `.github/workflows/validatoR.yml` file with GitHub Action similar to below: - -```yaml ---- -name: R Package Validation report - -on: # Run this action when a release is published - release: - types: [published] - -jobs: - r-pkg-validation: - name: Create report 📃 - runs-on: ubuntu-latest - container: - image: rocker/verse:4.1.1 - # Set Github token permissions - env: - GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - permissions: - contents: write - packages: write - deployments: write - steps: - - name: Checkout repo 🛎 - uses: actions/checkout@v3 - - - name: Build report 🏗 - id: validation - uses: insightsengineering/thevalidatoR@main - # see parameters above for custom templates and other formats - - # Upload the validation report to the release - - name: Upload report to release 🔼 - if: success() - uses: svenstaro/upload-release-action@v2 - with: - file: ${{ steps.validation.outputs.report_output_filename }} - asset_name: ${{ steps.validation.outputs.report_output_filename }} - repo_token: ${{ secrets.GITHUB_TOKEN }} - tag: ${{ github.ref }} - overwrite: false -``` - -### V1.0 Examples - -#### rbmi - -You can see an example report from [rbmi](https://github.com/insightsengineering/rbmi), using the first version of this gh-action, [here](readme_files/report-1.0.1-rbmi.pdf). - -This was built as a test on a fork of the original rbmi package. When we created a release in that fork, this PDF was automatically built and added to the release as can be seen below. - -![](readme_files/rbmi_action.png) - -And you can see the gh-action action that was triggered by the release being published. Note that it must install the package, run tests and -construct the metrics needed by cov-tracer and covr, which in the case of `rbmi` a computationally heavy package - took quite a while! - -![](readme_files/rbmi_release.png) - -#### admiral - -You can see an example report from [admiral](https://github.com/Roche-GSK/admiral), using the first version of this gh-action, [here](readme_files/report-0.1-admiral.pdf). +R Validation Hub's Regulatory R Repository WG diff --git a/action.yml b/action.yml index 2fb0743..ac96b70 100644 --- a/action.yml +++ b/action.yml @@ -7,18 +7,6 @@ inputs: description: Path to package's root. required: false default: "." - report_template_path: - description: | - File path of the R markdown template to use for the report. - The default template is available [here.](./template.Rmd) - required: false - default: "./template.Rmd" - report_rmarkdown_format: - description: | - The file format of the validation report. See `rmarkdown::render`'s - `output_format` parameter for accepted values. - required: false - default: "all" report_output_prefix: description: > The output filename prefix for the validation report. If left blank, @@ -26,11 +14,6 @@ inputs: `--validation-report`. required: false default: "" - additional_tlmgr_packages: - description: | - Additional tex packages to install with tlmgr. - required: false - default: "courier ec" no_cache: description: "Disable github action R dependency caching" required: false @@ -68,13 +51,6 @@ runs: cat ~/.Rprofile shell: bash - - name: Set tlmgr cache folder - if: "contains(inputs.no_cache, 'false')" - id: texlive_version - run: | - echo "TEX_LIVE_VERSION=$(tlmgr --version |tail -1 |awk '{print $NF}')" >> $GITHUB_OUTPUT - shell: bash - - name: Cache R packages if: "contains(inputs.no_cache, 'false')" uses: actions/cache@v2 @@ -83,26 +59,9 @@ runs: key: ${{ inputs.cache_version }}-${{ runner.os }}-${{ steps.r_version.outputs.R_VERSION }}-${{ hashFiles('DESCRIPTION') }} restore-keys: ${{ inputs.cache_version }}-${{ runner.os }}-${{ steps.r_version.outputs.R_VERSION }} - - name: Cache Tex packages - if: "contains(inputs.no_cache, 'false')" - uses: actions/cache@v2 - with: - path: /home/runner/work/_temp/TinyTeX - key: ${{ inputs.cache_version }}-${{ runner.os }}-${{ steps.texlive_version.outputs.TEX_LIVE_VERSION }}-${{ hashFiles(inputs.report_template_path) }} - restore-keys: ${{ inputs.cache_version }}-${{ runner.os }}-${{ steps.texlive_version.outputs.TEX_LIVE_VERSION }} - - name: Install dependencies run: | ${GITHUB_ACTION_PATH}/dependencies.R - export PATH=${RUNNER_TEMP}/TinyTeX/bin/x86_64-linux:${PATH} - echo "PATH=${PATH}" >> $GITHUB_ENV - tlmgr path add - tlmgr update --self - [ ! -f "./template.Rmd" ] && cp ${GITHUB_ACTION_PATH}/template.Rmd . || echo "./template.Rmd Already exists" - shell: bash - - - name: Install additional tex packages - run: tlmgr install ${{ inputs.additional_tlmgr_packages }} shell: bash - name: Run report generator @@ -116,7 +75,5 @@ runs: # Composite action doesn't set inputs as env vars. # We need to do this manually... INPUT_REPORT_PKG_DIR: ${{ inputs.report_pkg_dir }} - INPUT_REPORT_TEMPLATE_PATH: ${{ inputs.report_template_path }} - INPUT_REPORT_RMARKDOWN_FORMAT: ${{ inputs.report_rmarkdown_format }} INPUT_REPORT_OUTPUT_PREFIX: ${{ inputs.report_output_prefix }} DISABLE_INSTALL_DEV_DEPS: ${{ inputs.disable_install_dev_deps }} diff --git a/dependencies.R b/dependencies.R index e06bb94..ccd2818 100755 --- a/dependencies.R +++ b/dependencies.R @@ -14,29 +14,4 @@ lapply(github_packages, remotes::install_github) options(repos = c("https://cloud.r-project.org/")) ncores <- parallel::detectCores(all.tests = FALSE, logical = TRUE) if (!require("git2r")) install.packages("git2r", upgrade = "never", Ncpus = ncores) -if (!require("kableExtra")) install.packages("kableExtra", upgrade = "never", Ncpus = ncores) -if (!require("tinytex")) install.packages("tinytex", upgrade = "never", Ncpus = ncores) - -# Conditionally install TinyTex -if(!dir.exists(paste(Sys.getenv("RUNNER_TEMP"), "TinyTeX", sep="/"))) { -# nolint start - tinytex_installer <- ' -export TINYTEX_DIR=${RUNNER_TEMP}/TinyTeX -wget -qO- "https://raw.githubusercontent.com/yihui/tinytex/master/tools/install-unx.sh" | sh -s - --admin --no-path -mkdir -p ${RUNNER_TEMP}/TinyTeX -cp -r ~/.TinyTeX/. ${RUNNER_TEMP}/TinyTeX -rm -rf ~/.TinyTeX -${RUNNER_TEMP}/TinyTeX/bin/*/tlmgr path add -tlmgr update --self -tlmgr install latex-bin luatex xetex ae bibtex context inconsolata listings makeindex metafont mfware parskip pdfcrop tex tools url xkeyval -' - # nolint end - system(tinytex_installer) - tinytex::r_texmf() - permission_update <- ' -chown -R root:staff ${RUNNER_TEMP}/TinyTeX -chmod -R g+w ${RUNNER_TEMP}/TinyTeX -chmod -R g+wx ${RUNNER_TEMP}/TinyTeX/bin -' - system(permission_update) -} +if (!require("jsonlite")) install.packages("jsonlite", upgrade = "never", Ncpus = ncores) diff --git a/report-generator.R b/report-generator.R index 8e97ecc..28cdaf6 100755 --- a/report-generator.R +++ b/report-generator.R @@ -10,8 +10,6 @@ git_safe_dir <- system( # Get the action inputs from preset env vars pkg_dir <- normalizePath(Sys.getenv("INPUT_REPORT_PKG_DIR", ".")) -template_path <- Sys.getenv("INPUT_REPORT_TEMPLATE_PATH", "/template.Rmd") -report_format <- Sys.getenv("INPUT_REPORT_RMARKDOWN_FORMAT", "all") report_output_prefix <- Sys.getenv("INPUT_REPORT_OUTPUT_PREFIX", "") disable_install_dev_deps <- tolower( Sys.getenv("DISABLE_INSTALL_DEV_DEPS") @@ -41,6 +39,41 @@ if (!disable_install_dev_deps) { devtools::install_dev_deps(pkg_dir, upgrade = "never") } +# find .git dir containing the package directory +gd <- system( + sprintf("cd '%s' && git rev-parse --absolute-git-dir", pkg_dir), + intern = TRUE +) +# define reused git args to be sure we're picking up the right git info +gd <- sprintf("--git-dir='%s'", gd) +wt <- sprintf("--work-tree='%s'", pkg_dir) + +validation_report_json <- data.frame( + Field = c("document_typ","package_name","version", "repository", "commit_sha", "github_reference", "branch", "commit_date", "OS", "Platform", "System", "Execution Time"), + Value = c("val_rep_json", #document_typ + read.dcf(desc_file)[,'Package'], #package_name + read.dcf(desc_file)[,'Version'], #version + Sys.getenv('GITHUB_REPOSITORY'), #repository + Sys.getenv('GITHUB_SHA'), #commit_sha + Sys.getenv('GITHUB_REF'), #github_reference + system2( #branch + "git", + list(gd, wt, "rev-parse", "--abbrev-ref", "HEAD"), + stdout = TRUE + ), + system2( #commit_date + "git", + list(gd, wt, "show", "-s", "--format=%ci", "HEAD"), + stdout = TRUE + ), + sessionInfo()$running, #OS + R.version$platform, #Platform + R.version$system, #System + format(Sys.time(), tz = "UTC", usetz = TRUE) #Execution Time + )) +#creates json +json_object <- jsonlite::toJSON(validation_report_json, pretty = TRUE) + # Set the output file name if (report_output_prefix == "") { desc <- read.dcf(desc_file) @@ -51,14 +84,10 @@ if (report_output_prefix == "") { ) } -# allow rmarkdown to choose appropriate file extension for output format -report_file_path <- rmarkdown::render( - template_path, - output_dir = getwd(), # create report wherever R script was called - output_file = report_output_prefix, - output_format = report_format, - params = list(pkg_dir = pkg_dir) -) +report_file_path <- paste0(report_output_prefix,".json") + +# Write the JSON object to a file +write(json_object, file = report_file_path) # Create a tmp file which contains the final report filename writeLines(report_file_path, "/tmp/report_file_path.txt") diff --git a/template.Rmd b/template.Rmd deleted file mode 100644 index 718b3e1..0000000 --- a/template.Rmd +++ /dev/null @@ -1,256 +0,0 @@ ---- -title: "Validation Report" -subtitle: "`r sprintf('%s (v%s)', (dcf <- read.dcf(file.path(params$pkg_dir, 'DESCRIPTION')))[,'Package'], dcf[,'Version'])`" -date: "`r format(Sys.time(), '%a %b %d %X %Y')`" -author: - - "`r paste0('**Server**: ',Sys.getenv('GITHUB_SERVER_URL'))`" - - "`r paste0('**Repository**: ',Sys.getenv('GITHUB_REPOSITORY'))`" - - "`r paste0('**Reference**: ',Sys.getenv('GITHUB_REF'))`" - - "`r paste0('**Commit SHA**: ',Sys.getenv('GITHUB_SHA'))`" -params: - pkg_dir: "`r normalizePath(Sys.getenv('INPUT_REPORT_PACKAGE_DIR', '.'))`" -output: - pdf_document: - toc: true - number_sections: true -header-includes: -- \usepackage{fancyhdr} -- \pagestyle{fancy} -- \fancyfoot[RE,RO]{\thepage} ---- - -\fancyhead[LO,LE]{`r sprintf('%s (v%s)', (dcf <- read.dcf(file.path(params$pkg_dir, 'DESCRIPTION')))[,'Package'], dcf[,'Version'])`} -\fancyfoot[CO,CE]{`r paste0('Commit SHA: ',Sys.getenv('GITHUB_SHA'))`} - -```{r setup, include = FALSE} -options(width = 80L, covr.record_tests = TRUE) -remotes::install_local( - params$pkg_dir, - force = TRUE, - quiet = TRUE, - INSTALL_opts = "--with-keep.source", - upgrade = "never" -) -library(magrittr) -library(knitr) -library(kableExtra) -knitr::opts_chunk$set( - error = TRUE, - width = 80L, - comment = "" -) - -helper_tabulate <- function(x, caption = "", col_widths = c("10cm", "5cm")) { - x %>% - kbl( - caption = caption, - booktabs = TRUE, longtable = TRUE - ) %>% - kable_styling( - latex_options = c("striped", "repeat_header") - ) %>% - column_spec( - 1, width = col_widths[1] - ) %>% - column_spec( - 2, width = col_widths[2] - ) -} -``` - -# Context - -This report was generated via the GH-action insightsengineering/validatoR (gh-action ID: `r Sys.getenv("GITHUB_ACTION")`). It produces automated -documentation of the installation of this package on an open source R environment, focussing on: - -- Installation environment description -- Testing coverage -- Traceability matrix of specifications (documented behaviours) and testing -- Risk assessment benchmarks - -This report is fully automated, so is limited to assess whether unit tests and documentation are present and can execute without error. An assessment would be required that the tests and documentation are meaningful. Validation aims to be system independent as the underlying workflow is based on the ["composite"](https://docs.github.com/en/actions/creating-actions/creating-a-composite-action) type of Github Action. - -# Installation environment and package - -## System Info - -```{r execution_info, echo = FALSE} -data.frame( - Field = c("OS", "Platform", "System", "Execution Time"), - Value = c( - sessionInfo()$running, - R.version$platform, - R.version$system, - format(Sys.time(), tz = "UTC", usetz = TRUE) - )) %>% helper_tabulate("System info", col_widths = c("6cm", "6cm")) -``` - -## Package installed - -```{r version_control, echo = FALSE} -# find .git dir containing the package directory -gd <- system( - sprintf("cd '%s' && git rev-parse --absolute-git-dir", params$pkg_dir), - intern = TRUE -) -# define reused git args to be sure we're picking up the right git info -gd <- sprintf("--git-dir='%s'", gd) -wt <- sprintf("--work-tree='%s'", params$pkg_dir) -data.frame( - Field = c("branch", "commit `SHA1`", "commit date"), - Value = c( - system2( - "git", - list(gd, wt, "rev-parse", "--abbrev-ref", "HEAD"), - stdout = TRUE - ), - system2("git", list(gd, wt, "rev-parse", "HEAD"), stdout = TRUE), - system2( - "git", - list(gd, wt, "show", "-s", "--format=%ci", "HEAD"), - stdout = TRUE - ) - )) %>% helper_tabulate("Git information", col_widths = c("5cm", "8cm")) -``` - -## R Session Info - -```{r session_info, echo = TRUE, eval = TRUE} -sessionInfo() -capabilities() -``` - -# Metric based risk assessment - -The following metrics are derived from the `riskmetric` R package. Metrics overlapping with `covr` and `R CMD Check` are removed. - -```{r riskmetric, echo = FALSE, eval = TRUE} -d_riskmetric <- params$pkg_dir %>% - riskmetric::pkg_ref() %>% - riskmetric::pkg_assess() %>% - purrr::map(1) %>% - lapply(as.character) %>% - tibble::enframe() %>% - tidyr::unnest(cols = dplyr::everything()) %>% - # add labels - dplyr::left_join( - lapply(riskmetric::all_assessments(), attributes) %>% - purrr::map_df(tibble::as_tibble), - by = c("name" = "column_name") - ) - -d_riskmetric %>% - dplyr::filter( - name %in% c( - "news_current", "has_vignettes", - "license", "downloads_1yr" - ) - ) %>% - dplyr::select(Metric = label, Status = value) %>% - #table - helper_tabulate( - caption = "Package info assessed by the R package riskmetric" - ) -``` - -# Installation documentation - -## `R CMD check` - -```{r r_cmd_check, echo = FALSE, eval = TRUE} -rcmdcheck_results <- rcmdcheck::rcmdcheck( - params$pkg_dir, - args = c( - "--timings", # include execution times in output - "--no-build-vignettes", # run vignette code, but disable pdf rendering - "--no-manual" # disable pdf manual rendering - ), - quiet = TRUE -) - -cat(rcmdcheck_results$stdout) -cat(rcmdcheck_results$stderr) -``` - -## Testing Coverage - -```{r coverage, echo = FALSE, eval = TRUE} -covr_results <- covr::package_coverage(params$pkg_dir) -covr_results -``` - -## Traceability - -Tracebility matrix that maps each unit test to the corresponding documentation, creating a link -between intended use and testing. - -### Testing matrix - -```{r traceability, echo = FALSE, eval = TRUE} -if (require("covtracer", quietly = TRUE)) { - covtracer_df <- test_trace_df(covr_results) - covtracer_df$filename <- basename(covtracer_df$filepath) - - covtracer_df_clean <- covtracer_df %>% - dplyr::select( - alias, - test_name, - file, - is_exported, - doctype, - direct - ) %>% - dplyr::filter(!doctype %in% c("data", "class")) %>% # ignore objects without testable code - dplyr::filter(is_exported == TRUE) %>% # Only exported - dplyr::filter(!duplicated(.)) %>% - dplyr::mutate(alias = paste0(alias, "()")) %>% - dplyr::mutate(file = paste0("man/", file)) - - covtracer_df_clean %>% - dplyr::filter(!duplicated(.)) %>% - dplyr::filter(!is.na(test_name)) %>% - dplyr::filter(!is.na(file)) %>% # unexported? - dplyr::arrange(file) %>% - dplyr::select(`Test Description` = test_name, Documentation = file) %>% - helper_tabulate( - caption = "Tracebility matrix mapping unit tests to documented behaviours." - ) -} else { - cat("{covtracer} not available to produce a traceability matrix") -} -``` - -### Untested - -```{r traceability2, echo = FALSE, eval = TRUE} -if (require("covtracer", quietly = TRUE)) { - covtracer_df_clean %>% - dplyr::filter(is.na(test_name)) %>% - dplyr::arrange(alias) %>% - dplyr::select(`Exported package object` = alias, Documentation = file) %>% - helper_tabulate( - caption = "Untested behaviours: documentation that is not covered by any test." - ) -} else { - cat("{covtracer} not available to produce a traceability matrix") -} -``` - -### Testing granularity - -An indicator of test granularity by whether the function is directly tested. - -```{r traceability3, echo = FALSE, eval = TRUE} -if (require("covtracer", quietly = TRUE)) { - covtracer_df_clean %>% - dplyr::group_by(alias) %>% - dplyr::summarize(any_direct_tests = any(direct, na.rm = TRUE)) %>% - dplyr::arrange(alias) %>% - dplyr::select(`Exported package object` = alias, `Tested Directly` = any_direct_tests) %>% - helper_tabulate( - caption = "Granularity of unit tests: directly tested exported functions." - ) -} else { - cat("{covtracer} not available to produce a traceability matrix") -} -``` diff --git a/tests/packages/aragog/DESCRIPTION b/tests/packages/aragog/DESCRIPTION index e330596..00296e2 100644 --- a/tests/packages/aragog/DESCRIPTION +++ b/tests/packages/aragog/DESCRIPTION @@ -9,4 +9,7 @@ License: file LICENSE Type: Package Encoding: UTF-8 RoxygenNote: 7.0.2 +Imports: + logrx (>= 0.3.0), + tidytlg (>= 0.1.4) Suggests: testthat diff --git a/tests/packages/aragog/pkgs_cran.json b/tests/packages/aragog/pkgs_cran.json new file mode 100644 index 0000000..30683be --- /dev/null +++ b/tests/packages/aragog/pkgs_cran.json @@ -0,0 +1,4 @@ +[ + {"name": "logrx", "version": "0.3.0"}, + {"name": "tidytlg", "version": "0.1.4"} +] diff --git a/tests/packages/buckbeak/pkgs_cran.json b/tests/packages/buckbeak/pkgs_cran.json new file mode 100644 index 0000000..30683be --- /dev/null +++ b/tests/packages/buckbeak/pkgs_cran.json @@ -0,0 +1,4 @@ +[ + {"name": "logrx", "version": "0.3.0"}, + {"name": "tidytlg", "version": "0.1.4"} +] diff --git a/tests/packages/buckbeak/validation-template.Rmd b/tests/packages/buckbeak/validation-template.Rmd deleted file mode 100644 index 47c30e0..0000000 --- a/tests/packages/buckbeak/validation-template.Rmd +++ /dev/null @@ -1,49 +0,0 @@ ---- -title: "Buckbeak Validation Report" -output: - - "md_document" -params: - pkg_dir: "." ---- - - -```{r, include = FALSE} -options(width = 80L, covr.record_tests = TRUE) - -remotes::install_local( - params$pkg_dir, - force = TRUE, - quiet = TRUE, - INSTALL_opts = "--with-keep.source" -) - -library(magrittr) -library(knitr) -knitr::opts_chunk$set( - width = 80L, - comment = "" -) -``` - - -# Execution Info - -## System Info - -```{r execution_info, echo = FALSE} -kable(data.frame( - Field = c("OS", "Platform", "System", "Execution Time"), - Value = c( - sessionInfo()$running, - R.version$platform, - R.version$system, - format(Sys.time(), tz = "UTC", usetz = TRUE) - ))) -``` - -## Session Info - -```{r session_info, echo = TRUE} -sessionInfo() -capabilities() -```