Skip to content
This repository has been archived by the owner on Dec 12, 2024. It is now read-only.

Commit

Permalink
conformance wip
Browse files Browse the repository at this point in the history
  • Loading branch information
@leordev
leordev committed Sep 30, 2024
1 parent 4d482ad commit 8d9c783
Show file tree
Hide file tree
Showing 30 changed files with 1,877 additions and 107 deletions.
28 changes: 27 additions & 1 deletion .github/actions/specs-report/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,13 @@

[![Coverage](./badges/coverage.svg)](./badges/coverage.svg)

GitHub Action to Parse and Generate tbd Specs Reports
GitHub Action to Parse and Generate tbd Specs Reports and handle the TBD Spec
SDKs version Conformance matrix Dashboard

## Usage

### Standard Test Vector Reporting

```yaml
# add it as an step on your CI workflow right after your tests

Expand All @@ -32,6 +35,29 @@ Example report:
![Example Report](./example-report.png)
### Spec Conformance Dashboard Release Modes
To handle spec conformance dashboard automation, the action can operate in three
modes using the `release-mode` input:

- **Default Mode (`release-mode` not set):** Runs the standard test vector
reporting (as seen above).
- **Spec Release Mode (`release-mode: spec`):** When a new spec version is
released, adds a new entry to the `specReleases` array in the JSON.
- **SDK Release Mode (`release-mode: sdk`):** When a new SDK version is
released, updates the corresponding SDK entry under the relevant spec version
in the JSON.

This should only be used in the SDK report runner repo. To test it locally you
could see the example files inputs and dispatch the workflows using the `gh`
cli.

```sh
gh workflow run build-conformance-table.yaml --ref <testing-branch> --json < example-release-spec-inputs.json
gh workflow run build-conformance-table.yaml --ref <testing-branch> --json < example-release-sdk-inputs-web5-core-kt.json
gh workflow run build-conformance-table.yaml --ref <testing-branch> --json < example-release-sdk-inputs-web5-rs.json
```

## Initial Setup

1. :hammer_and_wrench: Install the dependencies
Expand Down
86 changes: 86 additions & 0 deletions ....0/kotlin-test-results/TEST-web5.sdk.Web5TestVectorsTest$Web5TestVectorsCryptoEd25519.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuite xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="https://maven.apache.org/surefire/maven-surefire-plugin/xsd/surefire-test-report-3.0.xsd" version="3.0" name="web5.sdk.Web5TestVectorsTest$Web5TestVectorsCryptoEd25519" time="2.565" tests="2" errors="0" skipped="0" failures="0">
<properties>
<property name="awt.toolkit" value="sun.awt.X11.XToolkit"/>
<property name="java.specification.version" value="11"/>
<property name="sun.cpu.isalist" value=""/>
<property name="sun.jnu.encoding" value="UTF-8"/>
<property name="java.class.path" value="/home/runner/work/web5-rs/web5-rs/bound/kt/target/test-classes:/home/runner/work/web5-rs/web5-rs/bound/kt/target/classes:/home/runner/.m2/repository/com/fasterxml/jackson/core/jackson-databind/2.17.0/jackson-databind-2.17.0.jar:/home/runner/.m2/repository/com/fasterxml/jackson/core/jackson-annotations/2.17.0/jackson-annotations-2.17.0.jar:/home/runner/.m2/repository/com/fasterxml/jackson/core/jackson-core/2.17.0/jackson-core-2.17.0.jar:/home/runner/.m2/repository/net/bytebuddy/byte-buddy/1.14.9/byte-buddy-1.14.9.jar:/home/runner/.m2/repository/com/fasterxml/jackson/module/jackson-module-kotlin/2.17.0/jackson-module-kotlin-2.17.0.jar:/home/runner/.m2/repository/org/jetbrains/kotlin/kotlin-reflect/1.7.22/kotlin-reflect-1.7.22.jar:/home/runner/.m2/repository/net/java/dev/jna/jna/5.12.0/jna-5.12.0.jar:/home/runner/.m2/repository/org/jetbrains/kotlinx/kotlinx-coroutines-core/1.8.1/kotlinx-coroutines-core-1.8.1.jar:/home/runner/.m2/repository/org/jetbrains/kotlinx/kotlinx-coroutines-core-jvm/1.8.1/kotlinx-coroutines-core-jvm-1.8.1.jar:/home/runner/.m2/repository/org/jetbrains/annotations/23.0.0/annotations-23.0.0.jar:/home/runner/.m2/repository/org/jetbrains/kotlin/kotlin-stdlib-jdk8/1.9.22/kotlin-stdlib-jdk8-1.9.22.jar:/home/runner/.m2/repository/org/jetbrains/kotlin/kotlin-stdlib/1.9.22/kotlin-stdlib-1.9.22.jar:/home/runner/.m2/repository/org/jetbrains/kotlin/kotlin-stdlib-jdk7/1.9.22/kotlin-stdlib-jdk7-1.9.22.jar:/home/runner/.m2/repository/org/junit/jupiter/junit-jupiter-api/5.10.1/junit-jupiter-api-5.10.1.jar:/home/runner/.m2/repository/org/opentest4j/opentest4j/1.3.0/opentest4j-1.3.0.jar:/home/runner/.m2/repository/org/junit/platform/junit-platform-commons/1.10.1/junit-platform-commons-1.10.1.jar:/home/runner/.m2/repository/org/apiguardian/apiguardian-api/1.1.2/apiguardian-api-1.1.2.jar:/home/runner/.m2/repository/com/squareup/okhttp3/mockwebserver/4.12.0/mockwebserver-4.12.0.jar:/home/runner/.m2/repository/com/squareup/okhttp3/okhttp/4.12.0/okhttp-4.12.0.jar:/home/runner/.m2/repository/com/squareup/okio/okio/3.6.0/okio-3.6.0.jar:/home/runner/.m2/repository/com/squareup/okio/okio-jvm/3.6.0/okio-jvm-3.6.0.jar:/home/runner/.m2/repository/org/jetbrains/kotlin/kotlin-stdlib-common/1.9.10/kotlin-stdlib-common-1.9.10.jar:/home/runner/.m2/repository/junit/junit/4.13.2/junit-4.13.2.jar:/home/runner/.m2/repository/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar:/home/runner/.m2/repository/com/nimbusds/nimbus-jose-jwt/9.39.3/nimbus-jose-jwt-9.39.3.jar:"/>
<property name="java.vm.vendor" value="Eclipse Adoptium"/>
<property name="sun.arch.data.model" value="64"/>
<property name="coverage.ignore.private.constructor.util.class" value="true"/>
<property name="java.vendor.url" value="https://adoptium.net/"/>
<property name="user.timezone" value="Etc/UTC"/>
<property name="java.vm.specification.version" value="11"/>
<property name="os.name" value="Linux"/>
<property name="sun.java.launcher" value="SUN_STANDARD"/>
<property name="sun.boot.library.path" value="/opt/hostedtoolcache/Java_Adopt_jdk/11.0.24-8/x64/lib"/>
<property name="sun.java.command" value="/home/runner/work/web5-rs/web5-rs/bound/kt/target/surefire/surefirebooter-20240925220445746_3.jar /home/runner/work/web5-rs/web5-rs/bound/kt/target/surefire 2024-09-25T22-04-45_631-jvmRun1 surefire-20240925220445746_1tmp surefire_0-20240925220445746_2tmp"/>
<property name="jdk.debug" value="release"/>
<property name="surefire.test.class.path" value="/home/runner/work/web5-rs/web5-rs/bound/kt/target/test-classes:/home/runner/work/web5-rs/web5-rs/bound/kt/target/classes:/home/runner/.m2/repository/com/fasterxml/jackson/core/jackson-databind/2.17.0/jackson-databind-2.17.0.jar:/home/runner/.m2/repository/com/fasterxml/jackson/core/jackson-annotations/2.17.0/jackson-annotations-2.17.0.jar:/home/runner/.m2/repository/com/fasterxml/jackson/core/jackson-core/2.17.0/jackson-core-2.17.0.jar:/home/runner/.m2/repository/net/bytebuddy/byte-buddy/1.14.9/byte-buddy-1.14.9.jar:/home/runner/.m2/repository/com/fasterxml/jackson/module/jackson-module-kotlin/2.17.0/jackson-module-kotlin-2.17.0.jar:/home/runner/.m2/repository/org/jetbrains/kotlin/kotlin-reflect/1.7.22/kotlin-reflect-1.7.22.jar:/home/runner/.m2/repository/net/java/dev/jna/jna/5.12.0/jna-5.12.0.jar:/home/runner/.m2/repository/org/jetbrains/kotlinx/kotlinx-coroutines-core/1.8.1/kotlinx-coroutines-core-1.8.1.jar:/home/runner/.m2/repository/org/jetbrains/kotlinx/kotlinx-coroutines-core-jvm/1.8.1/kotlinx-coroutines-core-jvm-1.8.1.jar:/home/runner/.m2/repository/org/jetbrains/annotations/23.0.0/annotations-23.0.0.jar:/home/runner/.m2/repository/org/jetbrains/kotlin/kotlin-stdlib-jdk8/1.9.22/kotlin-stdlib-jdk8-1.9.22.jar:/home/runner/.m2/repository/org/jetbrains/kotlin/kotlin-stdlib/1.9.22/kotlin-stdlib-1.9.22.jar:/home/runner/.m2/repository/org/jetbrains/kotlin/kotlin-stdlib-jdk7/1.9.22/kotlin-stdlib-jdk7-1.9.22.jar:/home/runner/.m2/repository/org/junit/jupiter/junit-jupiter-api/5.10.1/junit-jupiter-api-5.10.1.jar:/home/runner/.m2/repository/org/opentest4j/opentest4j/1.3.0/opentest4j-1.3.0.jar:/home/runner/.m2/repository/org/junit/platform/junit-platform-commons/1.10.1/junit-platform-commons-1.10.1.jar:/home/runner/.m2/repository/org/apiguardian/apiguardian-api/1.1.2/apiguardian-api-1.1.2.jar:/home/runner/.m2/repository/com/squareup/okhttp3/mockwebserver/4.12.0/mockwebserver-4.12.0.jar:/home/runner/.m2/repository/com/squareup/okhttp3/okhttp/4.12.0/okhttp-4.12.0.jar:/home/runner/.m2/repository/com/squareup/okio/okio/3.6.0/okio-3.6.0.jar:/home/runner/.m2/repository/com/squareup/okio/okio-jvm/3.6.0/okio-jvm-3.6.0.jar:/home/runner/.m2/repository/org/jetbrains/kotlin/kotlin-stdlib-common/1.9.10/kotlin-stdlib-common-1.9.10.jar:/home/runner/.m2/repository/junit/junit/4.13.2/junit-4.13.2.jar:/home/runner/.m2/repository/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar:/home/runner/.m2/repository/com/nimbusds/nimbus-jose-jwt/9.39.3/nimbus-jose-jwt-9.39.3.jar:"/>
<property name="sun.cpu.endian" value="little"/>
<property name="user.home" value="/home/runner"/>
<property name="user.language" value="en"/>
<property name="java.specification.vendor" value="Oracle Corporation"/>
<property name="java.version.date" value="2024-07-16"/>
<property name="java.home" value="/opt/hostedtoolcache/Java_Adopt_jdk/11.0.24-8/x64"/>
<property name="file.separator" value="/"/>
<property name="basedir" value="/home/runner/work/web5-rs/web5-rs/bound/kt"/>
<property name="java.vm.compressedOopsMode" value="Zero based"/>
<property name="line.separator" value="&#10;"/>
<property name="java.specification.name" value="Java Platform API Specification"/>
<property name="java.vm.specification.vendor" value="Oracle Corporation"/>
<property name="java.awt.graphicsenv" value="sun.awt.X11GraphicsEnvironment"/>
<property name="idea.coverage.calculate.hits" value="false"/>
<property name="surefire.real.class.path" value="/home/runner/work/web5-rs/web5-rs/bound/kt/target/surefire/surefirebooter-20240925220445746_3.jar"/>
<property name="sun.management.compiler" value="HotSpot 64-Bit Tiered Compilers"/>
<property name="java.runtime.version" value="11.0.24+8"/>
<property name="user.name" value="runner"/>
<property name="path.separator" value=":"/>
<property name="os.version" value="6.8.0-1014-azure"/>
<property name="java.runtime.name" value="OpenJDK Runtime Environment"/>
<property name="file.encoding" value="UTF-8"/>
<property name="jnidispatch.path" value="/home/runner/.cache/JNA/temp/jna798805214436154586.tmp"/>
<property name="java.vm.name" value="OpenJDK 64-Bit Server VM"/>
<property name="jna.platform.library.path" value="/usr/lib/x86_64-linux-gnu:/lib/x86_64-linux-gnu:/usr/lib64:/lib64:/usr/lib:/lib:/lib32:/usr/local/lib:/usr/lib/x86_64-linux-gnu/libfakeroot"/>
<property name="java.vendor.version" value="Temurin-11.0.24+8"/>
<property name="localRepository" value="/home/runner/.m2/repository"/>
<property name="jna.loaded" value="true"/>
<property name="java.vendor.url.bug" value="https://github.com/adoptium/adoptium-support/issues"/>
<property name="java.io.tmpdir" value="/tmp"/>
<property name="idea.coverage.log.level" value="error"/>
<property name="java.version" value="11.0.24"/>
<property name="user.dir" value="/home/runner/work/web5-rs/web5-rs/bound/kt"/>
<property name="os.arch" value="amd64"/>
<property name="java.vm.specification.name" value="Java Virtual Machine Specification"/>
<property name="java.awt.printerjob" value="sun.print.PSPrinterJob"/>
<property name="sun.os.patch.level" value="unknown"/>
<property name="java.library.path" value="/usr/java/packages/lib:/usr/lib64:/lib64:/lib:/usr/lib"/>
<property name="java.vm.info" value="mixed mode"/>
<property name="java.vendor" value="Eclipse Adoptium"/>
<property name="java.vm.version" value="11.0.24+8"/>
<property name="java.specification.maintenance.version" value="2"/>
<property name="sun.io.unicode.encoding" value="UnicodeLittle"/>
<property name="idea.new.tracing.coverage" value="true"/>
<property name="java.class.version" value="55.0"/>
</properties>
<testcase name="verify" classname="web5.sdk.Web5TestVectorsTest$Web5TestVectorsCryptoEd25519" time="2.508">
<system-out><![CDATA[web5 sdk SystemArchitecture System architecture: amd64
web5 sdk SystemArchitecture Operating system name: linux
web5 sdk SystemArchitecture OS release content: pretty_name="ubuntu 22.04.5 lts"
name="ubuntu"
version_id="22.04"
version="22.04.5 lts (jammy jellyfish)"
version_codename=jammy
id=ubuntu
id_like=debian
home_url="https://www.ubuntu.com/"
support_url="https://help.ubuntu.com/"
bug_report_url="https://bugs.launchpad.net/ubuntu/"
privacy_policy_url="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"
ubuntu_codename=jammy
]]></system-out>
</testcase>
<testcase name="sign" classname="web5.sdk.Web5TestVectorsTest$Web5TestVectorsCryptoEd25519" time="0.036"/>
</testsuite>
Loading

0 comments on commit 8d9c783

Please sign in to comment.