diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index fcf0eab..d3cfaf6 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -5,6 +5,9 @@ on: branches: - main pull_request: + workflow_dispatch: + schedule: + - cron: '10 10 * * *' jobs: selftest: @@ -22,8 +25,13 @@ jobs: - name: conformance test sigstore-python uses: ./ - id: sigstore-conformance with: entrypoint: ${{ github.workspace }}/sigstore-python-conformance - enable-staging: true + xfail: "test_verify_with_trust_root test_verify_dsse_bundle_with_trust_root" + + - name: Staging conformance test sigstore-python + uses: ./ + with: + entrypoint: ${{ github.workspace }}/sigstore-python-conformance + environment: staging xfail: "test_verify_with_trust_root test_verify_dsse_bundle_with_trust_root" diff --git a/Makefile b/Makefile index 0b43d5e..f6c097c 100644 --- a/Makefile +++ b/Makefile @@ -18,5 +18,5 @@ dev: env/pyvenv.cfg .PHONY: lint lint: env/pyvenv.cfg $(ALL_PY_SRCS) ./env/bin/python -m black $(ALL_PY_SRCS) - ./env/bin/python -m ruff --fix $(ALL_PY_SRCS) + ./env/bin/python -m ruff check --fix $(ALL_PY_SRCS) ./env/bin/python -m mypy action.py test/ diff --git a/README.md b/README.md index 56180fb..55bc393 100644 --- a/README.md +++ b/README.md @@ -43,10 +43,19 @@ client-under-test [CLI protocol](docs/cli_protocol.md). runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + # insert your client installation steps here + + # Run tests against production Sigstore environment + - uses: sigstore/sigstore-conformance@v0.0.10 + with: + entrypoint: my-conformance-client + + # Run tests against staging Sigstore environment - uses: sigstore/sigstore-conformance@v0.0.10 with: entrypoint: my-conformance-client + environment: staging ``` See [sigstore-python conformance test](https://github.com/sigstore/sigstore-python/blob/main/.github/workflows/conformance.yml) @@ -57,8 +66,8 @@ for a complete example. The important action inputs are * `entrypoint`: required string. A command that implements the client-under-test [CLI protocol](docs/cli_protocol.md) -* `enable-staging`: optional boolean. When true, the test suite will run tests against - staging infrastructure in addition to running them against production infrastructure +* `environment`: 'production' (default) or 'staging'. This selects the Sigstore environment to + run against * `xfail`: optional string. Whitespace separated test names that are expected to fail. See [action.yml](action.yml) for full list of inputs. @@ -77,8 +86,7 @@ The test suite can be configured with [CLI specification](https://github.com/sigstore/sigstore-conformance/blob/main/docs/cli_protocol.md) * `--identity-token=$GITHUB_TOKEN` where GITHUB_TOKEN is a GitHub token with actions:read access for public repositories (--identity-token is only required for signing tests) -* optional (and currently experimental) `--staging`: This instructs the test suite to run - against Sigstore staging infrastructure +* optional `--staging`: This instructs the test suite to run against Sigstore staging infrastructure * The environment variable `GHA_SIGSTORE_CONFORMANCE_XFAIL` can be used to set expected results diff --git a/action.py b/action.py index 741c4cd..f6038a5 100755 --- a/action.py +++ b/action.py @@ -14,7 +14,6 @@ _RENDER_SUMMARY = os.getenv("GHA_SIGSTORE_CONFORMANCE_SUMMARY", "true") == "true" _DEBUG = os.getenv("GHA_SIGSTORE_CONFORMANCE_INTERNAL_BE_CAREFUL_DEBUG", "false") != "false" _ACTION_PATH = Path(os.getenv("GITHUB_ACTION_PATH")) # type: ignore -_ENABLE_STAGING = os.getenv("GHA_SIGSTORE_CONFORMANCE_ENABLE_STAGING", "false").lower() == "true" def _summary(msg): @@ -27,7 +26,7 @@ def _debug(msg): print(f"\033[93mDEBUG: {msg}\033[0m", file=sys.stderr) -def _sigstore_conformance(staging: bool) -> int: +def _sigstore_conformance(environment: str) -> int: args = [] if _DEBUG: @@ -37,8 +36,10 @@ def _sigstore_conformance(staging: bool) -> int: if entrypoint: args.extend(["--entrypoint", entrypoint]) - if staging: + if environment == "staging": args.append("--staging") + elif environment != "production": + raise ValueError(f"Unknown environment '{environment}'") skip_signing = os.getenv("GHA_SIGSTORE_CONFORMANCE_SKIP_SIGNING", "false").lower() == "true" if skip_signing: @@ -48,17 +49,15 @@ def _sigstore_conformance(staging: bool) -> int: if gh_token: args.extend(["--github-token", gh_token]) - infra = "staging" if staging else "production" - print(f"running sigstore-conformance against Sigstore {infra} infrastructure") + print(f"running sigstore-conformance against Sigstore {environment} infrastructure") _debug(f"running: sigstore-conformance {[str(a) for a in args]}") return pytest.main([str(_ACTION_PATH / "test"), *args]) -# Run against production, then optionally against staging -status = _sigstore_conformance(staging=False) -if _ENABLE_STAGING: - status += _sigstore_conformance(staging=True) +# Run against chosen environment +environment = os.getenv("GHA_SIGSTORE_CONFORMANCE_ENVIRONMENT", "production") +status = _sigstore_conformance(environment) if status == 0: _summary("🎉 sigstore-conformance exited successfully") diff --git a/action.yml b/action.yml index 4ff0f5b..b0edb5e 100644 --- a/action.yml +++ b/action.yml @@ -15,10 +15,10 @@ inputs: description: "skip tests that involve signing (default false)" required: false default: "false" - enable-staging: - description: "Test against staging infrastructure as well as production (default false)" + environment: + description: "'production' (default) or 'staging'" required: false - default: "false" + default: "production" xfail: description: "one or more tests that are expected to fail, whitespace-separated" required: false @@ -38,7 +38,7 @@ runs: run: | ${{ github.action_path }}/action.py env: - GHA_SIGSTORE_CONFORMANCE_ENABLE_STAGING: "${{ inputs.enable-staging }}" + GHA_SIGSTORE_CONFORMANCE_ENVIRONMENT: "${{ inputs.environment }}" GHA_SIGSTORE_CONFORMANCE_ENTRYPOINT: "${{ inputs.entrypoint }}" GHA_SIGSTORE_CONFORMANCE_INTERNAL_BE_CAREFUL_DEBUG: "${{ inputs.internal-be-careful-debug }}" GHA_SIGSTORE_CONFORMANCE_SKIP_SIGNING: "${{ inputs.skip-signing }}" diff --git a/docs/cli_protocol.md b/docs/cli_protocol.md index b872aa8..947ac5e 100644 --- a/docs/cli_protocol.md +++ b/docs/cli_protocol.md @@ -24,26 +24,20 @@ client's native CLI accepts. This is the set of subcommands that the test CLI must support. Each subcommand has a provided syntax and list of descriptions for each argument. -To simplify argument parsing, all arguments are required, except `--staging`, and will **always** be -supplied by the conformance suite in the order that they are specified in the -templates below. - -All commands below are allowed to run against staging by appending the `--staging` in the command, for example: - -```console -${ENTRYPOINT} sign --identity-token TOKEN --signature FILE --certificate FILE FILE --staging -``` +To simplify argument parsing, arguments will **always** be supplied by the +conformance suite in the order that they are specified in the templates below. ### Sign #### Signature and certificate flow ```console -${ENTRYPOINT} sign --identity-token TOKEN --signature FILE --certificate FILE FILE +${ENTRYPOINT} sign [--staging] --identity-token TOKEN --signature FILE --certificate FILE FILE ``` | Option | Description | | --- | --- | +| `--staging` | Presence indicates client should use Sigstore staging infrastructure | | `--identity-token` | The OIDC identity token to use | | `--signature FILE` | The path to write the signature to | | `--certificate FILE` | The path to write the signing certificate to | @@ -52,11 +46,12 @@ ${ENTRYPOINT} sign --identity-token TOKEN --signature FILE --certificate FILE FI #### Bundle flow ```console -${ENTRYPOINT} sign-bundle --identity-token TOKEN --bundle FILE FILE +${ENTRYPOINT} sign-bundle [--staging] --identity-token TOKEN --bundle FILE FILE ``` | Option | Description | | --- | --- | +| `--staging` | Presence indicates client should use Sigstore staging infrastructure | | `--identity-token` | The OIDC identity token to use | | `--bundle FILE` | The path to write the bundle to | | `FILE` | The artifact to sign | @@ -66,11 +61,12 @@ ${ENTRYPOINT} sign-bundle --identity-token TOKEN --bundle FILE FILE #### Signature and certificate flow ```console -${ENTRYPOINT} verify --signature FILE --certificate FILE --certificate-identity IDENTITY --certificate-oidc-issuer URL [--trusted-root FILE] FILE +${ENTRYPOINT} verify [--staging] --signature FILE --certificate FILE --certificate-identity IDENTITY --certificate-oidc-issuer URL [--trusted-root FILE] FILE ``` | Option | Description | | --- | --- | +| `--staging` | Presence indicates client should use Sigstore staging infrastructure | | `--signature FILE` | The path to the signature to verify | | `--certificate FILE` | The path to the signing certificate to verify | | `--certificate-identity IDENTITY` | The expected identity in the signing certificate's SAN extension | @@ -81,11 +77,12 @@ ${ENTRYPOINT} verify --signature FILE --certificate FILE --certificate-identity #### Bundle flow ```console -${ENTRYPOINT} verify-bundle --bundle FILE --certificate-identity IDENTITY --certificate-oidc-issuer URL [--trusted-root FILE] FILE +${ENTRYPOINT} verify-bundle [--staging] --bundle FILE --certificate-identity IDENTITY --certificate-oidc-issuer URL [--trusted-root FILE] FILE ``` | Option | Description | | --- | --- | +| `--staging` | Presence indicates client should use Sigstore staging infrastructure | | `--bundle FILE` | The path to the Sigstore bundle to verify | | `--certificate-identity IDENTITY` | The expected identity in the signing certificate's SAN extension | | `--certificate-oidc-issuer URL` | The expected OIDC issuer for the signing certificate | diff --git a/pyproject.toml b/pyproject.toml index 9a4b3a5..da7a2b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,4 +3,4 @@ line-length = 100 [tool.ruff] line-length = 100 -select = ["E", "F", "I", "W", "UP"] +lint.select = ["E", "F", "I", "W", "UP"] diff --git a/test/client.py b/test/client.py index b986fc5..86d614a 100644 --- a/test/client.py +++ b/test/client.py @@ -128,8 +128,6 @@ def run(self, *args) -> None: """ self.completed_process = None full_command = [self.entrypoint, *args] - if self.staging: - full_command.append("--staging") try: self.completed_process = subprocess.run( @@ -187,16 +185,20 @@ def _sign_for_sigcrt( This is an overload of `sign` for the signature/certificate flow and should not be called directly. """ - args = [ - "sign", - "--identity-token", - self.identity_token, - "--signature", - materials.signature, - "--certificate", - materials.certificate, - artifact, - ] + args: list[str | os.PathLike] = ["sign"] + if self.staging: + args.append("--staging") + args.extend( + [ + "--identity-token", + self.identity_token, + "--signature", + materials.signature, + "--certificate", + materials.certificate, + artifact, + ] + ) self.run(*args) @@ -208,14 +210,18 @@ def _sign_for_bundle(self, materials: BundleMaterials, artifact: os.PathLike) -> This is an overload of `sign` for the bundle flow and should not be called directly. """ - args = [ - "sign-bundle", - "--identity-token", - self.identity_token, - "--bundle", - materials.bundle, - artifact, - ] + args: list[str | os.PathLike] = ["sign-bundle"] + if self.staging: + args.append("--staging") + args.extend( + [ + "--identity-token", + self.identity_token, + "--bundle", + materials.bundle, + artifact, + ] + ) self.run(*args) @@ -243,17 +249,21 @@ def _verify_for_sigcrt( not be called directly. """ - args = [ - "verify", - "--signature", - materials.signature, - "--certificate", - materials.certificate, - "--certificate-identity", - CERTIFICATE_IDENTITY, - "--certificate-oidc-issuer", - CERTIFICATE_OIDC_ISSUER, - ] + args: list[str | os.PathLike] = ["verify"] + if self.staging: + args.append("--staging") + args.extend( + [ + "--signature", + materials.signature, + "--certificate", + materials.certificate, + "--certificate-identity", + CERTIFICATE_IDENTITY, + "--certificate-oidc-issuer", + CERTIFICATE_OIDC_ISSUER, + ] + ) if getattr(materials, "trusted_root", None) is not None: args.extend(["--trusted-root", materials.trusted_root]) @@ -270,16 +280,19 @@ def _verify_for_bundle(self, materials: BundleMaterials, artifact: os.PathLike) This is an overload of `verify` for the bundle flow and should not be called directly. """ - - args = [ - "verify-bundle", - "--bundle", - materials.bundle, - "--certificate-identity", - CERTIFICATE_IDENTITY, - "--certificate-oidc-issuer", - CERTIFICATE_OIDC_ISSUER, - ] + args: list[str | os.PathLike] = ["verify-bundle"] + if self.staging: + args.append("--staging") + args.extend( + [ + "--bundle", + materials.bundle, + "--certificate-identity", + CERTIFICATE_IDENTITY, + "--certificate-oidc-issuer", + CERTIFICATE_OIDC_ISSUER, + ] + ) if getattr(materials, "trusted_root", None) is not None: args.extend(["--trusted-root", materials.trusted_root])