STACKITGit/cli
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #9838 from malancas/attestation-clean-up-tests

Browse source 
Clean up skipped online tests for `gh attestation verify`
This commit is contained in:
Meredith Lancaster 2024-10-30 13:28:44 -06:00 • committed by GitHub
commit bb5136007e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 135 additions and 213 deletions
Download patch file Download diff file Expand all files Collapse all files

90
pkg/cmd/attestation/verification/sigstore_integration_test.go
View file

@ -15,32 +15,54 @@ import (
)
func TestLiveSigstoreVerifier(t *testing.T) {
t.Run("with invalid signature", func(t *testing.T) {
attestations := getAttestationsFor(t, "../test/data/sigstoreBundle-invalid-signature.json")
require.NotNil(t, attestations)
type testcase struct {
name string
attestations []*api.Attestation
expectErr bool
errContains string
}
testcases := []testcase{
{
name: "with invalid signature",
attestations: getAttestationsFor(t, "../test/data/sigstoreBundle-invalid-signature.json"),
expectErr: true,
errContains: "verifying with issuer \"sigstore.dev\"",
},
{
name: "with valid artifact and JSON lines file containing multiple Sigstore bundles",
attestations: getAttestationsFor(t, "../test/data/sigstore-js-2.1.0_with_2_bundles.jsonl"),
},
{
name: "with invalid bundle version",
attestations: getAttestationsFor(t, "../test/data/sigstore-js-2.1.0-bundle-v0.1.json"),
expectErr: true,
errContains: "unsupported bundle version",
},
{
name: "with no attestations",
attestations: []*api.Attestation{},
expectErr: true,
errContains: "no attestations were verified",
},
}
for _, tc := range testcases {
verifier := NewLiveSigstoreVerifier(SigstoreConfig{
Logger: io.NewTestHandler(),
})
res := verifier.Verify(attestations, publicGoodPolicy(t))
require.Error(t, res.Error)
require.ErrorContains(t, res.Error, "verifying with issuer \"sigstore.dev\"")
require.Nil(t, res.VerifyResults)
})
res := verifier.Verify(tc.attestations, publicGoodPolicy(t))
t.Run("with valid artifact and JSON lines file containing multiple Sigstore bundles", func(t *testing.T) {
attestations := getAttestationsFor(t, "../test/data/sigstore-js-2.1.0_with_2_bundles.jsonl")
require.Len(t, attestations, 2)
verifier := NewLiveSigstoreVerifier(SigstoreConfig{
Logger: io.NewTestHandler(),
})
res := verifier.Verify(attestations, publicGoodPolicy(t))
require.Len(t, res.VerifyResults, 2)
require.NoError(t, res.Error)
})
if tc.expectErr {
require.Error(t, res.Error, "test case: %s", tc.name)
require.ErrorContains(t, res.Error, tc.errContains, "test case: %s", tc.name)
require.Nil(t, res.VerifyResults, "test case: %s", tc.name)
} else {
require.Equal(t, len(tc.attestations), len(res.VerifyResults), "test case: %s", tc.name)
require.NoError(t, res.Error, "test case: %s", tc.name)
}
}
t.Run("with GitHub Sigstore artifact", func(t *testing.T) {
githubArtifactPath := test.NormalizeRelativePath("../test/data/github_provenance_demo-0.0.12-py3-none-any.whl")
@ -72,34 +94,6 @@ func TestLiveSigstoreVerifier(t *testing.T) {
require.Len(t, res.VerifyResults, 2)
require.NoError(t, res.Error)
})
t.Run("with invalid bundle version", func(t *testing.T) {
attestations := getAttestationsFor(t, "../test/data/sigstore-js-2.1.0-bundle-v0.1.json")
require.Len(t, attestations, 1)
verifier := NewLiveSigstoreVerifier(SigstoreConfig{
Logger: io.NewTestHandler(),
})
res := verifier.Verify(attestations, publicGoodPolicy(t))
require.Len(t, res.VerifyResults, 0)
require.ErrorContains(t, res.Error, "unsupported bundle version")
})
t.Run("with no attestations", func(t *testing.T) {
attestations := []*api.Attestation{}
require.Len(t, attestations, 0)
verifier := NewLiveSigstoreVerifier(SigstoreConfig{
Logger: io.NewTestHandler(),
TrustedRoot: test.NormalizeRelativePath("../test/data/trusted_root.json"),
})
res := verifier.Verify(attestations, publicGoodPolicy(t))
require.Len(t, res.VerifyResults, 0)
require.NotNil(t, res.Error)
})
}
func publicGoodPolicy(t *testing.T) verify.PolicyBuilder {

112
pkg/cmd/attestation/verify/options_test.go
View file

@ -13,29 +13,28 @@ var (
publicGoodBundlePath = test.NormalizeRelativePath("../test/data/psigstore-js-2.1.0-bundle.json")
)
var baseOptions = Options{
ArtifactPath: publicGoodArtifactPath,
BundlePath: publicGoodBundlePath,
DigestAlgorithm: "sha512",
Limit: 1,
Owner: "sigstore",
OIDCIssuer: "some issuer",
}
func TestAreFlagsValid(t *testing.T) {
t.Run("has invalid Repo value", func(t *testing.T) {
opts := Options{
ArtifactPath: publicGoodArtifactPath,
DigestAlgorithm: "sha512",
OIDCIssuer: "some issuer",
Repo: "sigstoresigstore-js",
}
opts := baseOptions
opts.Repo = "sigstoresigstore-js"
err := opts.AreFlagsValid()
require.Error(t, err)
require.ErrorContains(t, err, "invalid value provided for repo")
})
t.Run("invalid limit < 0", func(t *testing.T) {
opts := Options{
ArtifactPath: publicGoodArtifactPath,
BundlePath: publicGoodBundlePath,
DigestAlgorithm: "sha512",
Owner: "sigstore",
OIDCIssuer: "some issuer",
Limit: 0,
}
t.Run("invalid limit == 0", func(t *testing.T) {
opts := baseOptions
opts.Limit = 0
err := opts.AreFlagsValid()
require.Error(t, err)
@ -43,19 +42,43 @@ func TestAreFlagsValid(t *testing.T) {
})
t.Run("invalid limit > 1000", func(t *testing.T) {
opts := Options{
ArtifactPath: publicGoodArtifactPath,
BundlePath: publicGoodBundlePath,
DigestAlgorithm: "sha512",
Owner: "sigstore",
OIDCIssuer: "some issuer",
Limit: 1001,
}
opts := baseOptions
opts.Limit = 1001
err := opts.AreFlagsValid()
require.Error(t, err)
require.ErrorContains(t, err, "limit 1001 not allowed, must be between 1 and 1000")
})
t.Run("returns error when UseBundleFromRegistry is true and ArtifactPath is not an OCI path", func(t *testing.T) {
opts := baseOptions
opts.BundlePath = ""
opts.UseBundleFromRegistry = true
err := opts.AreFlagsValid()
require.Error(t, err)
require.ErrorContains(t, err, "bundle-from-oci flag can only be used with OCI artifact paths")
})
t.Run("does not return error when UseBundleFromRegistry is true and ArtifactPath is an OCI path", func(t *testing.T) {
opts := baseOptions
opts.ArtifactPath = "oci://sigstore/sigstore-js:2.1.0"
opts.BundlePath = ""
opts.UseBundleFromRegistry = true
err := opts.AreFlagsValid()
require.NoError(t, err)
})
t.Run("returns error when UseBundleFromRegistry is true and BundlePath is provided", func(t *testing.T) {
opts := baseOptions
opts.ArtifactPath = "oci://sigstore/sigstore-js:2.1.0"
opts.UseBundleFromRegistry = true
err := opts.AreFlagsValid()
require.Error(t, err)
require.ErrorContains(t, err, "bundle-from-oci flag cannot be used with bundle-path flag")
})
}
func TestSetPolicyFlags(t *testing.T) {
@ -116,47 +139,4 @@ func TestSetPolicyFlags(t *testing.T) {
require.Equal(t, "sigstore", opts.Owner)
require.Equal(t, "^https://github/foo", opts.SANRegex)
})
t.Run("returns error when UseBundleFromRegistry is true and ArtifactPath is not an OCI path", func(t *testing.T) {
opts := Options{
ArtifactPath: publicGoodArtifactPath,
DigestAlgorithm: "sha512",
Owner: "sigstore",
UseBundleFromRegistry: true,
Limit: 1,
}
err := opts.AreFlagsValid()
require.Error(t, err)
require.ErrorContains(t, err, "bundle-from-oci flag can only be used with OCI artifact paths")
})
t.Run("does not return error when UseBundleFromRegistry is true and ArtifactPath is an OCI path", func(t *testing.T) {
opts := Options{
ArtifactPath: "oci://sigstore/sigstore-js:2.1.0",
DigestAlgorithm: "sha512",
OIDCIssuer: "some issuer",
Owner: "sigstore",
UseBundleFromRegistry: true,
Limit: 1,
}
err := opts.AreFlagsValid()
require.NoError(t, err)
})
t.Run("returns error when UseBundleFromRegistry is true and BundlePath is provided", func(t *testing.T) {
opts := Options{
ArtifactPath: "oci://sigstore/sigstore-js:2.1.0",
BundlePath: publicGoodBundlePath,
DigestAlgorithm: "sha512",
Owner: "sigstore",
UseBundleFromRegistry: true,
Limit: 1,
}
err := opts.AreFlagsValid()
require.Error(t, err)
require.ErrorContains(t, err, "bundle-from-oci flag cannot be used with bundle-path flag")
})
}

50
pkg/cmd/attestation/verify/policy_test.go
View file

@ -36,18 +36,28 @@ func TestValidateSignerWorkflow(t *testing.T) {
providedSignerWorkflow string
expectedWorkflowRegex string
host string
expectErr bool
errContains string
}
testcases := []testcase{
{
name: "workflow with no host specified",
providedSignerWorkflow: "github/artifact-attestations-workflows/.github/workflows/attest.yml",
expectedWorkflowRegex: "^https://github.com/github/artifact-attestations-workflows/.github/workflows/attest.yml",
expectErr: true,
errContains: "unknown host",
},
{
name: "workflow with host specified",
name: "workflow with default host",
providedSignerWorkflow: "github/artifact-attestations-workflows/.github/workflows/attest.yml",
expectedWorkflowRegex: "^https://github.com/github/artifact-attestations-workflows/.github/workflows/attest.yml",
host: "github.com",
},
{
name: "workflow with workflow URL included",
providedSignerWorkflow: "github.com/github/artifact-attestations-workflows/.github/workflows/attest.yml",
expectedWorkflowRegex: "^https://github.com/github/artifact-attestations-workflows/.github/workflows/attest.yml",
host: "github.com",
},
{
name: "workflow with GH_HOST set",
@ -61,45 +71,25 @@ func TestValidateSignerWorkflow(t *testing.T) {
expectedWorkflowRegex: "^https://authedhost.github.com/github/artifact-attestations-workflows/.github/workflows/attest.yml",
host: "authedhost.github.com",
},
{
name: "workflow with authenticated host",
providedSignerWorkflow: "github/artifact-attestations-workflows/.github/workflows/attest.yml",
expectedWorkflowRegex: "^https://authedhost.github.com/github/artifact-attestations-workflows/.github/workflows/attest.yml",
host: "authedhost.github.com",
},
}
for _, tc := range testcases {
cmdFactory := factory.New("test")
opts := &Options{
Config: cmdFactory.Config,
Config: factory.New("test").Config,
SignerWorkflow: tc.providedSignerWorkflow,
}
// All host resolution is done verify.go:RunE
if tc.host == "" {
// Set to default host
tc.host = "github.com"
}
opts.Hostname = tc.host
workflowRegex, err := validateSignerWorkflow(opts)
require.NoError(t, err)
require.Equal(t, tc.expectedWorkflowRegex, workflowRegex)
if tc.expectErr {
require.Error(t, err)
require.ErrorContains(t, err, tc.errContains)
} else {
require.NoError(t, err)
require.Equal(t, tc.expectedWorkflowRegex, workflowRegex)
}
}
}
func TestValidateSignerWorkflowNoHost(t *testing.T) {
cmdFactory := factory.New("test")
opts := &Options{
Config: cmdFactory.Config,
SignerWorkflow: "github/artifact-attestations-workflows/.github/workflows/attest.yml",
}
workflowRegex, err := validateSignerWorkflow(opts)
require.Error(t, err)
require.ErrorContains(t, err, "unknown host")
require.Equal(t, "", workflowRegex)
}

27
pkg/cmd/attestation/verify/verify_integration_test.go
View file

@ -83,6 +83,33 @@ func TestVerifyIntegration(t *testing.T) {
require.Error(t, err)
require.ErrorContains(t, err, "expected SourceRepositoryURI to be https://github.com/fakeowner/fakerepo, got https://github.com/sigstore/sigstore-js")
})
t.Run("with no matching OIDC issuer", func(t *testing.T) {
opts := publicGoodOpts
opts.OIDCIssuer = "some-other-issuer"
err := runVerify(&opts)
require.Error(t, err)
require.ErrorContains(t, err, "expected Issuer to be some-other-issuer, got https://token.actions.githubusercontent.com")
})
t.Run("with invalid SAN", func(t *testing.T) {
opts := publicGoodOpts
opts.SAN = "fake san"
err := runVerify(&opts)
require.Error(t, err)
require.ErrorContains(t, err, "verifying with issuer \"sigstore.dev\"")
})
t.Run("with invalid SAN regex", func(t *testing.T) {
opts := publicGoodOpts
opts.SANRegex = "^https://github.com/sigstore/not-real/"
err := runVerify(&opts)
require.Error(t, err)
require.ErrorContains(t, err, "verifying with issuer \"sigstore.dev\"")
})
}
func TestVerifyIntegrationCustomIssuer(t *testing.T) {

69
pkg/cmd/attestation/verify/verify_test.go
View file

@ -453,75 +453,6 @@ func TestRunVerify(t *testing.T) {
require.ErrorContains(t, err, "failed to fetch attestations from wrong-owner")
})
// TODO: this test can only be tested with a live SigstoreVerifier
// add integration tests or HTTP mocked sigstore verifier tests
// to test this case
t.Run("with invalid OIDC issuer", func(t *testing.T) {
t.Skip()
opts := publicGoodOpts
opts.OIDCIssuer = "not-a-real-issuer"
require.Error(t, runVerify(&opts))
})
// TODO: this test can only be tested with a live SigstoreVerifier
// add integration tests or HTTP mocked sigstore verifier tests
// to test this case
t.Run("with SAN enforcement", func(t *testing.T) {
t.Skip()
opts := Options{
ArtifactPath: artifactPath,
BundlePath: bundlePath,
APIClient: api.NewTestClient(),
DigestAlgorithm: "sha512",
Logger: logger,
OIDCIssuer: verification.GitHubOIDCIssuer,
Owner: "sigstore",
SAN: SigstoreSanValue,
SigstoreVerifier: verification.NewMockSigstoreVerifier(t),
}
require.Nil(t, runVerify(&opts))
})
// TODO: this test can only be tested with a live SigstoreVerifier
// add integration tests or HTTP mocked sigstore verifier tests
// to test this case
t.Run("with invalid SAN", func(t *testing.T) {
t.Skip()
opts := publicGoodOpts
opts.SAN = "fake san"
require.Error(t, runVerify(&opts))
})
// TODO: this test can only be tested with a live SigstoreVerifier
// add integration tests or HTTP mocked sigstore verifier tests
// to test this case
t.Run("with SAN regex enforcement", func(t *testing.T) {
t.Skip()
opts := publicGoodOpts
opts.SANRegex = SigstoreSanRegex
require.Nil(t, runVerify(&opts))
})
// TODO: this test can only be tested with a live SigstoreVerifier
// add integration tests or HTTP mocked sigstore verifier tests
// to test this case
t.Run("with invalid SAN regex", func(t *testing.T) {
t.Skip()
opts := publicGoodOpts
opts.SANRegex = "^https://github.com/sigstore/not-real/"
require.Error(t, runVerify(&opts))
})
// TODO: this test can only be tested with a live SigstoreVerifier
// add integration tests or HTTP mocked sigstore verifier tests
// to test this case
t.Run("with no matching OIDC issuer", func(t *testing.T) {
t.Skip()
opts := publicGoodOpts
opts.OIDCIssuer = "some-other-issuer"
require.Error(t, runVerify(&opts))
})
t.Run("with missing API client", func(t *testing.T) {
customOpts := publicGoodOpts
customOpts.APIClient = nil