From cbd57deb115c11e99ef1ac3c4ca74c6301805e3c Mon Sep 17 00:00:00 2001 From: Meredith Lancaster Date: Mon, 4 Mar 2024 11:05:06 -0700 Subject: [PATCH] add download cmd Signed-off-by: Meredith Lancaster --- pkg/cmd/attestation/attestation.go | 4 +- pkg/cmd/attestation/download/download.go | 165 +++++++++++++++++ pkg/cmd/attestation/download/download_test.go | 173 ++++++++++++++++++ pkg/cmd/attestation/download/options.go | 55 ++++++ pkg/cmd/attestation/download/options_test.go | 53 ++++++ 5 files changed, 449 insertions(+), 1 deletion(-) create mode 100644 pkg/cmd/attestation/download/download.go create mode 100644 pkg/cmd/attestation/download/download_test.go create mode 100644 pkg/cmd/attestation/download/options.go create mode 100644 pkg/cmd/attestation/download/options_test.go diff --git a/pkg/cmd/attestation/attestation.go b/pkg/cmd/attestation/attestation.go index a97026956..a57c46c69 100644 --- a/pkg/cmd/attestation/attestation.go +++ b/pkg/cmd/attestation/attestation.go @@ -1,6 +1,7 @@ package attestation import ( + "github.com/cli/cli/v2/pkg/cmd/attestation/download" "github.com/cli/cli/v2/pkg/cmd/attestation/verify" "github.com/cli/cli/v2/pkg/cmdutil" @@ -11,7 +12,7 @@ import ( func NewCmdAttestation(f *cmdutil.Factory) *cobra.Command { root := &cobra.Command{ Use: "attestation [subcommand]", - Short: "Work with attestations.", + Short: "Work with attestations", Aliases: []string{"at"}, Long: heredoc.Docf(` Work with attestations that represent trusted metadata about artifacts and images. @@ -25,6 +26,7 @@ func NewCmdAttestation(f *cmdutil.Factory) *cobra.Command { `, "`"), } + root.AddCommand(download.NewDownloadCmd(f)) root.AddCommand(verify.NewVerifyCmd(f)) return root diff --git a/pkg/cmd/attestation/download/download.go b/pkg/cmd/attestation/download/download.go new file mode 100644 index 000000000..830610c43 --- /dev/null +++ b/pkg/cmd/attestation/download/download.go @@ -0,0 +1,165 @@ +package download + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/cli/cli/v2/pkg/cmdutil" + "github.com/cli/cli/v2/pkg/cmd/attestation/artifact" + "github.com/cli/cli/v2/pkg/cmd/attestation/api" + + "github.com/MakeNowJust/heredoc" + "github.com/spf13/cobra" +) + +func NewDownloadCmd(f *cmdutil.Factory) *cobra.Command { + opts := &Options{} + downloadCmd := &cobra.Command{ + Use: "download [ | oci://]", + Args: cobra.ExactArgs(1), + Short: "Download trusted metadata about a binary artifact for offline use", + Long: heredoc.Docf(` + Download trusted metadata about a binary artifact for offline use. + + The command accepts either: + * a relative path to a local artifact + * a container image URI (e.g. oci://) + + Note that you must already be authenticated with a container registry + if you provide an OCI image URI as the artifact. + + The command also requires you provide either the %[1]s--owner%[1]s or %[1]s--repo%[1]s flag. + The value of the %[1]s--owner%[1]s flag should be the name of the GitHub organization + that the artifact is associated with. + The value of the %[1]s--repo%[1]s flag should be the name of the GitHub repository + that the artifact is associated with. + + Metadata is written to a file in the current directory named after the artifact's digest. + For example, if the artifact's digest is "sha256:1234", the metadata will be + written to "sha256:1234.jsonl". + `, "`"), + Example: heredoc.Doc(` + # Download trusted metadata for a local artifact associated with a GitHub organization + $ gh attestation download -o + + # Download trusted metadata for a local artifact associated with a GitHub repository + $ gh attestation download -R + + # Download trusted metadata for an OCI image associated with a GitHub organization + $ gh attestation download oci:// -o + `), + // PreRunE is used to validate flags before the command is run + // If an error is returned, its message will be printed to the terminal + // along with information about how use the command + PreRunE: func(cmd *cobra.Command, args []string) error { + opts.APIClient = api.NewLiveClient() + + // Create a logger for use throughout the download command + opts.ConfigureLogger() + + // Configure the live OCI client + opts.ConfigureOCIClient() + + // set the artifact path + opts.ArtifactPath = args[0] + + // check that the provided flags are valid + if err := opts.AreFlagsValid(); err != nil { + return err + } + return nil + }, + // Use Run instead of RunE because if an error is returned by RunVerify + // when RunE is used, the command usage will be printed + // We only want to print the error, not usage + Run: func(cmd *cobra.Command, args []string) { + if err := RunDownload(opts); err != nil { + opts.Logger.Println(opts.Logger.IO.Out, opts.Logger.ColorScheme.Redf("Failed to download the artifact's trusted metadata: %s", err.Error())) + os.Exit(1) + } + }, + } + + downloadCmd.Flags().StringVarP(&opts.Owner, "owner", "o", "", "a GitHub organization to scope attestation lookup by") + downloadCmd.MarkFlagRequired("owner") //nolint:errcheck + downloadCmd.Flags().StringVarP(&opts.DigestAlgorithm, "digest-alg", "d", "sha256", "The algorithm used to compute a digest of the artifact (sha256 or sha512)") + downloadCmd.Flags().IntVarP(&opts.Limit, "limit", "L", api.DefaultLimit, "Maximum number of attestations to fetch") + downloadCmd.Flags().BoolVarP(&opts.Verbose, "verbose", "v", false, "If set to true, the CLI will output verbose information.") + + return downloadCmd +} + + +func RunDownload(opts *Options) error { + artifact, err := artifact.NewDigestedArtifact(opts.OCIClient, opts.ArtifactPath, opts.DigestAlgorithm) + if err != nil { + return fmt.Errorf("failed to digest artifact: %w", err) + } + + opts.Logger.VerbosePrintf("Downloading trusted metadata for artifact %s\n\n", opts.ArtifactPath) + attestations, err := opts.APIClient.GetByOwnerAndDigest(opts.Owner, artifact.DigestWithAlg(), opts.Limit) + if err != nil { + return fmt.Errorf("failed to fetch attestations: %w", err) + } + + if attestations == nil { + fmt.Fprintf(opts.Logger.IO.Out, "No attestations found for %s\n", opts.ArtifactPath) + return nil + } + + filePath := createJSONLinesFilePath(artifact.DigestWithAlg(), opts.OutputPath) + fmt.Fprintf(opts.Logger.IO.Out, "Writing attestations to file %s.\nAny previous content will be overwritten\n\n", filePath) + + metadataFilePath, err := createMetadataFile(attestations, filePath) + if err != nil { + return fmt.Errorf("failed to write attestation: %w", err) + } + + fmt.Fprint(opts.Logger.IO.Out, + opts.Logger.ColorScheme.Greenf( + "The trusted metadata is now available at %s\n", metadataFilePath, + ), + ) + + return nil +} + +func createJSONLinesFilePath(artifact, outputPath string) string { + path := fmt.Sprintf("%s.jsonl", artifact) + if outputPath != "" { + return fmt.Sprintf("%s/%s", outputPath, path) + } + return path +} + +func createMetadataFile(attestationsResp []*api.Attestation, filePath string) (string, error) { + f, err := os.Create(filePath) + if err != nil { + return "", fmt.Errorf("failed to create trusted metadata file: %w", err) + } + + for _, resp := range attestationsResp { + bundle := resp.Bundle + attBytes, err := json.Marshal(bundle) + if err != nil { + return "", fmt.Errorf("failed to marshall attestation to JSON: %w", err) + } + + withNewline := fmt.Sprintf("%s\n", attBytes) + _, err = f.Write([]byte(withNewline)) + if err != nil { + if err = f.Close(); err != nil { + return "", fmt.Errorf("failed to close file while handling write error: %w", err) + } + + return "", fmt.Errorf("failed to write trusted metadata: %w", err) + } + } + + if err = f.Close(); err != nil { + return "", fmt.Errorf("failed ot close file after writing metadata: %w", err) + } + + return filePath, nil +} diff --git a/pkg/cmd/attestation/download/download_test.go b/pkg/cmd/attestation/download/download_test.go new file mode 100644 index 000000000..73f59f731 --- /dev/null +++ b/pkg/cmd/attestation/download/download_test.go @@ -0,0 +1,173 @@ +package download + +import ( + "bufio" + "fmt" + "os" + "testing" + + "github.com/cli/cli/v2/pkg/cmd/attestation/api" + "github.com/cli/cli/v2/pkg/cmd/attestation/artifact" + "github.com/cli/cli/v2/pkg/cmd/attestation/artifact/oci" + "github.com/cli/cli/v2/pkg/cmd/attestation/logger" + "github.com/cli/cli/v2/pkg/cmd/attestation/test" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRunDownload(t *testing.T) { + res := test.SuppressAndRestoreOutput() + defer res() + + tempDir, err := os.MkdirTemp("", "gh-attestation-test") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + baseOpts := Options{ + ArtifactPath: "../test/data/sigstore-js-2.1.0.tgz", + APIClient: api.NewTestClient(), + OCIClient: oci.NewMockClient(), + DigestAlgorithm: "sha512", + OutputPath: tempDir, + Limit: 30, + Logger: logger.NewDefaultLogger(), + } + + t.Run("fetch and store attestations successfully", func(t *testing.T) { + err = RunDownload(&baseOpts) + assert.NoError(t, err) + + artifact, err := artifact.NewDigestedArtifact(baseOpts.OCIClient, baseOpts.ArtifactPath, baseOpts.DigestAlgorithm) + require.NoError(t, err) + + assert.FileExists(t, fmt.Sprintf("%s/%s.jsonl", tempDir, artifact.DigestWithAlg())) + + actualLineCount, err := countLines(fmt.Sprintf("%s/%s.jsonl", tempDir, artifact.DigestWithAlg())) + require.NoError(t, err) + + expectedLineCount := 2 + assert.Equal(t, expectedLineCount, actualLineCount) + + err = os.Remove(fmt.Sprintf("%s/%s.jsonl", tempDir, artifact.DigestWithAlg())) + require.NoError(t, err) + }) + + t.Run("download OCI image attestations successfully", func(t *testing.T) { + opts := baseOpts + opts.ArtifactPath = "oci://ghcr.io/github/test" + + err = RunDownload(&opts) + assert.NoError(t, err) + + artifact, err := artifact.NewDigestedArtifact(opts.OCIClient, opts.ArtifactPath, opts.DigestAlgorithm) + require.NoError(t, err) + + assert.FileExists(t, fmt.Sprintf("%s/%s.jsonl", tempDir, artifact.DigestWithAlg())) + + actualLineCount, err := countLines(fmt.Sprintf("%s/%s.jsonl", tempDir, artifact.DigestWithAlg())) + require.NoError(t, err) + + expectedLineCount := 2 + assert.Equal(t, expectedLineCount, actualLineCount) + + err = os.Remove(fmt.Sprintf("%s/%s.jsonl", tempDir, artifact.DigestWithAlg())) + require.NoError(t, err) + }) + + t.Run("cannot find artifact", func(t *testing.T) { + opts := baseOpts + opts.ArtifactPath = "../test/data/not-real.zip" + + err := RunDownload(&opts) + assert.Error(t, err) + }) + + t.Run("no attestations found", func(t *testing.T) { + opts := baseOpts + opts.APIClient = api.MockClient{ + OnGetByOwnerAndDigest: func(repo, digest string, limit int) ([]*api.Attestation, error) { + return nil, nil + }, + } + + err := RunDownload(&opts) + assert.NoError(t, err) + + artifact, err := artifact.NewDigestedArtifact(opts.OCIClient, opts.ArtifactPath, opts.DigestAlgorithm) + require.NoError(t, err) + assert.NoFileExists(t, artifact.DigestWithAlg()) + }) + + t.Run("cannot download OCI artifact", func(t *testing.T) { + opts := baseOpts + opts.ArtifactPath = "oci://ghcr.io/github/test" + opts.OCIClient = oci.NewReferenceFailClient() + + err := RunDownload(&opts) + assert.Error(t, err) + assert.ErrorContains(t, err, "failed to digest artifact") + }) +} + +func TestCreateJSONLinesFilePath(t *testing.T) { + tempDir, err := os.MkdirTemp("", "gh-attestation-test") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + t.Run("with output path", func(t *testing.T) { + artifact, err := artifact.NewDigestedArtifact(oci.NewMockClient(), "../test/data/sigstore-js-2.1.0.tgz", "sha512") + require.NoError(t, err) + path := createJSONLinesFilePath(artifact.DigestWithAlg(), tempDir) + + expectedPath := fmt.Sprintf("%s/%s.jsonl", tempDir, artifact.DigestWithAlg()) + assert.Equal(t, expectedPath, path) + }) + + t.Run("with nested output path", func(t *testing.T) { + artifact, err := artifact.NewDigestedArtifact(oci.NewMockClient(), "../test/data/sigstore-js-2.1.0.tgz", "sha512") + require.NoError(t, err) + + nestedPath := fmt.Sprintf("%s/subdir", tempDir) + path := createJSONLinesFilePath(artifact.DigestWithAlg(), nestedPath) + + expectedPath := fmt.Sprintf("%s/subdir/%s.jsonl", tempDir, artifact.DigestWithAlg()) + assert.Equal(t, expectedPath, path) + }) + + t.Run("with output path with beginning slash", func(t *testing.T) { + artifact, err := artifact.NewDigestedArtifact(oci.NewMockClient(), "../test/data/sigstore-js-2.1.0.tgz", "sha512") + require.NoError(t, err) + + nestedPath := fmt.Sprintf("/%s/subdir", tempDir) + path := createJSONLinesFilePath(artifact.DigestWithAlg(), nestedPath) + + expectedPath := fmt.Sprintf("/%s/subdir/%s.jsonl", tempDir, artifact.DigestWithAlg()) + assert.Equal(t, expectedPath, path) + }) + + t.Run("without output path", func(t *testing.T) { + artifact, err := artifact.NewDigestedArtifact(oci.NewMockClient(), "../test/data/sigstore-js-2.1.0.tgz", "sha512") + require.NoError(t, err) + path := createJSONLinesFilePath(artifact.DigestWithAlg(), "") + + expectedPath := fmt.Sprintf("%s.jsonl", artifact.DigestWithAlg()) + assert.Equal(t, expectedPath, path) + }) +} + +func countLines(path string) (int, error) { + f, err := os.Open(path) + if err != nil { + return 0, err + } + defer f.Close() + + counter := 0 + scanner := bufio.NewScanner(f) + for scanner.Scan() { + counter += 1 + } + + return counter, nil +} diff --git a/pkg/cmd/attestation/download/options.go b/pkg/cmd/attestation/download/options.go new file mode 100644 index 000000000..216558540 --- /dev/null +++ b/pkg/cmd/attestation/download/options.go @@ -0,0 +1,55 @@ +package download + +import ( + "fmt" + + "github.com/cli/cli/v2/pkg/cmd/attestation/api" + "github.com/cli/cli/v2/pkg/cmd/attestation/artifact/digest" + "github.com/cli/cli/v2/pkg/cmd/attestation/artifact/oci" + "github.com/cli/cli/v2/pkg/cmd/attestation/logger" +) + +type Options struct { + ArtifactPath string + DigestAlgorithm string + APIClient api.Client + Logger *logger.Logger + Limit int + OCIClient oci.Client + OutputPath string + Owner string + Verbose bool +} + +// ConfigureLogger configures a logger using configuration provided +// through the options +func (opts *Options) ConfigureLogger() { + opts.Logger = logger.NewLogger(false, opts.Verbose) +} + +// ConfigureOCIClient configures an OCI client +func (opts *Options) ConfigureOCIClient() { + opts.OCIClient = oci.NewLiveClient() +} + +func (opts *Options) AreFlagsValid() error { + if opts.Owner == "" { + return fmt.Errorf("owner must be provided") + } + + // DigestAlgorithm must not be empty + if opts.DigestAlgorithm == "" { + return fmt.Errorf("digest-alg cannot be empty") + } + + if !digest.IsValidDigestAlgorithm(opts.DigestAlgorithm) { + return fmt.Errorf("invalid digest algorithm '%s' provided in digest-alg", opts.DigestAlgorithm) + } + + // Check that limit is between 1 and 1000 + if opts.Limit < 1 || opts.Limit > 1000 { + return fmt.Errorf("limit %d not allowed, must be between 1 and 1000", opts.Limit) + } + + return nil +} diff --git a/pkg/cmd/attestation/download/options_test.go b/pkg/cmd/attestation/download/options_test.go new file mode 100644 index 000000000..84d2cd2a2 --- /dev/null +++ b/pkg/cmd/attestation/download/options_test.go @@ -0,0 +1,53 @@ +package download + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestAreFlagsValid(t *testing.T) { + t.Run("missing Owner", func(t *testing.T) { + opts := Options{ + DigestAlgorithm: "sha512", + } + + err := opts.AreFlagsValid() + assert.Error(t, err) + assert.ErrorContains(t, err, "owner must be provided") + }) + + t.Run("missing DigestAlgorithm", func(t *testing.T) { + opts := Options{ + Owner: "github", + } + + err := opts.AreFlagsValid() + assert.Error(t, err) + assert.ErrorContains(t, err, "digest-alg cannot be empty") + }) + + t.Run("Limit is too low", func(t *testing.T) { + opts := Options{ + DigestAlgorithm: "sha512", + Limit: 0, + Owner: "github", + } + + err := opts.AreFlagsValid() + assert.Error(t, err) + assert.ErrorContains(t, err, "limit 0 not allowed, must be between 1 and 1000") + }) + + t.Run("Limit is too high", func(t *testing.T) { + opts := Options{ + DigestAlgorithm: "sha512", + Limit: 1001, + Owner: "github", + } + + err := opts.AreFlagsValid() + assert.Error(t, err) + assert.ErrorContains(t, err, "limit 1001 not allowed, must be between 1 and 1000") + }) +}