diff --git a/pkg/cmd/run/download/download.go b/pkg/cmd/run/download/download.go new file mode 100644 index 000000000..15dafa6e8 --- /dev/null +++ b/pkg/cmd/run/download/download.go @@ -0,0 +1,129 @@ +package download + +import ( + "errors" + "fmt" + "path/filepath" + + "github.com/MakeNowJust/heredoc" + "github.com/cli/cli/pkg/cmdutil" + "github.com/cli/cli/pkg/iostreams" + "github.com/spf13/cobra" +) + +type DownloadOptions struct { + IO *iostreams.IOStreams + Platform platform + + RunID string + DestinationDir string + FilePatterns []string +} + +type platform interface { + List(runID string) ([]Artifact, error) + Download(url string, dir string) error +} + +func NewCmdDownload(f *cmdutil.Factory, runF func(*DownloadOptions) error) *cobra.Command { + opts := &DownloadOptions{ + IO: f.IOStreams, + } + + cmd := &cobra.Command{ + Use: "download []", + Short: "Download artifacts generated by a workflow run", + Args: cobra.MaximumNArgs(1), + Example: heredoc.Doc(` + # Download all artifacts generated by a workflow run + $ gh run download + + # Download a specific artifact within a run + $ gh run download -p + + # Download specific artifacts across all runs in a repository + $ gh run download -p -p + `), + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) > 0 { + opts.RunID = args[0] + } else if len(opts.FilePatterns) == 0 { + return &cmdutil.FlagError{Err: errors.New("either run ID or `--pattern` is required")} + } + + // support `-R, --repo` override + baseRepo, err := f.BaseRepo() + if err != nil { + return err + } + httpClient, err := f.HttpClient() + if err != nil { + return err + } + opts.Platform = &apiPlatform{ + client: httpClient, + repo: baseRepo, + } + + if runF != nil { + return runF(opts) + } + return runDownload(opts) + }, + } + + cmd.Flags().StringVarP(&opts.DestinationDir, "dir", "D", ".", "The directory to download artifacts into") + cmd.Flags().StringArrayVarP(&opts.FilePatterns, "pattern", "p", nil, "Download only artifacts that match a glob pattern") + + return cmd +} + +func runDownload(opts *DownloadOptions) error { + opts.IO.StartProgressIndicator() + defer opts.IO.StopProgressIndicator() + + artifacts, err := opts.Platform.List(opts.RunID) + if err != nil { + return fmt.Errorf("error fetching artifacts: %w", err) + } + + // track downloaded artifacts and avoid re-downloading any of the same name + downloaded := map[string]struct{}{} + numArtifacts := 0 + + for _, a := range artifacts { + if a.Expired { + continue + } + numArtifacts++ + if _, found := downloaded[a.Name]; found { + continue + } + if len(opts.FilePatterns) > 0 && !matchAny(opts.FilePatterns, a.Name) { + continue + } + err := opts.Platform.Download(a.DownloadURL, opts.DestinationDir) + if err != nil { + return fmt.Errorf("error downloading %s: %w", a.Name, err) + } + downloaded[a.Name] = struct{}{} + } + + if numArtifacts == 0 { + return errors.New("no valid artifacts found to download") + } + if len(downloaded) == 0 { + return errors.New("no artifact matches any of the patterns provided") + } + + return nil +} + +func matchAny(patterns []string, name string) bool { + for _, p := range patterns { + if isMatch, err := filepath.Match(p, name); err == nil && isMatch { + return true + } + } + return false +} diff --git a/pkg/cmd/run/download/download_test.go b/pkg/cmd/run/download/download_test.go new file mode 100644 index 000000000..15060f922 --- /dev/null +++ b/pkg/cmd/run/download/download_test.go @@ -0,0 +1,181 @@ +package download + +import ( + "bytes" + "io/ioutil" + "net/http" + "testing" + + "github.com/cli/cli/internal/ghrepo" + "github.com/cli/cli/pkg/cmdutil" + "github.com/cli/cli/pkg/iostreams" + "github.com/google/shlex" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_NewCmdDownload(t *testing.T) { + tests := []struct { + name string + args string + isTTY bool + want DownloadOptions + wantErr string + }{ + { + name: "empty", + args: "", + isTTY: true, + wantErr: "either run ID or `--pattern` is required", + }, + { + name: "with run ID", + args: "2345", + isTTY: true, + want: DownloadOptions{ + RunID: "2345", + FilePatterns: []string(nil), + DestinationDir: ".", + }, + }, + { + name: "to destination", + args: "2345 -D tmp/dest", + isTTY: true, + want: DownloadOptions{ + RunID: "2345", + FilePatterns: []string(nil), + DestinationDir: "tmp/dest", + }, + }, + { + name: "repo level with patterns", + args: "-p one -p two", + isTTY: true, + want: DownloadOptions{ + RunID: "", + FilePatterns: []string{"one", "two"}, + DestinationDir: ".", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + io, _, _, _ := iostreams.Test() + io.SetStdoutTTY(tt.isTTY) + io.SetStdinTTY(tt.isTTY) + io.SetStderrTTY(tt.isTTY) + + f := &cmdutil.Factory{ + IOStreams: io, + HttpClient: func() (*http.Client, error) { + return nil, nil + }, + BaseRepo: func() (ghrepo.Interface, error) { + return nil, nil + }, + } + + var opts *DownloadOptions + cmd := NewCmdDownload(f, func(o *DownloadOptions) error { + opts = o + return nil + }) + cmd.PersistentFlags().StringP("repo", "R", "", "") + + argv, err := shlex.Split(tt.args) + require.NoError(t, err) + cmd.SetArgs(argv) + + cmd.SetIn(&bytes.Buffer{}) + cmd.SetOut(ioutil.Discard) + cmd.SetErr(ioutil.Discard) + + _, err = cmd.ExecuteC() + if tt.wantErr != "" { + require.EqualError(t, err, tt.wantErr) + return + } else { + require.NoError(t, err) + } + + assert.Equal(t, tt.want.RunID, opts.RunID) + assert.Equal(t, tt.want.FilePatterns, opts.FilePatterns) + assert.Equal(t, tt.want.DestinationDir, opts.DestinationDir) + }) + } +} + +func Test_runDownload(t *testing.T) { + tests := []struct { + name string + opts DownloadOptions + artifacts []Artifact + wantDownloaded []string + wantErr string + }{ + { + name: "download non-expired", + opts: DownloadOptions{ + RunID: "2345", + DestinationDir: ".", + FilePatterns: []string(nil), + }, + artifacts: []Artifact{ + { + Name: "artifact-1", + DownloadURL: "http://download.com/artifact-1.zip", + Expired: false, + }, + { + Name: "expired-artifact", + DownloadURL: "http://download.com/expired.zip", + Expired: true, + }, + { + Name: "artifact-2", + DownloadURL: "http://download.com/artifact-2.zip", + Expired: false, + }, + }, + wantDownloaded: []string{ + "http://download.com/artifact-1.zip", + "http://download.com/artifact-2.zip", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + opts := &tt.opts + io, _, stdout, stderr := iostreams.Test() + opts.IO = io + platform := &stubPlatform{listResults: tt.artifacts} + opts.Platform = platform + + err := runDownload(opts) + if tt.wantErr != "" { + require.EqualError(t, err, tt.wantErr) + } else { + require.NoError(t, err) + } + + assert.Equal(t, tt.wantDownloaded, platform.downloaded) + assert.Equal(t, "", stdout.String()) + assert.Equal(t, "", stderr.String()) + }) + } +} + +type stubPlatform struct { + listResults []Artifact + downloaded []string +} + +func (p *stubPlatform) List(runID string) ([]Artifact, error) { + return p.listResults, nil +} + +func (p *stubPlatform) Download(url string, dir string) error { + p.downloaded = append(p.downloaded, url) + return nil +} diff --git a/pkg/cmd/run/download/http.go b/pkg/cmd/run/download/http.go new file mode 100644 index 000000000..e48cb176d --- /dev/null +++ b/pkg/cmd/run/download/http.go @@ -0,0 +1,109 @@ +package download + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + + "github.com/cli/cli/api" + "github.com/cli/cli/internal/ghinstance" + "github.com/cli/cli/internal/ghrepo" +) + +type Artifact struct { + Name string `json:"name"` + Size uint32 `json:"size_in_bytes"` + DownloadURL string `json:"archive_download_url"` + Expired bool `json:"expired"` +} + +type apiPlatform struct { + client *http.Client + repo ghrepo.Interface +} + +func (p *apiPlatform) List(runID string) ([]Artifact, error) { + return listArtifacts(p.client, p.repo, runID) +} + +func (p *apiPlatform) Download(url string, dir string) error { + return downloadArtifact(p.client, url, dir) +} + +func listArtifacts(httpClient *http.Client, repo ghrepo.Interface, runID string) ([]Artifact, error) { + perPage := 100 + path := fmt.Sprintf("repos/%s/%s/actions/artifacts?per_page=%d", repo.RepoOwner(), repo.RepoName(), perPage) + if runID != "" { + path = fmt.Sprintf("repos/%s/%s/actions/runs/%s/artifacts?per_page=%d", repo.RepoOwner(), repo.RepoName(), runID, perPage) + } + + req, err := http.NewRequest("GET", ghinstance.RESTPrefix(repo.RepoHost())+path, nil) + if err != nil { + return nil, err + } + + resp, err := httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode > 299 { + return nil, api.HandleHTTPError(resp) + } + + var response struct { + TotalCount uint16 `json:"total_count"` + Artifacts []Artifact + } + + dec := json.NewDecoder(resp.Body) + if err := dec.Decode(&response); err != nil { + return response.Artifacts, fmt.Errorf("error reading JSON: %w", err) + } + + return response.Artifacts, nil +} + +func downloadArtifact(httpClient *http.Client, url, destDir string) error { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return err + } + // The API rejects requesting the "application/octet-stream" type :( + req.Header.Set("Accept", "application/json") + + resp, err := httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode > 299 { + return api.HandleHTTPError(resp) + } + + tmpfile, err := ioutil.TempFile("", "gh-artifact.*.zip") + if err != nil { + return fmt.Errorf("error initializing temporary file: %w", err) + } + defer tmpfile.Close() + + _, err = io.Copy(tmpfile, resp.Body) + if err != nil { + return fmt.Errorf("error writing zip archive: %w", err) + } + + tmpfile.Close() + if err := extractZip(tmpfile, destDir); err != nil { + return fmt.Errorf("error extracting zip archive: %w", err) + } + if err := os.Remove(tmpfile.Name()); err != nil { + return err + } + + return nil +} diff --git a/pkg/cmd/run/download/zip.go b/pkg/cmd/run/download/zip.go new file mode 100644 index 000000000..962ca9b9d --- /dev/null +++ b/pkg/cmd/run/download/zip.go @@ -0,0 +1,61 @@ +package download + +import ( + "archive/zip" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" +) + +func extractZip(zipfile *os.File, destDir string) error { + zr, err := zip.OpenReader(zipfile.Name()) + if err != nil { + return err + } + defer zr.Close() + + for _, zf := range zr.File { + fpath := filepath.Join(destDir, zf.Name) + if err := extractZipFile(zf, fpath); err != nil { + return fmt.Errorf("error extracting %q: %w", zf.Name, err) + } + } + + return nil +} + +func extractZipFile(zf *zip.File, dest string) error { + zm := zf.Mode() + if zm.IsDir() { + return os.MkdirAll(dest, 0755) + } + + f, err := zf.Open() + if err != nil { + return err + } + defer f.Close() + + if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil { + return err + } + + var mode os.FileMode = 0644 + if isBinary(zm) { + mode = 0755 + } + df, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode) + if err != nil { + return err + } + defer df.Close() + + _, err = io.Copy(df, f) + return err +} + +func isBinary(m fs.FileMode) bool { + return m&0111 != 0 +} diff --git a/pkg/cmd/run/run.go b/pkg/cmd/run/run.go index a1620e3b4..716f35ea6 100644 --- a/pkg/cmd/run/run.go +++ b/pkg/cmd/run/run.go @@ -1,6 +1,7 @@ package run import ( + cmdDownload "github.com/cli/cli/pkg/cmd/run/download" cmdList "github.com/cli/cli/pkg/cmd/run/list" cmdView "github.com/cli/cli/pkg/cmd/run/view" "github.com/cli/cli/pkg/cmdutil" @@ -20,6 +21,7 @@ func NewCmdRun(f *cmdutil.Factory) *cobra.Command { cmd.AddCommand(cmdList.NewCmdList(f, nil)) cmd.AddCommand(cmdView.NewCmdView(f, nil)) + cmd.AddCommand(cmdDownload.NewCmdDownload(f, nil)) return cmd }