Add run download command for downloading workflow artifacts
This commit is contained in:
Mislav Marohnić
parent
a35d451b67
commit
c54e3c9ca8
5 changed files with 482 additions and 0 deletions
129
pkg/cmd/run/download/download.go
Normal file
129
pkg/cmd/run/download/download.go
Normal file
|
|
@ -0,0 +1,129 @@
|
|||
package download
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/MakeNowJust/heredoc"
|
||||
"github.com/cli/cli/pkg/cmdutil"
|
||||
"github.com/cli/cli/pkg/iostreams"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
type DownloadOptions struct {
|
||||
IO *iostreams.IOStreams
|
||||
Platform platform
|
||||
|
||||
RunID string
|
||||
DestinationDir string
|
||||
FilePatterns []string
|
||||
}
|
||||
|
||||
type platform interface {
|
||||
List(runID string) ([]Artifact, error)
|
||||
Download(url string, dir string) error
|
||||
}
|
||||
|
||||
func NewCmdDownload(f *cmdutil.Factory, runF func(*DownloadOptions) error) *cobra.Command {
|
||||
opts := &DownloadOptions{
|
||||
IO: f.IOStreams,
|
||||
}
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "download [<run-id>]",
|
||||
Short: "Download artifacts generated by a workflow run",
|
||||
Args: cobra.MaximumNArgs(1),
|
||||
Example: heredoc.Doc(`
|
||||
# Download all artifacts generated by a workflow run
|
||||
$ gh run download <run-id>
|
||||
|
||||
# Download a specific artifact within a run
|
||||
$ gh run download <run-id> -p <name>
|
||||
|
||||
# Download specific artifacts across all runs in a repository
|
||||
$ gh run download -p <name1> -p <name2>
|
||||
`),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if len(args) > 0 {
|
||||
opts.RunID = args[0]
|
||||
} else if len(opts.FilePatterns) == 0 {
|
||||
return &cmdutil.FlagError{Err: errors.New("either run ID or `--pattern` is required")}
|
||||
}
|
||||
|
||||
// support `-R, --repo` override
|
||||
baseRepo, err := f.BaseRepo()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
httpClient, err := f.HttpClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
opts.Platform = &apiPlatform{
|
||||
client: httpClient,
|
||||
repo: baseRepo,
|
||||
}
|
||||
|
||||
if runF != nil {
|
||||
return runF(opts)
|
||||
}
|
||||
return runDownload(opts)
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().StringVarP(&opts.DestinationDir, "dir", "D", ".", "The directory to download artifacts into")
|
||||
cmd.Flags().StringArrayVarP(&opts.FilePatterns, "pattern", "p", nil, "Download only artifacts that match a glob pattern")
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
func runDownload(opts *DownloadOptions) error {
|
||||
opts.IO.StartProgressIndicator()
|
||||
defer opts.IO.StopProgressIndicator()
|
||||
|
||||
artifacts, err := opts.Platform.List(opts.RunID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error fetching artifacts: %w", err)
|
||||
}
|
||||
|
||||
// track downloaded artifacts and avoid re-downloading any of the same name
|
||||
downloaded := map[string]struct{}{}
|
||||
numArtifacts := 0
|
||||
|
||||
for _, a := range artifacts {
|
||||
if a.Expired {
|
||||
continue
|
||||
}
|
||||
numArtifacts++
|
||||
if _, found := downloaded[a.Name]; found {
|
||||
continue
|
||||
}
|
||||
if len(opts.FilePatterns) > 0 && !matchAny(opts.FilePatterns, a.Name) {
|
||||
continue
|
||||
}
|
||||
err := opts.Platform.Download(a.DownloadURL, opts.DestinationDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error downloading %s: %w", a.Name, err)
|
||||
}
|
||||
downloaded[a.Name] = struct{}{}
|
||||
}
|
||||
|
||||
if numArtifacts == 0 {
|
||||
return errors.New("no valid artifacts found to download")
|
||||
}
|
||||
if len(downloaded) == 0 {
|
||||
return errors.New("no artifact matches any of the patterns provided")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func matchAny(patterns []string, name string) bool {
|
||||
for _, p := range patterns {
|
||||
if isMatch, err := filepath.Match(p, name); err == nil && isMatch {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
181
pkg/cmd/run/download/download_test.go
Normal file
181
pkg/cmd/run/download/download_test.go
Normal file
|
|
@ -0,0 +1,181 @@
|
|||
package download
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/cli/cli/internal/ghrepo"
|
||||
"github.com/cli/cli/pkg/cmdutil"
|
||||
"github.com/cli/cli/pkg/iostreams"
|
||||
"github.com/google/shlex"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func Test_NewCmdDownload(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
args string
|
||||
isTTY bool
|
||||
want DownloadOptions
|
||||
wantErr string
|
||||
}{
|
||||
{
|
||||
name: "empty",
|
||||
args: "",
|
||||
isTTY: true,
|
||||
wantErr: "either run ID or `--pattern` is required",
|
||||
},
|
||||
{
|
||||
name: "with run ID",
|
||||
args: "2345",
|
||||
isTTY: true,
|
||||
want: DownloadOptions{
|
||||
RunID: "2345",
|
||||
FilePatterns: []string(nil),
|
||||
DestinationDir: ".",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "to destination",
|
||||
args: "2345 -D tmp/dest",
|
||||
isTTY: true,
|
||||
want: DownloadOptions{
|
||||
RunID: "2345",
|
||||
FilePatterns: []string(nil),
|
||||
DestinationDir: "tmp/dest",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "repo level with patterns",
|
||||
args: "-p one -p two",
|
||||
isTTY: true,
|
||||
want: DownloadOptions{
|
||||
RunID: "",
|
||||
FilePatterns: []string{"one", "two"},
|
||||
DestinationDir: ".",
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
io, _, _, _ := iostreams.Test()
|
||||
io.SetStdoutTTY(tt.isTTY)
|
||||
io.SetStdinTTY(tt.isTTY)
|
||||
io.SetStderrTTY(tt.isTTY)
|
||||
|
||||
f := &cmdutil.Factory{
|
||||
IOStreams: io,
|
||||
HttpClient: func() (*http.Client, error) {
|
||||
return nil, nil
|
||||
},
|
||||
BaseRepo: func() (ghrepo.Interface, error) {
|
||||
return nil, nil
|
||||
},
|
||||
}
|
||||
|
||||
var opts *DownloadOptions
|
||||
cmd := NewCmdDownload(f, func(o *DownloadOptions) error {
|
||||
opts = o
|
||||
return nil
|
||||
})
|
||||
cmd.PersistentFlags().StringP("repo", "R", "", "")
|
||||
|
||||
argv, err := shlex.Split(tt.args)
|
||||
require.NoError(t, err)
|
||||
cmd.SetArgs(argv)
|
||||
|
||||
cmd.SetIn(&bytes.Buffer{})
|
||||
cmd.SetOut(ioutil.Discard)
|
||||
cmd.SetErr(ioutil.Discard)
|
||||
|
||||
_, err = cmd.ExecuteC()
|
||||
if tt.wantErr != "" {
|
||||
require.EqualError(t, err, tt.wantErr)
|
||||
return
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
assert.Equal(t, tt.want.RunID, opts.RunID)
|
||||
assert.Equal(t, tt.want.FilePatterns, opts.FilePatterns)
|
||||
assert.Equal(t, tt.want.DestinationDir, opts.DestinationDir)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_runDownload(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
opts DownloadOptions
|
||||
artifacts []Artifact
|
||||
wantDownloaded []string
|
||||
wantErr string
|
||||
}{
|
||||
{
|
||||
name: "download non-expired",
|
||||
opts: DownloadOptions{
|
||||
RunID: "2345",
|
||||
DestinationDir: ".",
|
||||
FilePatterns: []string(nil),
|
||||
},
|
||||
artifacts: []Artifact{
|
||||
{
|
||||
Name: "artifact-1",
|
||||
DownloadURL: "http://download.com/artifact-1.zip",
|
||||
Expired: false,
|
||||
},
|
||||
{
|
||||
Name: "expired-artifact",
|
||||
DownloadURL: "http://download.com/expired.zip",
|
||||
Expired: true,
|
||||
},
|
||||
{
|
||||
Name: "artifact-2",
|
||||
DownloadURL: "http://download.com/artifact-2.zip",
|
||||
Expired: false,
|
||||
},
|
||||
},
|
||||
wantDownloaded: []string{
|
||||
"http://download.com/artifact-1.zip",
|
||||
"http://download.com/artifact-2.zip",
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
opts := &tt.opts
|
||||
io, _, stdout, stderr := iostreams.Test()
|
||||
opts.IO = io
|
||||
platform := &stubPlatform{listResults: tt.artifacts}
|
||||
opts.Platform = platform
|
||||
|
||||
err := runDownload(opts)
|
||||
if tt.wantErr != "" {
|
||||
require.EqualError(t, err, tt.wantErr)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
assert.Equal(t, tt.wantDownloaded, platform.downloaded)
|
||||
assert.Equal(t, "", stdout.String())
|
||||
assert.Equal(t, "", stderr.String())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type stubPlatform struct {
|
||||
listResults []Artifact
|
||||
downloaded []string
|
||||
}
|
||||
|
||||
func (p *stubPlatform) List(runID string) ([]Artifact, error) {
|
||||
return p.listResults, nil
|
||||
}
|
||||
|
||||
func (p *stubPlatform) Download(url string, dir string) error {
|
||||
p.downloaded = append(p.downloaded, url)
|
||||
return nil
|
||||
}
|
||||
109
pkg/cmd/run/download/http.go
Normal file
109
pkg/cmd/run/download/http.go
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
package download
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
"github.com/cli/cli/api"
|
||||
"github.com/cli/cli/internal/ghinstance"
|
||||
"github.com/cli/cli/internal/ghrepo"
|
||||
)
|
||||
|
||||
type Artifact struct {
|
||||
Name string `json:"name"`
|
||||
Size uint32 `json:"size_in_bytes"`
|
||||
DownloadURL string `json:"archive_download_url"`
|
||||
Expired bool `json:"expired"`
|
||||
}
|
||||
|
||||
type apiPlatform struct {
|
||||
client *http.Client
|
||||
repo ghrepo.Interface
|
||||
}
|
||||
|
||||
func (p *apiPlatform) List(runID string) ([]Artifact, error) {
|
||||
return listArtifacts(p.client, p.repo, runID)
|
||||
}
|
||||
|
||||
func (p *apiPlatform) Download(url string, dir string) error {
|
||||
return downloadArtifact(p.client, url, dir)
|
||||
}
|
||||
|
||||
func listArtifacts(httpClient *http.Client, repo ghrepo.Interface, runID string) ([]Artifact, error) {
|
||||
perPage := 100
|
||||
path := fmt.Sprintf("repos/%s/%s/actions/artifacts?per_page=%d", repo.RepoOwner(), repo.RepoName(), perPage)
|
||||
if runID != "" {
|
||||
path = fmt.Sprintf("repos/%s/%s/actions/runs/%s/artifacts?per_page=%d", repo.RepoOwner(), repo.RepoName(), runID, perPage)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("GET", ghinstance.RESTPrefix(repo.RepoHost())+path, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode > 299 {
|
||||
return nil, api.HandleHTTPError(resp)
|
||||
}
|
||||
|
||||
var response struct {
|
||||
TotalCount uint16 `json:"total_count"`
|
||||
Artifacts []Artifact
|
||||
}
|
||||
|
||||
dec := json.NewDecoder(resp.Body)
|
||||
if err := dec.Decode(&response); err != nil {
|
||||
return response.Artifacts, fmt.Errorf("error reading JSON: %w", err)
|
||||
}
|
||||
|
||||
return response.Artifacts, nil
|
||||
}
|
||||
|
||||
func downloadArtifact(httpClient *http.Client, url, destDir string) error {
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The API rejects requesting the "application/octet-stream" type :(
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode > 299 {
|
||||
return api.HandleHTTPError(resp)
|
||||
}
|
||||
|
||||
tmpfile, err := ioutil.TempFile("", "gh-artifact.*.zip")
|
||||
if err != nil {
|
||||
return fmt.Errorf("error initializing temporary file: %w", err)
|
||||
}
|
||||
defer tmpfile.Close()
|
||||
|
||||
_, err = io.Copy(tmpfile, resp.Body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error writing zip archive: %w", err)
|
||||
}
|
||||
|
||||
tmpfile.Close()
|
||||
if err := extractZip(tmpfile, destDir); err != nil {
|
||||
return fmt.Errorf("error extracting zip archive: %w", err)
|
||||
}
|
||||
if err := os.Remove(tmpfile.Name()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
61
pkg/cmd/run/download/zip.go
Normal file
61
pkg/cmd/run/download/zip.go
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
package download
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
func extractZip(zipfile *os.File, destDir string) error {
|
||||
zr, err := zip.OpenReader(zipfile.Name())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer zr.Close()
|
||||
|
||||
for _, zf := range zr.File {
|
||||
fpath := filepath.Join(destDir, zf.Name)
|
||||
if err := extractZipFile(zf, fpath); err != nil {
|
||||
return fmt.Errorf("error extracting %q: %w", zf.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func extractZipFile(zf *zip.File, dest string) error {
|
||||
zm := zf.Mode()
|
||||
if zm.IsDir() {
|
||||
return os.MkdirAll(dest, 0755)
|
||||
}
|
||||
|
||||
f, err := zf.Open()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var mode os.FileMode = 0644
|
||||
if isBinary(zm) {
|
||||
mode = 0755
|
||||
}
|
||||
df, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer df.Close()
|
||||
|
||||
_, err = io.Copy(df, f)
|
||||
return err
|
||||
}
|
||||
|
||||
func isBinary(m fs.FileMode) bool {
|
||||
return m&0111 != 0
|
||||
}
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
package run
|
||||
|
||||
import (
|
||||
cmdDownload "github.com/cli/cli/pkg/cmd/run/download"
|
||||
cmdList "github.com/cli/cli/pkg/cmd/run/list"
|
||||
cmdView "github.com/cli/cli/pkg/cmd/run/view"
|
||||
"github.com/cli/cli/pkg/cmdutil"
|
||||
|
|
@ -20,6 +21,7 @@ func NewCmdRun(f *cmdutil.Factory) *cobra.Command {
|
|||
|
||||
cmd.AddCommand(cmdList.NewCmdList(f, nil))
|
||||
cmd.AddCommand(cmdView.NewCmdView(f, nil))
|
||||
cmd.AddCommand(cmdDownload.NewCmdDownload(f, nil))
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue