Merge pull request #3012 from cli/api-jq

Add `--filter` to api command to filter data using jq syntax
This commit is contained in:
Mislav Marohnić 2021-03-04 17:47:03 +01:00 committed by GitHub
commit dbf1145cc0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 270 additions and 8 deletions

3
go.mod
View file

@ -15,6 +15,7 @@ require (
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
github.com/hashicorp/go-version v1.2.1
github.com/henvic/httpretty v0.0.6
github.com/itchyny/gojq v0.12.1
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
github.com/mattn/go-colorable v0.1.8
github.com/mattn/go-isatty v0.0.12
@ -31,7 +32,7 @@ require (
golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897
golang.org/x/sync v0.0.0-20190423024810-112230192c58
golang.org/x/text v0.3.4 // indirect
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
)
replace github.com/shurcooL/graphql => github.com/cli/shurcooL-graphql v0.0.0-20200707151639-0f7232a2bf7e

14
go.sum
View file

@ -143,6 +143,13 @@ github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174 h1:WlZsjVhE8Af9IcZDG
github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174/go.mod h1:DqJ97dSdRW1W22yXSB90986pcOyQ7r45iio1KN2ez1A=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/itchyny/astgen-go v0.0.0-20210113000433-0da0671862a3 h1:l7vogWrq+zj8v5t/G69/eT13nAGs2H7cq+CI2nlnKdk=
github.com/itchyny/astgen-go v0.0.0-20210113000433-0da0671862a3/go.mod h1:296z3W7Xsrp2mlIY88ruDKscuvrkL6zXCNRtaYVshzw=
github.com/itchyny/go-flags v1.5.0/go.mod h1:lenkYuCobuxLBAd/HGFE4LRoW8D3B6iXRQfWYJ+MNbA=
github.com/itchyny/gojq v0.12.1 h1:pQJrG8LXgEbZe9hvpfjKg7UlBfieQQydIw3YQq+7WIA=
github.com/itchyny/gojq v0.12.1/go.mod h1:Y5Lz0qoT54ii+ucY/K3yNDy19qzxZvWNBMBpKUDQR/4=
github.com/itchyny/timefmt-go v0.1.1 h1:rLpnm9xxb39PEEVzO0n4IRp0q6/RmBc7Dy/rE4HrA0U=
github.com/itchyny/timefmt-go v0.1.1/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@ -349,12 +356,15 @@ golang.org/x/sys v0.0.0-20190530182044-ad28b68e88f1/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200413165638-669c56c373c4 h1:opSr2sbRXk5X5/givKrrKj9HXxFpW2sdCiP8MJSKLQY=
golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78 h1:nVuTkr9L6Bq62qpUqKo/RnZCFfzDBL0bYo6w9OJUqZY=
golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -420,8 +430,8 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

View file

@ -43,6 +43,7 @@ type ApiOptions struct {
Silent bool
Template string
CacheTTL time.Duration
FilterOutput string
HttpClient func() (*http.Client, error)
BaseRepo func() (ghrepo.Interface, error)
@ -97,6 +98,11 @@ func NewCmdApi(f *cmdutil.Factory, runF func(*ApiOptions) error) *cobra.Command
original query accepts an %[1]s$endCursor: String%[1]s variable and that it fetches the
%[1]spageInfo{ hasNextPage, endCursor }%[1]s set of fields from a collection.
The %[1]s--jq%[1]s option accepts a query in jq syntax and will print only the resulting
values that match the query. This is equivalent to piping the output to %[1]sjq -r%[1]s,
but does not require the jq utility to be installed on the system. To learn more
about the query syntax, see: https://stedolan.github.io/jq/manual/v1.6/
With %[1]s--template%[1]s, the provided Go template is rendered using the JSON data as input.
For the syntax of Go templates, see: https://golang.org/pkg/text/template/
@ -124,6 +130,9 @@ func NewCmdApi(f *cmdutil.Factory, runF func(*ApiOptions) error) *cobra.Command
# opt into GitHub API previews
$ gh api --preview baptiste,nebula ...
# print only specific fields from the response
$ gh api repos/:owner/:repo/issues --filter '.[].title'
# use a template for the output
$ gh api repos/:owner/:repo/issues --template \
'{{range .}}{{.title}} ({{.labels | pluck "name" | join ", " | color "yellow"}}){{"\n"}}{{end}}'
@ -172,15 +181,29 @@ func NewCmdApi(f *cmdutil.Factory, runF func(*ApiOptions) error) *cobra.Command
if c.Flags().Changed("hostname") {
if err := ghinstance.HostnameValidator(opts.Hostname); err != nil {
return &cmdutil.FlagError{Err: fmt.Errorf("error parsing --hostname: %w", err)}
return &cmdutil.FlagError{Err: fmt.Errorf("error parsing `--hostname`: %w", err)}
}
}
if opts.Paginate && !strings.EqualFold(opts.RequestMethod, "GET") && opts.RequestPath != "graphql" {
return &cmdutil.FlagError{Err: errors.New(`the '--paginate' option is not supported for non-GET requests`)}
return &cmdutil.FlagError{Err: errors.New("the `--paginate` option is not supported for non-GET requests")}
}
if opts.Paginate && opts.RequestInputFile != "" {
return &cmdutil.FlagError{Err: errors.New(`the '--paginate' option is not supported with '--input'`)}
if err := cmdutil.MutuallyExclusive(
"the `--paginate` option is not supported with `--input`",
opts.Paginate,
opts.RequestInputFile != "",
); err != nil {
return err
}
if err := cmdutil.MutuallyExclusive(
"only one of `--template`, `--jq`, or `--silent` may be used",
opts.Silent,
opts.FilterOutput != "",
opts.Template != "",
); err != nil {
return err
}
if runF != nil {
@ -201,6 +224,7 @@ func NewCmdApi(f *cmdutil.Factory, runF func(*ApiOptions) error) *cobra.Command
cmd.Flags().StringVar(&opts.RequestInputFile, "input", "", "The `file` to use as body for the HTTP request")
cmd.Flags().BoolVar(&opts.Silent, "silent", false, "Do not print the response body")
cmd.Flags().StringVarP(&opts.Template, "template", "t", "", "Format the response using a Go template")
cmd.Flags().StringVarP(&opts.FilterOutput, "jq", "q", "", "Query to select values from the response using jq syntax")
cmd.Flags().DurationVar(&opts.CacheTTL, "cache", 0, "Cache the response, e.g. \"3600s\", \"60m\", \"1h\"")
return cmd
}
@ -332,7 +356,13 @@ func processResponse(resp *http.Response, opts *ApiOptions, headersOutputStream
responseBody = io.TeeReader(responseBody, bodyCopy)
}
if opts.Template != "" {
if opts.FilterOutput != "" {
// TODO: reuse parsed query across pagination invocations
err = filterJSON(opts.IO.Out, responseBody, opts.FilterOutput)
if err != nil {
return
}
} else if opts.Template != "" {
// TODO: reuse parsed template across pagination invocations
err = executeTemplate(opts.IO.Out, responseBody, opts.Template, opts.IO.ColorEnabled())
if err != nil {

View file

@ -48,6 +48,7 @@ func Test_NewCmdApi(t *testing.T) {
Silent: false,
CacheTTL: 0,
Template: "",
FilterOutput: "",
},
wantsErr: false,
},
@ -68,6 +69,7 @@ func Test_NewCmdApi(t *testing.T) {
Silent: false,
CacheTTL: 0,
Template: "",
FilterOutput: "",
},
wantsErr: false,
},
@ -88,6 +90,7 @@ func Test_NewCmdApi(t *testing.T) {
Silent: false,
CacheTTL: 0,
Template: "",
FilterOutput: "",
},
wantsErr: false,
},
@ -108,6 +111,7 @@ func Test_NewCmdApi(t *testing.T) {
Silent: false,
CacheTTL: 0,
Template: "",
FilterOutput: "",
},
wantsErr: false,
},
@ -128,6 +132,7 @@ func Test_NewCmdApi(t *testing.T) {
Silent: false,
CacheTTL: 0,
Template: "",
FilterOutput: "",
},
wantsErr: false,
},
@ -148,6 +153,7 @@ func Test_NewCmdApi(t *testing.T) {
Silent: true,
CacheTTL: 0,
Template: "",
FilterOutput: "",
},
wantsErr: false,
},
@ -173,6 +179,7 @@ func Test_NewCmdApi(t *testing.T) {
Silent: false,
CacheTTL: 0,
Template: "",
FilterOutput: "",
},
wantsErr: false,
},
@ -198,6 +205,7 @@ func Test_NewCmdApi(t *testing.T) {
Silent: false,
CacheTTL: 0,
Template: "",
FilterOutput: "",
},
wantsErr: false,
},
@ -223,6 +231,7 @@ func Test_NewCmdApi(t *testing.T) {
Silent: false,
CacheTTL: 0,
Template: "",
FilterOutput: "",
},
wantsErr: false,
},
@ -243,6 +252,7 @@ func Test_NewCmdApi(t *testing.T) {
Silent: false,
CacheTTL: time.Minute * 5,
Template: "",
FilterOutput: "",
},
wantsErr: false,
},
@ -263,9 +273,46 @@ func Test_NewCmdApi(t *testing.T) {
Silent: false,
CacheTTL: 0,
Template: "hello {{.name}}",
FilterOutput: "",
},
wantsErr: false,
},
{
name: "with jq filter",
cli: "user -q .name",
wants: ApiOptions{
Hostname: "",
RequestMethod: "GET",
RequestMethodPassed: false,
RequestPath: "user",
RequestInputFile: "",
RawFields: []string(nil),
MagicFields: []string(nil),
RequestHeaders: []string(nil),
ShowResponseHeaders: false,
Paginate: false,
Silent: false,
CacheTTL: 0,
Template: "",
FilterOutput: ".name",
},
wantsErr: false,
},
{
name: "--silent with --jq",
cli: "user --silent -q .foo",
wantsErr: true,
},
{
name: "--silent with --template",
cli: "user --silent -t '{{.foo}}'",
wantsErr: true,
},
{
name: "--jq with --template",
cli: "user --jq .foo -t '{{.foo}}'",
wantsErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@ -301,6 +348,7 @@ func Test_NewCmdApi(t *testing.T) {
assert.Equal(t, tt.wants.Silent, opts.Silent)
assert.Equal(t, tt.wants.CacheTTL, opts.CacheTTL)
assert.Equal(t, tt.wants.Template, opts.Template)
assert.Equal(t, tt.wants.FilterOutput, opts.FilterOutput)
})
}
}
@ -440,6 +488,20 @@ func Test_apiRun(t *testing.T) {
stdout: "not a cat",
stderr: ``,
},
{
name: "jq filter",
options: ApiOptions{
FilterOutput: `.[].name`,
},
httpResponse: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(bytes.NewBufferString(`[{"name":"Mona"},{"name":"Hubot"}]`)),
Header: http.Header{"Content-Type": []string{"application/json"}},
},
err: nil,
stdout: "Mona\nHubot\n",
stderr: ``,
},
}
for _, tt := range tests {

61
pkg/cmd/api/filter.go Normal file
View file

@ -0,0 +1,61 @@
package api
import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"github.com/itchyny/gojq"
)
func filterJSON(w io.Writer, input io.Reader, queryStr string) error {
query, err := gojq.Parse(queryStr)
if err != nil {
return err
}
jsonData, err := ioutil.ReadAll(input)
if err != nil {
return err
}
var responseData interface{}
err = json.Unmarshal(jsonData, &responseData)
if err != nil {
return err
}
iter := query.Run(responseData)
for {
v, ok := iter.Next()
if !ok {
break
}
if err, isErr := v.(error); isErr {
return err
}
if text, e := jsonScalarToString(v); e == nil {
_, err := fmt.Fprintln(w, text)
if err != nil {
return err
}
} else {
var jsonFragment []byte
jsonFragment, err = json.Marshal(v)
if err != nil {
return err
}
_, err = w.Write(jsonFragment)
if err != nil {
return err
}
_, err = fmt.Fprint(w, "\n")
if err != nil {
return err
}
}
}
return nil
}

View file

@ -0,0 +1,85 @@
package api
import (
"bytes"
"io"
"strings"
"testing"
"github.com/MakeNowJust/heredoc"
)
func Test_filterJSON(t *testing.T) {
type args struct {
json io.Reader
query string
}
tests := []struct {
name string
args args
wantW string
wantErr bool
}{
{
name: "simple",
args: args{
json: strings.NewReader(`{"name":"Mona", "arms":8}`),
query: `.name`,
},
wantW: "Mona\n",
},
{
name: "multiple queries",
args: args{
json: strings.NewReader(`{"name":"Mona", "arms":8}`),
query: `.name,.arms`,
},
wantW: "Mona\n8\n",
},
{
name: "object as JSON",
args: args{
json: strings.NewReader(`{"user":{"login":"monalisa"}}`),
query: `.user`,
},
wantW: "{\"login\":\"monalisa\"}\n",
},
{
name: "complex",
args: args{
json: strings.NewReader(heredoc.Doc(`[
{
"title": "First title",
"labels": [{"name":"bug"}, {"name":"help wanted"}]
},
{
"title": "Second but not last",
"labels": []
},
{
"title": "Alas, tis' the end",
"labels": [{}, {"name":"feature"}]
}
]`)),
query: `.[] | [.title,(.labels | map(.name) | join(","))] | @tsv`,
},
wantW: heredoc.Doc(`
First title bug,help wanted
Second but not last
Alas, tis' the end ,feature
`),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
w := &bytes.Buffer{}
if err := filterJSON(w, tt.args.json, tt.args.query); (err != nil) != tt.wantErr {
t.Errorf("filterJSON() error = %v, wantErr %v", err, tt.wantErr)
return
}
if gotW := w.String(); gotW != tt.wantW {
t.Errorf("filterJSON() = %q, want %q", gotW, tt.wantW)
}
})
}
}

View file

@ -28,3 +28,16 @@ var CancelError = errors.New("CancelError")
func IsUserCancellation(err error) bool {
return errors.Is(err, CancelError) || errors.Is(err, terminal.InterruptErr)
}
func MutuallyExclusive(message string, conditions ...bool) error {
numTrue := 0
for _, ok := range conditions {
if ok {
numTrue++
}
}
if numTrue > 1 {
return &FlagError{Err: errors.New(message)}
}
return nil
}