diff --git a/pkg/cmd/api/api.go b/pkg/cmd/api/api.go index c23458aff..dc80298b2 100644 --- a/pkg/cmd/api/api.go +++ b/pkg/cmd/api/api.go @@ -29,6 +29,10 @@ import ( "github.com/spf13/cobra" ) +const ( + ttyIndent = " " +) + type ApiOptions struct { AppVersion string BaseRepo func() (ghrepo.Interface, error) @@ -48,6 +52,7 @@ type ApiOptions struct { Previews []string ShowResponseHeaders bool Paginate bool + Slurp bool Silent bool Template string CacheTTL time.Duration @@ -114,7 +119,9 @@ func NewCmdApi(f *cmdutil.Factory, runF func(*ApiOptions) error) *cobra.Command In %[1]s--paginate%[1]s mode, all pages of results will sequentially be requested until there are no more pages of results. For GraphQL requests, this requires that the original query accepts an %[1]s$endCursor: String%[1]s variable and that it fetches the - %[1]spageInfo{ hasNextPage, endCursor }%[1]s set of fields from a collection. + %[1]spageInfo{ hasNextPage, endCursor }%[1]s set of fields from a collection. Each page is a separate + JSON array or object. Pass %[1]s--slurp%[1]s to wrap all pages of JSON arrays or objects + into an outer JSON array. `, "`"), Example: heredoc.Doc(` # list releases in the current repository @@ -174,6 +181,22 @@ func NewCmdApi(f *cmdutil.Factory, runF func(*ApiOptions) error) *cobra.Command } } ' + + # get the percentage of forks for the current user + $ gh api graphql --paginate --slurp -f query=' + query($endCursor: String) { + viewer { + repositories(first: 100, after: $endCursor) { + nodes { isFork } + pageInfo { + hasNextPage + endCursor + } + } + } + } + ' | jq 'def count(e): reduce e as $_ (0;.+1); + [.[].data.viewer.repositories.nodes[]] as $r | count(select($r[].isFork))/count($r[])' `), Annotations: map[string]string{ "help:environment": heredoc.Doc(` @@ -216,6 +239,19 @@ func NewCmdApi(f *cmdutil.Factory, runF func(*ApiOptions) error) *cobra.Command return err } + if opts.Slurp && !opts.Paginate { + return cmdutil.FlagErrorf("`--paginate` required when passing `--slurp`") + } + + if err := cmdutil.MutuallyExclusive( + "the `--slurp` option is not supported with `--jq` or `--template`", + opts.Slurp, + opts.FilterOutput != "", + opts.Template != "", + ); err != nil { + return err + } + if err := cmdutil.MutuallyExclusive( "only one of `--template`, `--jq`, `--silent`, or `--verbose` may be used", opts.Verbose, @@ -240,6 +276,7 @@ func NewCmdApi(f *cmdutil.Factory, runF func(*ApiOptions) error) *cobra.Command cmd.Flags().StringArrayVarP(&opts.RequestHeaders, "header", "H", nil, "Add a HTTP request header in `key:value` format") cmd.Flags().StringSliceVarP(&opts.Previews, "preview", "p", nil, "GitHub API preview `names` to request (without the \"-preview\" suffix)") cmd.Flags().BoolVarP(&opts.ShowResponseHeaders, "include", "i", false, "Include HTTP response status line and headers in the output") + cmd.Flags().BoolVar(&opts.Slurp, "slurp", false, "Use with \"--paginate\" to return an array of all pages of either JSON arrays or objects") cmd.Flags().BoolVar(&opts.Paginate, "paginate", false, "Make additional HTTP requests to fetch all pages of results") cmd.Flags().StringVar(&opts.RequestInputFile, "input", "", "The `file` to use as body for the HTTP request (use \"-\" to read from standard input)") cmd.Flags().BoolVar(&opts.Silent, "silent", false, "Do not print the response body") @@ -272,10 +309,38 @@ func apiRun(opts *ApiOptions) error { method = "POST" } + var bodyWriter io.Writer = opts.IO.Out + var headersWriter io.Writer = opts.IO.Out + if opts.Silent { + bodyWriter = io.Discard + } + if opts.Verbose { + // httpClient handles output when verbose flag is specified. + bodyWriter = io.Discard + headersWriter = io.Discard + } + if opts.Paginate && !isGraphQL { requestPath = addPerPage(requestPath, 100, params) } + // Execute defers in FIFO order. + deferQueue := queue{} + defer deferQueue.Close() + + // Similar to `jq --slurp`, write all pages JSON arrays or objects into a JSON array. + if opts.Paginate && opts.Slurp { + w := &jsonArrayWriter{ + Writer: bodyWriter, + color: opts.IO.ColorEnabled(), + } + deferQueue.Enqueue(func() { + _ = w.Close() + }) + + bodyWriter = w + } + if opts.RequestInputFile != "" { file, size, err := openUserFile(opts.RequestInputFile, opts.IO.In) if err != nil { @@ -323,23 +388,12 @@ func apiRun(opts *ApiOptions) error { if !opts.Silent { if err := opts.IO.StartPager(); err == nil { - defer opts.IO.StopPager() + deferQueue.Enqueue(opts.IO.StopPager) } else { fmt.Fprintf(opts.IO.ErrOut, "failed to start pager: %v\n", err) } } - var bodyWriter io.Writer = opts.IO.Out - var headersWriter io.Writer = opts.IO.Out - if opts.Silent { - bodyWriter = io.Discard - } - if opts.Verbose { - // httpClient handles output when verbose flag is specified. - bodyWriter = io.Discard - headersWriter = io.Discard - } - host, _ := cfg.Authentication().DefaultHost() if opts.Hostname != "" { @@ -365,6 +419,12 @@ func apiRun(opts *ApiOptions) error { requestBody = nil // prevent repeating GET parameters } + // Tell optional jsonArrayWriter to start a new page. + err = startPage(bodyWriter) + if err != nil { + return err + } + endCursor, err := processResponse(resp, opts, bodyWriter, headersWriter, tmpl, isFirstPage, !hasNextPage) if err != nil { return err @@ -424,7 +484,7 @@ func processResponse(resp *http.Response, opts *ApiOptions, bodyWriter, headersW // TODO: reuse parsed query across pagination invocations indent := "" if opts.IO.IsStdoutTTY() { - indent = " " + indent = ttyIndent } err = jq.EvaluateFormatted(responseBody, bodyWriter, opts.FilterOutput, indent, opts.IO.ColorEnabled()) if err != nil { @@ -436,9 +496,9 @@ func processResponse(resp *http.Response, opts *ApiOptions, bodyWriter, headersW return } } else if isJSON && opts.IO.ColorEnabled() { - err = jsoncolor.Write(bodyWriter, responseBody, " ") + err = jsoncolor.Write(bodyWriter, responseBody, ttyIndent) } else { - if isJSON && opts.Paginate && !isGraphQLPaginate && !opts.ShowResponseHeaders { + if isJSON && opts.Paginate && !opts.Slurp && !isGraphQLPaginate && !opts.ShowResponseHeaders { responseBody = &paginatedArrayReader{ Reader: responseBody, isFirstPage: isFirstPage, @@ -633,3 +693,15 @@ func previewNamesToMIMETypes(names []string) string { } return strings.Join(types, ", ") } + +type queue []func() + +func (q *queue) Enqueue(f func()) { + *q = append(*q, f) +} + +func (q *queue) Close() { + for _, f := range *q { + f() + } +} diff --git a/pkg/cmd/api/api_test.go b/pkg/cmd/api/api_test.go index 02b08f842..38218fcaf 100644 --- a/pkg/cmd/api/api_test.go +++ b/pkg/cmd/api/api_test.go @@ -328,6 +328,21 @@ func Test_NewCmdApi(t *testing.T) { cli: "user --jq .foo -t '{{.foo}}'", wantsErr: true, }, + { + name: "--slurp without --paginate", + cli: "user --slurp", + wantsErr: true, + }, + { + name: "slurp with --jq", + cli: "user --paginate --slurp --jq .foo", + wantsErr: true, + }, + { + name: "slurp with --template", + cli: "user --paginate --slurp --template '{{.foo}}'", + wantsErr: true, + }, { name: "with verbose", cli: "user --verbose", @@ -551,6 +566,34 @@ func Test_apiRun(t *testing.T) { stderr: ``, isatty: false, }, + { + name: "output template with range", + options: ApiOptions{ + Template: `{{range .}}{{.title}} ({{.labels | pluck "name" | join ", " }}){{"\n"}}{{end}}`, + }, + httpResponse: &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewBufferString(`[ + { + "title": "First title", + "labels": [{"name":"bug"}, {"name":"help wanted"}] + }, + { + "title": "Second but not last" + }, + { + "title": "Alas, tis' the end", + "labels": [{}, {"name":"feature"}] + } + ]`)), + Header: http.Header{"Content-Type": []string{"application/json"}}, + }, + stdout: heredoc.Doc(` + First title (bug, help wanted) + Second but not last () + Alas, tis' the end (, feature) + `), + }, { name: "output template when REST error", options: ApiOptions{ @@ -650,23 +693,36 @@ func Test_apiRun_paginationREST(t *testing.T) { requestCount := 0 responses := []*http.Response{ { + Proto: "HTTP/1.1", + Status: "200 OK", StatusCode: 200, Body: io.NopCloser(bytes.NewBufferString(`{"page":1}`)), Header: http.Header{ - "Link": []string{`; rel="next", ; rel="last"`}, + "Content-Type": []string{"application/json"}, + "Link": []string{`; rel="next", ; rel="last"`}, + "X-Github-Request-Id": []string{"1"}, }, }, { + Proto: "HTTP/1.1", + Status: "200 OK", StatusCode: 200, Body: io.NopCloser(bytes.NewBufferString(`{"page":2}`)), Header: http.Header{ - "Link": []string{`; rel="next", ; rel="last"`}, + "Content-Type": []string{"application/json"}, + "Link": []string{`; rel="next", ; rel="last"`}, + "X-Github-Request-Id": []string{"2"}, }, }, { + Proto: "HTTP/1.1", + Status: "200 OK", StatusCode: 200, Body: io.NopCloser(bytes.NewBufferString(`{"page":3}`)), - Header: http.Header{}, + Header: http.Header{ + "Content-Type": []string{"application/json"}, + "X-Github-Request-Id": []string{"3"}, + }, }, } @@ -775,6 +831,79 @@ func Test_apiRun_arrayPaginationREST(t *testing.T) { assert.Equal(t, "https://api.github.com/repositories/1227/issues?page=3", responses[2].Request.URL.String()) } +func Test_apiRun_arrayPaginationREST_with_headers(t *testing.T) { + ios, _, stdout, stderr := iostreams.Test() + + requestCount := 0 + responses := []*http.Response{ + { + Proto: "HTTP/1.1", + Status: "200 OK", + StatusCode: 200, + Body: io.NopCloser(bytes.NewBufferString(`[{"page":1}]`)), + Header: http.Header{ + "Content-Type": []string{"application/json"}, + "Link": []string{`; rel="next", ; rel="last"`}, + "X-Github-Request-Id": []string{"1"}, + }, + }, + { + Proto: "HTTP/1.1", + Status: "200 OK", + StatusCode: 200, + Body: io.NopCloser(bytes.NewBufferString(`[{"page":2}]`)), + Header: http.Header{ + "Content-Type": []string{"application/json"}, + "Link": []string{`; rel="next", ; rel="last"`}, + "X-Github-Request-Id": []string{"2"}, + }, + }, + { + Proto: "HTTP/1.1", + Status: "200 OK", + StatusCode: 200, + Body: io.NopCloser(bytes.NewBufferString(`[{"page":3}]`)), + Header: http.Header{ + "Content-Type": []string{"application/json"}, + "X-Github-Request-Id": []string{"3"}, + }, + }, + } + + options := ApiOptions{ + IO: ios, + HttpClient: func() (*http.Client, error) { + var tr roundTripper = func(req *http.Request) (*http.Response, error) { + resp := responses[requestCount] + resp.Request = req + requestCount++ + return resp, nil + } + return &http.Client{Transport: tr}, nil + }, + Config: func() (config.Config, error) { + return config.NewBlankConfig(), nil + }, + + RequestMethod: "GET", + RequestMethodPassed: true, + RequestPath: "issues", + Paginate: true, + RawFields: []string{"per_page=50", "page=1"}, + ShowResponseHeaders: true, + } + + err := apiRun(&options) + assert.NoError(t, err) + + assert.Equal(t, "HTTP/1.1 200 OK\nContent-Type: application/json\r\nLink: ; rel=\"next\", ; rel=\"last\"\r\nX-Github-Request-Id: 1\r\n\r\n[{\"page\":1}]\nHTTP/1.1 200 OK\nContent-Type: application/json\r\nLink: ; rel=\"next\", ; rel=\"last\"\r\nX-Github-Request-Id: 2\r\n\r\n[{\"page\":2}]\nHTTP/1.1 200 OK\nContent-Type: application/json\r\nX-Github-Request-Id: 3\r\n\r\n[{\"page\":3}]", stdout.String(), "stdout") + assert.Equal(t, "", stderr.String(), "stderr") + + assert.Equal(t, "https://api.github.com/issues?page=1&per_page=50", responses[0].Request.URL.String()) + assert.Equal(t, "https://api.github.com/repositories/1227/issues?page=2", responses[1].Request.URL.String()) + assert.Equal(t, "https://api.github.com/repositories/1227/issues?page=3", responses[2].Request.URL.String()) +} + func Test_apiRun_paginationGraphQL(t *testing.T) { ios, _, stdout, stderr := iostreams.Test() @@ -783,7 +912,8 @@ func Test_apiRun_paginationGraphQL(t *testing.T) { { StatusCode: 200, Header: http.Header{"Content-Type": []string{`application/json`}}, - Body: io.NopCloser(bytes.NewBufferString(`{ + Body: io.NopCloser(bytes.NewBufferString(heredoc.Doc(` + { "data": { "nodes": ["page one"], "pageInfo": { @@ -791,12 +921,13 @@ func Test_apiRun_paginationGraphQL(t *testing.T) { "hasNextPage": true } } - }`)), + }`))), }, { StatusCode: 200, Header: http.Header{"Content-Type": []string{`application/json`}}, - Body: io.NopCloser(bytes.NewBufferString(`{ + Body: io.NopCloser(bytes.NewBufferString(heredoc.Doc(` + { "data": { "nodes": ["page two"], "pageInfo": { @@ -804,7 +935,7 @@ func Test_apiRun_paginationGraphQL(t *testing.T) { "hasNextPage": false } } - }`)), + }`))), }, } @@ -832,8 +963,127 @@ func Test_apiRun_paginationGraphQL(t *testing.T) { err := apiRun(&options) require.NoError(t, err) - assert.Contains(t, stdout.String(), `"page one"`) - assert.Contains(t, stdout.String(), `"page two"`) + assert.Equal(t, heredoc.Doc(` + { + "data": { + "nodes": ["page one"], + "pageInfo": { + "endCursor": "PAGE1_END", + "hasNextPage": true + } + } + }{ + "data": { + "nodes": ["page two"], + "pageInfo": { + "endCursor": "PAGE2_END", + "hasNextPage": false + } + } + }`), stdout.String()) + assert.Equal(t, "", stderr.String(), "stderr") + + var requestData struct { + Variables map[string]interface{} + } + + bb, err := io.ReadAll(responses[0].Request.Body) + require.NoError(t, err) + err = json.Unmarshal(bb, &requestData) + require.NoError(t, err) + _, hasCursor := requestData.Variables["endCursor"].(string) + assert.Equal(t, false, hasCursor) + + bb, err = io.ReadAll(responses[1].Request.Body) + require.NoError(t, err) + err = json.Unmarshal(bb, &requestData) + require.NoError(t, err) + endCursor, hasCursor := requestData.Variables["endCursor"].(string) + assert.Equal(t, true, hasCursor) + assert.Equal(t, "PAGE1_END", endCursor) +} + +func Test_apiRun_paginationGraphQL_slurp(t *testing.T) { + ios, _, stdout, stderr := iostreams.Test() + + requestCount := 0 + responses := []*http.Response{ + { + StatusCode: 200, + Header: http.Header{"Content-Type": []string{`application/json`}}, + Body: io.NopCloser(bytes.NewBufferString(heredoc.Doc(` + { + "data": { + "nodes": ["page one"], + "pageInfo": { + "endCursor": "PAGE1_END", + "hasNextPage": true + } + } + }`))), + }, + { + StatusCode: 200, + Header: http.Header{"Content-Type": []string{`application/json`}}, + Body: io.NopCloser(bytes.NewBufferString(heredoc.Doc(` + { + "data": { + "nodes": ["page two"], + "pageInfo": { + "endCursor": "PAGE2_END", + "hasNextPage": false + } + } + }`))), + }, + } + + options := ApiOptions{ + IO: ios, + HttpClient: func() (*http.Client, error) { + var tr roundTripper = func(req *http.Request) (*http.Response, error) { + resp := responses[requestCount] + resp.Request = req + requestCount++ + return resp, nil + } + return &http.Client{Transport: tr}, nil + }, + Config: func() (config.Config, error) { + return config.NewBlankConfig(), nil + }, + + RawFields: []string{"foo=bar"}, + RequestMethod: "POST", + RequestPath: "graphql", + Paginate: true, + Slurp: true, + } + + err := apiRun(&options) + require.NoError(t, err) + + assert.JSONEq(t, stdout.String(), `[ + { + "data": { + "nodes": ["page one"], + "pageInfo": { + "endCursor": "PAGE1_END", + "hasNextPage": true + } + } + }, + { + + "data": { + "nodes": ["page two"], + "pageInfo": { + "endCursor": "PAGE2_END", + "hasNextPage": false + } + } + } + ]`) assert.Equal(t, "", stderr.String(), "stderr") var requestData struct { @@ -1511,3 +1761,20 @@ func Test_apiRun_acceptHeader(t *testing.T) { }) } } + +func TestQueue_Close(t *testing.T) { + sut := queue{} + actual := make([]int, 0, 2) + + func() { + defer sut.Close() + sut.Enqueue(func() { + actual = append(actual, 0) + }) + sut.Enqueue(func() { + actual = append(actual, 1) + }) + }() + + assert.Equal(t, []int{0, 1}, actual) +} diff --git a/pkg/cmd/api/pagination.go b/pkg/cmd/api/pagination.go index 057a5a7fa..bf4a2f794 100644 --- a/pkg/cmd/api/pagination.go +++ b/pkg/cmd/api/pagination.go @@ -8,6 +8,8 @@ import ( "net/url" "regexp" "strings" + + "github.com/cli/cli/v2/pkg/jsoncolor" ) var linkRE = regexp.MustCompile(`<([^>]+)>;\s*rel="([^"]+)"`) @@ -146,3 +148,94 @@ func (r *paginatedArrayReader) Read(p []byte) (int, error) { r.isSubsequentRead = true return n, err } + +// jsonArrayWriter wraps a Writer which writes multiple pages of both JSON arrays +// and objects. Call Close to write the end of the array. +type jsonArrayWriter struct { + io.Writer + started bool + color bool +} + +func (w *jsonArrayWriter) Preface() []json.Delim { + if w.started { + return []json.Delim{'['} + } + return nil +} + +// ReadFrom implements io.ReaderFrom to write more data than read, +// which otherwise results in an error from io.Copy(). +func (w *jsonArrayWriter) ReadFrom(r io.Reader) (int64, error) { + var written int64 + buf := make([]byte, 4069) + for { + n, err := r.Read(buf) + if n > 0 { + n, err := w.Write(buf[:n]) + written += int64(n) + + if err != nil { + return written, err + } + } + if err == io.EOF { + break + } + if err != nil { + return written, err + } + } + + return written, nil +} + +func (w *jsonArrayWriter) Close() error { + var delims string + if w.started { + delims = "]" + } else { + delims = "[]" + } + + w.started = false + if w.color { + return jsoncolor.WriteDelims(w, delims, ttyIndent) + } + + _, err := w.Writer.Write([]byte(delims)) + return err +} + +func startPage(w io.Writer) error { + if jaw, ok := w.(*jsonArrayWriter); ok { + var delims string + var indent bool + + if !jaw.started { + delims = "[" + jaw.started = true + } else { + delims = "," + indent = true + } + + if jaw.color { + if indent { + _, err := jaw.Write([]byte(ttyIndent)) + if err != nil { + return err + } + } + + return jsoncolor.WriteDelims(w, delims, ttyIndent) + } + + _, err := jaw.Write([]byte(delims)) + if err != nil { + return err + } + } + + return nil +} diff --git a/pkg/cmd/api/pagination_test.go b/pkg/cmd/api/pagination_test.go index 3bb1a8ec5..746a73c4a 100644 --- a/pkg/cmd/api/pagination_test.go +++ b/pkg/cmd/api/pagination_test.go @@ -5,6 +5,9 @@ import ( "io" "net/http" "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func Test_findNextPage(t *testing.T) { @@ -167,3 +170,107 @@ func Test_addPerPage(t *testing.T) { }) } } + +func TestJsonArrayWriter(t *testing.T) { + tests := []struct { + name string + pages []string + want string + }{ + { + name: "empty", + pages: nil, + want: "[]", + }, + { + name: "single array", + pages: []string{`[1,2]`}, + want: `[[1,2]]`, + }, + { + name: "multiple arrays", + pages: []string{`[1,2]`, `[3]`}, + want: `[[1,2],[3]]`, + }, + { + name: "single object", + pages: []string{`{"foo":1,"bar":"a"}`}, + want: `[{"foo":1,"bar":"a"}]`, + }, + { + name: "multiple pages", + pages: []string{`{"foo":1,"bar":"a"}`, `{"foo":2,"bar":"b"}`}, + want: `[{"foo":1,"bar":"a"},{"foo":2,"bar":"b"}]`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := &bytes.Buffer{} + w := &jsonArrayWriter{ + Writer: buf, + } + + for _, page := range tt.pages { + require.NoError(t, startPage(w)) + + n, err := w.Write([]byte(page)) + require.NoError(t, err) + assert.Equal(t, len(page), n) + } + + require.NoError(t, w.Close()) + assert.Equal(t, tt.want, buf.String()) + }) + } +} + +func TestJsonArrayWriter_Copy(t *testing.T) { + tests := []struct { + name string + limit int + }{ + { + name: "unlimited", + }, + { + name: "limited", + limit: 2, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := &bytes.Buffer{} + w := &jsonArrayWriter{ + Writer: buf, + } + + r := &noWriteToReader{ + Reader: bytes.NewBufferString(`[1,2]`), + limit: tt.limit, + } + + require.NoError(t, startPage(w)) + + n, err := io.Copy(w, r) + require.NoError(t, err) + assert.Equal(t, int64(5), n) + + require.NoError(t, w.Close()) + assert.Equal(t, `[[1,2]]`, buf.String()) + }) + } +} + +type noWriteToReader struct { + io.Reader + limit int +} + +func (r *noWriteToReader) Read(p []byte) (int, error) { + if r.limit > 0 { + p = p[:r.limit] + } + return r.Reader.Read(p) +} diff --git a/pkg/jsoncolor/jsoncolor.go b/pkg/jsoncolor/jsoncolor.go index dbe3d9a4b..8e20a1161 100644 --- a/pkg/jsoncolor/jsoncolor.go +++ b/pkg/jsoncolor/jsoncolor.go @@ -16,6 +16,10 @@ const ( colorBool = "33" // yellow ) +type JsonWriter interface { + Preface() []json.Delim +} + // Write colorized JSON output parsed from reader func Write(w io.Writer, r io.Reader, indent string) error { dec := json.NewDecoder(r) @@ -24,6 +28,10 @@ func Write(w io.Writer, r io.Reader, indent string) error { var idx int var stack []json.Delim + if jsonWriter, ok := w.(JsonWriter); ok { + stack = append(stack, jsonWriter.Preface()...) + } + for { t, err := dec.Token() if err == io.EOF { @@ -96,6 +104,20 @@ func Write(w io.Writer, r io.Reader, indent string) error { return nil } +// WriteDelims writes delims in color and with the appropriate indent +// based on the stack size returned from an io.Writer that implements JsonWriter.Preface(). +func WriteDelims(w io.Writer, delims, indent string) error { + var stack []json.Delim + if jaw, ok := w.(JsonWriter); ok { + stack = jaw.Preface() + } + + fmt.Fprintf(w, "\x1b[%sm%s\x1b[m", colorDelim, delims) + fmt.Fprint(w, "\n", strings.Repeat(indent, len(stack))) + + return nil +} + // marshalJSON works like json.Marshal but with HTML-escaping disabled func marshalJSON(v interface{}) ([]byte, error) { buf := bytes.Buffer{}