Merge pull request #10767 from leudz/fix-9749
Fix multi pages search for gh search
This commit is contained in:
commit
0ef0c42665
3 changed files with 439 additions and 102 deletions
|
|
@ -93,25 +93,29 @@ var PullRequestFields = append(IssueFields,
|
|||
type CodeResult struct {
|
||||
IncompleteResults bool `json:"incomplete_results"`
|
||||
Items []Code `json:"items"`
|
||||
Total int `json:"total_count"`
|
||||
// Number of code search results matching the query on the server. Ignoring limit.
|
||||
Total int `json:"total_count"`
|
||||
}
|
||||
|
||||
type CommitsResult struct {
|
||||
IncompleteResults bool `json:"incomplete_results"`
|
||||
Items []Commit `json:"items"`
|
||||
Total int `json:"total_count"`
|
||||
// Number of commits matching the query on the server. Ignoring limit.
|
||||
Total int `json:"total_count"`
|
||||
}
|
||||
|
||||
type RepositoriesResult struct {
|
||||
IncompleteResults bool `json:"incomplete_results"`
|
||||
Items []Repository `json:"items"`
|
||||
Total int `json:"total_count"`
|
||||
// Number of repositories matching the query on the server. Ignoring limit.
|
||||
Total int `json:"total_count"`
|
||||
}
|
||||
|
||||
type IssuesResult struct {
|
||||
IncompleteResults bool `json:"incomplete_results"`
|
||||
Items []Issue `json:"items"`
|
||||
Total int `json:"total_count"`
|
||||
// Number of isssues matching the query on the server. Ignoring limit.
|
||||
Total int `json:"total_count"`
|
||||
}
|
||||
|
||||
type Code struct {
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import (
|
|||
)
|
||||
|
||||
const (
|
||||
// GitHub API has a limit of 100 per page
|
||||
maxPerPage = 100
|
||||
orderKey = "order"
|
||||
sortKey = "sort"
|
||||
|
|
@ -60,100 +61,145 @@ func NewSearcher(client *http.Client, host string) Searcher {
|
|||
|
||||
func (s searcher) Code(query Query) (CodeResult, error) {
|
||||
result := CodeResult{}
|
||||
toRetrieve := query.Limit
|
||||
|
||||
var resp *http.Response
|
||||
var err error
|
||||
for toRetrieve > 0 {
|
||||
query.Limit = min(toRetrieve, maxPerPage)
|
||||
|
||||
// We will request either the query limit if it's less than 1 page, or our max page size.
|
||||
// This number doesn't change to keep a valid offset.
|
||||
//
|
||||
// For example, say we want 150 items out of 500.
|
||||
// We request page #1 for 100 items and get items 0 to 99.
|
||||
// Then we request page #2 for 100 items, we get items 100 to 199 and only keep 100 to 149.
|
||||
// If we were to request page #2 for 50 items, we would instead get items 50 to 99.
|
||||
numItemsToRetrieve := query.Limit
|
||||
query.Limit = min(numItemsToRetrieve, maxPerPage)
|
||||
|
||||
for numItemsToRetrieve > 0 {
|
||||
query.Page = nextPage(resp)
|
||||
if query.Page == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
page := CodeResult{}
|
||||
resp, err = s.search(query, &page)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
// If we're going to reach the requested limit, only add that many items,
|
||||
// otherwise add all the results.
|
||||
numItemsToAdd := min(len(page.Items), numItemsToRetrieve)
|
||||
result.IncompleteResults = page.IncompleteResults
|
||||
// The API returns how many items match the query in every response.
|
||||
// With the example above, this would be 500.
|
||||
result.Total = page.Total
|
||||
result.Items = append(result.Items, page.Items...)
|
||||
toRetrieve = toRetrieve - len(page.Items)
|
||||
result.Items = append(result.Items, page.Items[:numItemsToAdd]...)
|
||||
numItemsToRetrieve = numItemsToRetrieve - numItemsToAdd
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s searcher) Commits(query Query) (CommitsResult, error) {
|
||||
result := CommitsResult{}
|
||||
toRetrieve := query.Limit
|
||||
|
||||
var resp *http.Response
|
||||
var err error
|
||||
for toRetrieve > 0 {
|
||||
query.Limit = min(toRetrieve, maxPerPage)
|
||||
|
||||
numItemsToRetrieve := query.Limit
|
||||
query.Limit = min(numItemsToRetrieve, maxPerPage)
|
||||
|
||||
for numItemsToRetrieve > 0 {
|
||||
query.Page = nextPage(resp)
|
||||
if query.Page == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
page := CommitsResult{}
|
||||
resp, err = s.search(query, &page)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
numItemsToAdd := min(len(page.Items), numItemsToRetrieve)
|
||||
result.IncompleteResults = page.IncompleteResults
|
||||
result.Total = page.Total
|
||||
result.Items = append(result.Items, page.Items...)
|
||||
toRetrieve = toRetrieve - len(page.Items)
|
||||
result.Items = append(result.Items, page.Items[:numItemsToAdd]...)
|
||||
numItemsToRetrieve = numItemsToRetrieve - numItemsToAdd
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s searcher) Repositories(query Query) (RepositoriesResult, error) {
|
||||
result := RepositoriesResult{}
|
||||
toRetrieve := query.Limit
|
||||
|
||||
var resp *http.Response
|
||||
var err error
|
||||
for toRetrieve > 0 {
|
||||
query.Limit = min(toRetrieve, maxPerPage)
|
||||
|
||||
numItemsToRetrieve := query.Limit
|
||||
query.Limit = min(numItemsToRetrieve, maxPerPage)
|
||||
|
||||
for numItemsToRetrieve > 0 {
|
||||
query.Page = nextPage(resp)
|
||||
if query.Page == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
page := RepositoriesResult{}
|
||||
resp, err = s.search(query, &page)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
numItemsToAdd := min(len(page.Items), numItemsToRetrieve)
|
||||
result.IncompleteResults = page.IncompleteResults
|
||||
result.Total = page.Total
|
||||
result.Items = append(result.Items, page.Items...)
|
||||
toRetrieve = toRetrieve - len(page.Items)
|
||||
result.Items = append(result.Items, page.Items[:numItemsToAdd]...)
|
||||
numItemsToRetrieve = numItemsToRetrieve - numItemsToAdd
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s searcher) Issues(query Query) (IssuesResult, error) {
|
||||
result := IssuesResult{}
|
||||
toRetrieve := query.Limit
|
||||
|
||||
var resp *http.Response
|
||||
var err error
|
||||
for toRetrieve > 0 {
|
||||
query.Limit = min(toRetrieve, maxPerPage)
|
||||
|
||||
numItemsToRetrieve := query.Limit
|
||||
query.Limit = min(numItemsToRetrieve, maxPerPage)
|
||||
|
||||
for numItemsToRetrieve > 0 {
|
||||
query.Page = nextPage(resp)
|
||||
if query.Page == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
page := IssuesResult{}
|
||||
resp, err = s.search(query, &page)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
numItemsToAdd := min(len(page.Items), numItemsToRetrieve)
|
||||
result.IncompleteResults = page.IncompleteResults
|
||||
result.Total = page.Total
|
||||
result.Items = append(result.Items, page.Items...)
|
||||
toRetrieve = toRetrieve - len(page.Items)
|
||||
result.Items = append(result.Items, page.Items[:numItemsToAdd]...)
|
||||
numItemsToRetrieve = numItemsToRetrieve - numItemsToAdd
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// search makes a single-page REST search request for code, commits, issues, prs, or repos.
|
||||
//
|
||||
// The result argument is populated with the following information:
|
||||
//
|
||||
// - Total: the number of search results matching the query, which may exceed the number of items returned
|
||||
// - IncompleteResults: whether the search request exceeded search time limit, potentially being incomplete
|
||||
// - Items: the actual matching search results, up to 100 max items per page
|
||||
//
|
||||
// For more information, see https://docs.github.com/en/rest/search/search?apiVersion=2022-11-28.
|
||||
func (s searcher) search(query Query, result interface{}) (*http.Response, error) {
|
||||
path := fmt.Sprintf("%ssearch/%s", ghinstance.RESTPrefix(s.host), query.Kind)
|
||||
qs := url.Values{}
|
||||
|
|
@ -236,10 +282,15 @@ func handleHTTPError(resp *http.Response) error {
|
|||
return httpError
|
||||
}
|
||||
|
||||
// https://docs.github.com/en/rest/using-the-rest-api/using-pagination-in-the-rest-api
|
||||
func nextPage(resp *http.Response) (page int) {
|
||||
if resp == nil {
|
||||
return 1
|
||||
}
|
||||
|
||||
// When using pagination, responses get a "Link" field in their header.
|
||||
// When a next page is available, "Link" contains a link to the next page
|
||||
// tagged with rel="next".
|
||||
for _, m := range linkRE.FindAllStringSubmatch(resp.Header.Get("Link"), -1) {
|
||||
if !(len(m) > 2 && m[2] == "next") {
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -1,8 +1,10 @@
|
|||
package search
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/MakeNowJust/heredoc"
|
||||
|
|
@ -46,10 +48,14 @@ func TestSearcherCode(t *testing.T) {
|
|||
httpStubs: func(reg *httpmock.Registry) {
|
||||
reg.Register(
|
||||
httpmock.QueryMatcher("GET", "search/code", values),
|
||||
httpmock.JSONResponse(CodeResult{
|
||||
IncompleteResults: false,
|
||||
Items: []Code{{Name: "file.go"}},
|
||||
Total: 1,
|
||||
httpmock.JSONResponse(map[string]interface{}{
|
||||
"incomplete_results": false,
|
||||
"total_count": 1,
|
||||
"items": []interface{}{
|
||||
map[string]interface{}{
|
||||
"name": "file.go",
|
||||
},
|
||||
},
|
||||
}),
|
||||
)
|
||||
},
|
||||
|
|
@ -66,10 +72,14 @@ func TestSearcherCode(t *testing.T) {
|
|||
httpStubs: func(reg *httpmock.Registry) {
|
||||
reg.Register(
|
||||
httpmock.QueryMatcher("GET", "api/v3/search/code", values),
|
||||
httpmock.JSONResponse(CodeResult{
|
||||
IncompleteResults: false,
|
||||
Items: []Code{{Name: "file.go"}},
|
||||
Total: 1,
|
||||
httpmock.JSONResponse(map[string]interface{}{
|
||||
"incomplete_results": false,
|
||||
"total_count": 1,
|
||||
"items": []interface{}{
|
||||
map[string]interface{}{
|
||||
"name": "file.go",
|
||||
},
|
||||
},
|
||||
}),
|
||||
)
|
||||
},
|
||||
|
|
@ -84,25 +94,83 @@ func TestSearcherCode(t *testing.T) {
|
|||
},
|
||||
httpStubs: func(reg *httpmock.Registry) {
|
||||
firstReq := httpmock.QueryMatcher("GET", "search/code", values)
|
||||
firstRes := httpmock.JSONResponse(CodeResult{
|
||||
IncompleteResults: false,
|
||||
Items: []Code{{Name: "file.go"}},
|
||||
Total: 2,
|
||||
firstRes := httpmock.JSONResponse(map[string]interface{}{
|
||||
"incomplete_results": false,
|
||||
"total_count": 2,
|
||||
"items": []interface{}{
|
||||
map[string]interface{}{
|
||||
"name": "file.go",
|
||||
},
|
||||
},
|
||||
})
|
||||
firstRes = httpmock.WithHeader(firstRes, "Link", `<https://api.github.com/search/code?page=2&per_page=30&q=org%3Agithub>; rel="next"`)
|
||||
secondReq := httpmock.QueryMatcher("GET", "search/code", url.Values{
|
||||
"page": []string{"2"},
|
||||
"per_page": []string{"30"},
|
||||
"q": []string{"keyword language:go"},
|
||||
})
|
||||
secondRes := httpmock.JSONResponse(map[string]interface{}{
|
||||
"incomplete_results": false,
|
||||
"total_count": 2,
|
||||
"items": []interface{}{
|
||||
map[string]interface{}{
|
||||
"name": "file2.go",
|
||||
},
|
||||
},
|
||||
})
|
||||
reg.Register(firstReq, firstRes)
|
||||
reg.Register(secondReq, secondRes)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "collect full and partial pages under total number of matching search results",
|
||||
query: Query{
|
||||
Keywords: []string{"keyword"},
|
||||
Kind: "code",
|
||||
Limit: 110,
|
||||
Qualifiers: Qualifiers{
|
||||
Language: "go",
|
||||
},
|
||||
)
|
||||
},
|
||||
result: CodeResult{
|
||||
IncompleteResults: false,
|
||||
Items: initialize(0, 110, func(i int) Code {
|
||||
return Code{
|
||||
Name: fmt.Sprintf("name%d.go", i),
|
||||
}
|
||||
}),
|
||||
Total: 287,
|
||||
},
|
||||
httpStubs: func(reg *httpmock.Registry) {
|
||||
firstReq := httpmock.QueryMatcher("GET", "search/code", url.Values{
|
||||
"page": []string{"1"},
|
||||
"per_page": []string{"100"},
|
||||
"q": []string{"keyword language:go"},
|
||||
})
|
||||
firstRes := httpmock.JSONResponse(map[string]interface{}{
|
||||
"incomplete_results": false,
|
||||
"total_count": 287,
|
||||
"items": initialize(0, 100, func(i int) interface{} {
|
||||
return map[string]interface{}{
|
||||
"name": fmt.Sprintf("name%d.go", i),
|
||||
}
|
||||
}),
|
||||
})
|
||||
firstRes = httpmock.WithHeader(firstRes, "Link", `<https://api.github.com/search/code?page=2&per_page=100&q=org%3Agithub>; rel="next"`)
|
||||
secondReq := httpmock.QueryMatcher("GET", "search/code", url.Values{
|
||||
"page": []string{"2"},
|
||||
"per_page": []string{"29"},
|
||||
"per_page": []string{"100"},
|
||||
"q": []string{"keyword language:go"},
|
||||
},
|
||||
)
|
||||
secondRes := httpmock.JSONResponse(CodeResult{
|
||||
IncompleteResults: false,
|
||||
Items: []Code{{Name: "file2.go"}},
|
||||
Total: 2,
|
||||
},
|
||||
)
|
||||
})
|
||||
secondRes := httpmock.JSONResponse(map[string]interface{}{
|
||||
"incomplete_results": false,
|
||||
"total_count": 287,
|
||||
"items": initialize(100, 200, func(i int) interface{} {
|
||||
return map[string]interface{}{
|
||||
"name": fmt.Sprintf("name%d.go", i),
|
||||
}
|
||||
}),
|
||||
})
|
||||
reg.Register(firstReq, firstRes)
|
||||
reg.Register(secondReq, secondRes)
|
||||
},
|
||||
|
|
@ -201,10 +269,14 @@ func TestSearcherCommits(t *testing.T) {
|
|||
httpStubs: func(reg *httpmock.Registry) {
|
||||
reg.Register(
|
||||
httpmock.QueryMatcher("GET", "search/commits", values),
|
||||
httpmock.JSONResponse(CommitsResult{
|
||||
IncompleteResults: false,
|
||||
Items: []Commit{{Sha: "abc"}},
|
||||
Total: 1,
|
||||
httpmock.JSONResponse(map[string]interface{}{
|
||||
"incomplete_results": false,
|
||||
"total_count": 1,
|
||||
"items": []interface{}{
|
||||
map[string]interface{}{
|
||||
"sha": "abc",
|
||||
},
|
||||
},
|
||||
}),
|
||||
)
|
||||
},
|
||||
|
|
@ -221,10 +293,14 @@ func TestSearcherCommits(t *testing.T) {
|
|||
httpStubs: func(reg *httpmock.Registry) {
|
||||
reg.Register(
|
||||
httpmock.QueryMatcher("GET", "api/v3/search/commits", values),
|
||||
httpmock.JSONResponse(CommitsResult{
|
||||
IncompleteResults: false,
|
||||
Items: []Commit{{Sha: "abc"}},
|
||||
Total: 1,
|
||||
httpmock.JSONResponse(map[string]interface{}{
|
||||
"incomplete_results": false,
|
||||
"total_count": 1,
|
||||
"items": []interface{}{
|
||||
map[string]interface{}{
|
||||
"sha": "abc",
|
||||
},
|
||||
},
|
||||
}),
|
||||
)
|
||||
},
|
||||
|
|
@ -239,27 +315,92 @@ func TestSearcherCommits(t *testing.T) {
|
|||
},
|
||||
httpStubs: func(reg *httpmock.Registry) {
|
||||
firstReq := httpmock.QueryMatcher("GET", "search/commits", values)
|
||||
firstRes := httpmock.JSONResponse(CommitsResult{
|
||||
IncompleteResults: false,
|
||||
Items: []Commit{{Sha: "abc"}},
|
||||
Total: 2,
|
||||
},
|
||||
)
|
||||
firstRes = httpmock.WithHeader(firstRes, "Link", `<https://api.github.com/search/commits?page=2&per_page=100&q=org%3Agithub>; rel="next"`)
|
||||
firstRes := httpmock.JSONResponse(map[string]interface{}{
|
||||
"incomplete_results": false,
|
||||
"total_count": 2,
|
||||
"items": []interface{}{
|
||||
map[string]interface{}{
|
||||
"sha": "abc",
|
||||
},
|
||||
},
|
||||
})
|
||||
firstRes = httpmock.WithHeader(firstRes, "Link", `<https://api.github.com/search/commits?page=2&per_page=30&q=org%3Agithub>; rel="next"`)
|
||||
secondReq := httpmock.QueryMatcher("GET", "search/commits", url.Values{
|
||||
"page": []string{"2"},
|
||||
"per_page": []string{"29"},
|
||||
"per_page": []string{"30"},
|
||||
"order": []string{"desc"},
|
||||
"sort": []string{"committer-date"},
|
||||
"q": []string{"keyword author:foobar committer-date:>2021-02-28"},
|
||||
})
|
||||
secondRes := httpmock.JSONResponse(map[string]interface{}{
|
||||
"incomplete_results": false,
|
||||
"total_count": 2,
|
||||
"items": []interface{}{
|
||||
map[string]interface{}{
|
||||
"sha": "def",
|
||||
},
|
||||
},
|
||||
})
|
||||
reg.Register(firstReq, firstRes)
|
||||
reg.Register(secondReq, secondRes)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "collect full and partial pages under total number of matching search results",
|
||||
query: Query{
|
||||
Keywords: []string{"keyword"},
|
||||
Kind: "commits",
|
||||
Limit: 110,
|
||||
Order: "desc",
|
||||
Sort: "committer-date",
|
||||
Qualifiers: Qualifiers{
|
||||
Author: "foobar",
|
||||
CommitterDate: ">2021-02-28",
|
||||
},
|
||||
)
|
||||
secondRes := httpmock.JSONResponse(CommitsResult{
|
||||
IncompleteResults: false,
|
||||
Items: []Commit{{Sha: "def"}},
|
||||
Total: 2,
|
||||
},
|
||||
)
|
||||
},
|
||||
result: CommitsResult{
|
||||
IncompleteResults: false,
|
||||
Items: initialize(0, 110, func(i int) Commit {
|
||||
return Commit{
|
||||
Sha: strconv.Itoa(i),
|
||||
}
|
||||
}),
|
||||
Total: 287,
|
||||
},
|
||||
httpStubs: func(reg *httpmock.Registry) {
|
||||
firstReq := httpmock.QueryMatcher("GET", "search/commits", url.Values{
|
||||
"page": []string{"1"},
|
||||
"per_page": []string{"100"},
|
||||
"order": []string{"desc"},
|
||||
"sort": []string{"committer-date"},
|
||||
"q": []string{"keyword author:foobar committer-date:>2021-02-28"},
|
||||
})
|
||||
firstRes := httpmock.JSONResponse(map[string]interface{}{
|
||||
"incomplete_results": false,
|
||||
"total_count": 287,
|
||||
"items": initialize(0, 100, func(i int) map[string]interface{} {
|
||||
return map[string]interface{}{
|
||||
"sha": strconv.Itoa(i),
|
||||
}
|
||||
}),
|
||||
})
|
||||
firstRes = httpmock.WithHeader(firstRes, "Link", `<https://api.github.com/search/commits?page=2&per_page=100&q=org%3Agithub>; rel="next"`)
|
||||
secondReq := httpmock.QueryMatcher("GET", "search/commits", url.Values{
|
||||
"page": []string{"2"},
|
||||
"per_page": []string{"100"},
|
||||
"order": []string{"desc"},
|
||||
"sort": []string{"committer-date"},
|
||||
"q": []string{"keyword author:foobar committer-date:>2021-02-28"},
|
||||
})
|
||||
secondRes := httpmock.JSONResponse(map[string]interface{}{
|
||||
"incomplete_results": false,
|
||||
"total_count": 287,
|
||||
"items": initialize(100, 200, func(i int) map[string]interface{} {
|
||||
return map[string]interface{}{
|
||||
"sha": strconv.Itoa(i),
|
||||
}
|
||||
}),
|
||||
})
|
||||
reg.Register(firstReq, firstRes)
|
||||
reg.Register(secondReq, secondRes)
|
||||
},
|
||||
|
|
@ -269,8 +410,8 @@ func TestSearcherCommits(t *testing.T) {
|
|||
query: query,
|
||||
wantErr: true,
|
||||
errMsg: heredoc.Doc(`
|
||||
Invalid search query "keyword author:foobar committer-date:>2021-02-28".
|
||||
"blah" is not a recognized date/time format. Please provide an ISO 8601 date/time value, such as YYYY-MM-DD.`),
|
||||
Invalid search query "keyword author:foobar committer-date:>2021-02-28".
|
||||
"blah" is not a recognized date/time format. Please provide an ISO 8601 date/time value, such as YYYY-MM-DD.`),
|
||||
httpStubs: func(reg *httpmock.Registry) {
|
||||
reg.Register(
|
||||
httpmock.QueryMatcher("GET", "search/commits", values),
|
||||
|
|
@ -413,15 +554,14 @@ func TestSearcherRepositories(t *testing.T) {
|
|||
},
|
||||
},
|
||||
})
|
||||
firstRes = httpmock.WithHeader(firstRes, "Link", `<https://api.github.com/search/repositories?page=2&per_page=100&q=org%3Agithub>; rel="next"`)
|
||||
firstRes = httpmock.WithHeader(firstRes, "Link", `<https://api.github.com/search/repositories?page=2&per_page=30&q=org%3Agithub>; rel="next"`)
|
||||
secondReq := httpmock.QueryMatcher("GET", "search/repositories", url.Values{
|
||||
"page": []string{"2"},
|
||||
"per_page": []string{"29"},
|
||||
"per_page": []string{"30"},
|
||||
"order": []string{"desc"},
|
||||
"sort": []string{"stars"},
|
||||
"q": []string{"keyword stars:>=5 topic:topic"},
|
||||
},
|
||||
)
|
||||
})
|
||||
secondRes := httpmock.JSONResponse(map[string]interface{}{
|
||||
"incomplete_results": false,
|
||||
"total_count": 2,
|
||||
|
|
@ -435,13 +575,73 @@ func TestSearcherRepositories(t *testing.T) {
|
|||
reg.Register(secondReq, secondRes)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "collect full and partial pages under total number of matching search results",
|
||||
query: Query{
|
||||
Keywords: []string{"keyword"},
|
||||
Kind: "repositories",
|
||||
Limit: 110,
|
||||
Order: "desc",
|
||||
Sort: "stars",
|
||||
Qualifiers: Qualifiers{
|
||||
Stars: ">=5",
|
||||
Topic: []string{"topic"},
|
||||
},
|
||||
},
|
||||
result: RepositoriesResult{
|
||||
IncompleteResults: false,
|
||||
Items: initialize(0, 110, func(i int) Repository {
|
||||
return Repository{
|
||||
Name: fmt.Sprintf("name%d", i),
|
||||
}
|
||||
}),
|
||||
Total: 287,
|
||||
},
|
||||
httpStubs: func(reg *httpmock.Registry) {
|
||||
firstReq := httpmock.QueryMatcher("GET", "search/repositories", url.Values{
|
||||
"page": []string{"1"},
|
||||
"per_page": []string{"100"},
|
||||
"order": []string{"desc"},
|
||||
"sort": []string{"stars"},
|
||||
"q": []string{"keyword stars:>=5 topic:topic"},
|
||||
})
|
||||
firstRes := httpmock.JSONResponse(map[string]interface{}{
|
||||
"incomplete_results": false,
|
||||
"total_count": 287,
|
||||
"items": initialize(0, 100, func(i int) interface{} {
|
||||
return map[string]interface{}{
|
||||
"name": fmt.Sprintf("name%d", i),
|
||||
}
|
||||
}),
|
||||
})
|
||||
firstRes = httpmock.WithHeader(firstRes, "Link", `<https://api.github.com/search/repositories?page=2&per_page=100&q=org%3Agithub>; rel="next"`)
|
||||
secondReq := httpmock.QueryMatcher("GET", "search/repositories", url.Values{
|
||||
"page": []string{"2"},
|
||||
"per_page": []string{"100"},
|
||||
"order": []string{"desc"},
|
||||
"sort": []string{"stars"},
|
||||
"q": []string{"keyword stars:>=5 topic:topic"},
|
||||
})
|
||||
secondRes := httpmock.JSONResponse(map[string]interface{}{
|
||||
"incomplete_results": false,
|
||||
"total_count": 287,
|
||||
"items": initialize(100, 200, func(i int) interface{} {
|
||||
return map[string]interface{}{
|
||||
"name": fmt.Sprintf("name%d", i),
|
||||
}
|
||||
}),
|
||||
})
|
||||
reg.Register(firstReq, firstRes)
|
||||
reg.Register(secondReq, secondRes)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "handles search errors",
|
||||
query: query,
|
||||
wantErr: true,
|
||||
errMsg: heredoc.Doc(`
|
||||
Invalid search query "keyword stars:>=5 topic:topic".
|
||||
"blah" is not a recognized date/time format. Please provide an ISO 8601 date/time value, such as YYYY-MM-DD.`),
|
||||
Invalid search query "keyword stars:>=5 topic:topic".
|
||||
"blah" is not a recognized date/time format. Please provide an ISO 8601 date/time value, such as YYYY-MM-DD.`),
|
||||
httpStubs: func(reg *httpmock.Registry) {
|
||||
reg.Register(
|
||||
httpmock.QueryMatcher("GET", "search/repositories", values),
|
||||
|
|
@ -529,10 +729,14 @@ func TestSearcherIssues(t *testing.T) {
|
|||
httpStubs: func(reg *httpmock.Registry) {
|
||||
reg.Register(
|
||||
httpmock.QueryMatcher("GET", "search/issues", values),
|
||||
httpmock.JSONResponse(IssuesResult{
|
||||
IncompleteResults: false,
|
||||
Items: []Issue{{Number: 1234}},
|
||||
Total: 1,
|
||||
httpmock.JSONResponse(map[string]interface{}{
|
||||
"incomplete_results": false,
|
||||
"total_count": 1,
|
||||
"items": []interface{}{
|
||||
map[string]interface{}{
|
||||
"number": 1234,
|
||||
},
|
||||
},
|
||||
}),
|
||||
)
|
||||
},
|
||||
|
|
@ -549,10 +753,14 @@ func TestSearcherIssues(t *testing.T) {
|
|||
httpStubs: func(reg *httpmock.Registry) {
|
||||
reg.Register(
|
||||
httpmock.QueryMatcher("GET", "api/v3/search/issues", values),
|
||||
httpmock.JSONResponse(IssuesResult{
|
||||
IncompleteResults: false,
|
||||
Items: []Issue{{Number: 1234}},
|
||||
Total: 1,
|
||||
httpmock.JSONResponse(map[string]interface{}{
|
||||
"incomplete_results": false,
|
||||
"total_count": 1,
|
||||
"items": []interface{}{
|
||||
map[string]interface{}{
|
||||
"number": 1234,
|
||||
},
|
||||
},
|
||||
}),
|
||||
)
|
||||
},
|
||||
|
|
@ -567,27 +775,92 @@ func TestSearcherIssues(t *testing.T) {
|
|||
},
|
||||
httpStubs: func(reg *httpmock.Registry) {
|
||||
firstReq := httpmock.QueryMatcher("GET", "search/issues", values)
|
||||
firstRes := httpmock.JSONResponse(IssuesResult{
|
||||
IncompleteResults: false,
|
||||
Items: []Issue{{Number: 1234}},
|
||||
Total: 2,
|
||||
},
|
||||
)
|
||||
firstRes = httpmock.WithHeader(firstRes, "Link", `<https://api.github.com/search/issues?page=2&per_page=100&q=org%3Agithub>; rel="next"`)
|
||||
firstRes := httpmock.JSONResponse(map[string]interface{}{
|
||||
"incomplete_results": false,
|
||||
"total_count": 2,
|
||||
"items": []interface{}{
|
||||
map[string]interface{}{
|
||||
"number": 1234,
|
||||
},
|
||||
},
|
||||
})
|
||||
firstRes = httpmock.WithHeader(firstRes, "Link", `<https://api.github.com/search/issues?page=2&per_page=30&q=org%3Agithub>; rel="next"`)
|
||||
secondReq := httpmock.QueryMatcher("GET", "search/issues", url.Values{
|
||||
"page": []string{"2"},
|
||||
"per_page": []string{"29"},
|
||||
"per_page": []string{"30"},
|
||||
"order": []string{"desc"},
|
||||
"sort": []string{"comments"},
|
||||
"q": []string{"keyword is:locked is:public language:go"},
|
||||
})
|
||||
secondRes := httpmock.JSONResponse(map[string]interface{}{
|
||||
"incomplete_results": false,
|
||||
"total_count": 2,
|
||||
"items": []interface{}{
|
||||
map[string]interface{}{
|
||||
"number": 5678,
|
||||
},
|
||||
},
|
||||
})
|
||||
reg.Register(firstReq, firstRes)
|
||||
reg.Register(secondReq, secondRes)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "collect full and partial pages under total number of matching search results",
|
||||
query: Query{
|
||||
Keywords: []string{"keyword"},
|
||||
Kind: "issues",
|
||||
Limit: 110,
|
||||
Order: "desc",
|
||||
Sort: "comments",
|
||||
Qualifiers: Qualifiers{
|
||||
Language: "go",
|
||||
Is: []string{"public", "locked"},
|
||||
},
|
||||
)
|
||||
secondRes := httpmock.JSONResponse(IssuesResult{
|
||||
IncompleteResults: false,
|
||||
Items: []Issue{{Number: 5678}},
|
||||
Total: 2,
|
||||
},
|
||||
)
|
||||
},
|
||||
result: IssuesResult{
|
||||
IncompleteResults: false,
|
||||
Items: initialize(0, 110, func(i int) Issue {
|
||||
return Issue{
|
||||
Number: i,
|
||||
}
|
||||
}),
|
||||
Total: 287,
|
||||
},
|
||||
httpStubs: func(reg *httpmock.Registry) {
|
||||
firstReq := httpmock.QueryMatcher("GET", "search/issues", url.Values{
|
||||
"page": []string{"1"},
|
||||
"per_page": []string{"100"},
|
||||
"order": []string{"desc"},
|
||||
"sort": []string{"comments"},
|
||||
"q": []string{"keyword is:locked is:public language:go"},
|
||||
})
|
||||
firstRes := httpmock.JSONResponse(map[string]interface{}{
|
||||
"incomplete_results": false,
|
||||
"total_count": 287,
|
||||
"items": initialize(0, 100, func(i int) interface{} {
|
||||
return map[string]interface{}{
|
||||
"number": i,
|
||||
}
|
||||
}),
|
||||
})
|
||||
firstRes = httpmock.WithHeader(firstRes, "Link", `<https://api.github.com/search/issues?page=2&per_page=100&q=org%3Agithub>; rel="next"`)
|
||||
secondReq := httpmock.QueryMatcher("GET", "search/issues", url.Values{
|
||||
"page": []string{"2"},
|
||||
"per_page": []string{"100"},
|
||||
"order": []string{"desc"},
|
||||
"sort": []string{"comments"},
|
||||
"q": []string{"keyword is:locked is:public language:go"},
|
||||
})
|
||||
secondRes := httpmock.JSONResponse(map[string]interface{}{
|
||||
"incomplete_results": false,
|
||||
"total_count": 287,
|
||||
"items": initialize(100, 200, func(i int) interface{} {
|
||||
return map[string]interface{}{
|
||||
"number": i,
|
||||
}
|
||||
}),
|
||||
})
|
||||
reg.Register(firstReq, firstRes)
|
||||
reg.Register(secondReq, secondRes)
|
||||
},
|
||||
|
|
@ -597,8 +870,8 @@ func TestSearcherIssues(t *testing.T) {
|
|||
query: query,
|
||||
wantErr: true,
|
||||
errMsg: heredoc.Doc(`
|
||||
Invalid search query "keyword is:locked is:public language:go".
|
||||
"blah" is not a recognized date/time format. Please provide an ISO 8601 date/time value, such as YYYY-MM-DD.`),
|
||||
Invalid search query "keyword is:locked is:public language:go".
|
||||
"blah" is not a recognized date/time format. Please provide an ISO 8601 date/time value, such as YYYY-MM-DD.`),
|
||||
httpStubs: func(reg *httpmock.Registry) {
|
||||
reg.Register(
|
||||
httpmock.QueryMatcher("GET", "search/issues", values),
|
||||
|
|
@ -686,3 +959,12 @@ func TestSearcherURL(t *testing.T) {
|
|||
})
|
||||
}
|
||||
}
|
||||
|
||||
// initialize generate slices over a range for test scenarios using the provided initializer.
|
||||
func initialize[T any](start int, stop int, initializer func(i int) T) []T {
|
||||
results := make([]T, 0, (stop - start))
|
||||
for i := start; i < stop; i++ {
|
||||
results = append(results, initializer(i))
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue