cli/pkg/search/searcher.go

package search

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"

	"github.com/cli/cli/v2/internal/ghinstance"
)

const (
	maxPerPage = 100
	orderKey   = "order"
	sortKey    = "sort"
)

var linkRE = regexp.MustCompile(`<([^>]+)>;\s*rel="([^"]+)"`)
var pageRE = regexp.MustCompile(`(\?|&)page=(\d*)`)
var jsonTypeRE = regexp.MustCompile(`[/+]json($|;)`)

//go:generate moq -rm -out searcher_mock.go . Searcher
type Searcher interface {
	Code(Query) (CodeResult, error)
	Commits(Query) (CommitsResult, error)
	Repositories(Query) (RepositoriesResult, error)
	Issues(Query) (IssuesResult, error)
	URL(Query) string
}

type searcher struct {
	client *http.Client
	host   string
}

type httpError struct {
	Errors     []httpErrorItem
	Message    string
	RequestURL *url.URL
	StatusCode int
}

type httpErrorItem struct {
	Code     string
	Field    string
	Message  string
	Resource string
}

func NewSearcher(client *http.Client, host string) Searcher {
	return &searcher{
		client: client,
		host:   host,
	}
}

func (s searcher) Code(query Query) (CodeResult, error) {
	result := CodeResult{}
	toRetrieve := query.Limit
	var resp *http.Response
	var err error
	for toRetrieve > 0 {
		query.Limit = min(toRetrieve, maxPerPage)
		query.Page = nextPage(resp)
		if query.Page == 0 {
			break
		}
		page := CodeResult{}
		resp, err = s.search(query, &page)
		if err != nil {
			return result, err
		}
		result.IncompleteResults = page.IncompleteResults
		result.Total = page.Total
		result.Items = append(result.Items, page.Items...)
		toRetrieve = toRetrieve - len(page.Items)
	}
	return result, nil
}

func (s searcher) Commits(query Query) (CommitsResult, error) {
	result := CommitsResult{}
	toRetrieve := query.Limit
	var resp *http.Response
	var err error
	for toRetrieve > 0 {
		query.Limit = min(toRetrieve, maxPerPage)
		query.Page = nextPage(resp)
		if query.Page == 0 {
			break
		}
		page := CommitsResult{}
		resp, err = s.search(query, &page)
		if err != nil {
			return result, err
		}
		result.IncompleteResults = page.IncompleteResults
		result.Total = page.Total
		result.Items = append(result.Items, page.Items...)
		toRetrieve = toRetrieve - len(page.Items)
	}
	return result, nil
}

func (s searcher) Repositories(query Query) (RepositoriesResult, error) {
	result := RepositoriesResult{}
	toRetrieve := query.Limit
	var resp *http.Response
	var err error
	for toRetrieve > 0 {
		query.Limit = min(toRetrieve, maxPerPage)
		query.Page = nextPage(resp)
		if query.Page == 0 {
			break
		}
		page := RepositoriesResult{}
		resp, err = s.search(query, &page)
		if err != nil {
			return result, err
		}
		result.IncompleteResults = page.IncompleteResults
		result.Total = page.Total
		result.Items = append(result.Items, page.Items...)
		toRetrieve = toRetrieve - len(page.Items)
	}
	return result, nil
}

func (s searcher) Issues(query Query) (IssuesResult, error) {
	result := IssuesResult{}
	toRetrieve := query.Limit
	var resp *http.Response
	var err error
	for toRetrieve > 0 {
		query.Limit = min(toRetrieve, maxPerPage)
		query.Page = nextPage(resp)
		if query.Page == 0 {
			break
		}
		page := IssuesResult{}
		resp, err = s.search(query, &page)
		if err != nil {
			return result, err
		}
		result.IncompleteResults = page.IncompleteResults
		result.Total = page.Total
		result.Items = append(result.Items, page.Items...)
		toRetrieve = toRetrieve - len(page.Items)
	}
	return result, nil
}

func (s searcher) search(query Query, result interface{}) (*http.Response, error) {
	path := fmt.Sprintf("%ssearch/%s", ghinstance.RESTPrefix(s.host), query.Kind)
	qs := url.Values{}
	qs.Set("page", strconv.Itoa(query.Page))
	qs.Set("per_page", strconv.Itoa(query.Limit))
	qs.Set("q", query.String())
	if query.Order != "" {
		qs.Set(orderKey, query.Order)
	}
	if query.Sort != "" {
		qs.Set(sortKey, query.Sort)
	}
	url := fmt.Sprintf("%s?%s", path, qs.Encode())
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", "application/json; charset=utf-8")
	req.Header.Set("Accept", "application/vnd.github.v3+json")
	if query.Kind == KindCode {
		req.Header.Set("Accept", "application/vnd.github.text-match+json")
	}

	resp, err := s.client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	success := resp.StatusCode >= 200 && resp.StatusCode < 300
	if !success {
		return resp, handleHTTPError(resp)
	}
	decoder := json.NewDecoder(resp.Body)
	err = decoder.Decode(result)
	if err != nil {
		return resp, err
	}
	return resp, nil
}

func (s searcher) URL(query Query) string {
	path := fmt.Sprintf("https://%s/search", s.host)
	qs := url.Values{}
	qs.Set("type", query.Kind)
	qs.Set("q", query.String())
	if query.Order != "" {
		qs.Set(orderKey, query.Order)
	}
	if query.Sort != "" {
		qs.Set(sortKey, query.Sort)
	}
	url := fmt.Sprintf("%s?%s", path, qs.Encode())
	return url
}

func (err httpError) Error() string {
	if err.StatusCode != 422 || len(err.Errors) == 0 {
		return fmt.Sprintf("HTTP %d: %s (%s)", err.StatusCode, err.Message, err.RequestURL)
	}
	query := strings.TrimSpace(err.RequestURL.Query().Get("q"))
	return fmt.Sprintf("Invalid search query %q.\n%s", query, err.Errors[0].Message)
}

func handleHTTPError(resp *http.Response) error {
	httpError := httpError{
		RequestURL: resp.Request.URL,
		StatusCode: resp.StatusCode,
	}
	if !jsonTypeRE.MatchString(resp.Header.Get("Content-Type")) {
		httpError.Message = resp.Status
		return httpError
	}
	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	if err := json.Unmarshal(body, &httpError); err != nil {
		return err
	}
	return httpError
}

func nextPage(resp *http.Response) (page int) {
	if resp == nil {
		return 1
	}
	for _, m := range linkRE.FindAllStringSubmatch(resp.Header.Get("Link"), -1) {
		if !(len(m) > 2 && m[2] == "next") {
			continue
		}
		p := pageRE.FindStringSubmatch(m[1])
		if len(p) == 3 {
			i, err := strconv.Atoi(p[2])
			if err == nil {
				return i
			}
		}
	}
	return 0
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}