diff --git a/.github/workflows/scripts/spam-detection/check-issue.sh b/.github/workflows/scripts/spam-detection/check-issue.sh new file mode 100755 index 000000000..d2c60d8fe --- /dev/null +++ b/.github/workflows/scripts/spam-detection/check-issue.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Check if an issue is spam or not and output "PASS" (not spam) or "FAIL" (spam). +# +# Regardless of the spam detection result, the script always exits with a zero +# exit code, unless there's a runtime error. +# +# This script must be run from the root directory of the repository. + +set -euo pipefail + +_prompt_file=".github/workflows/scripts/spam-detection/prompt.yml" +_generate_sys_prompt_script=".github/workflows/scripts/spam-detection/generate-sys-prompt.sh" +_generate_prompt_script=".github/workflows/scripts/spam-detection/generate-prompt.sh" + +_issue_url="$1" +if [[ -z "$_issue_url" ]]; then + echo "error: issue URL is empty" >&2 + exit 1 +fi + +_issue="$(gh issue view --json title,body "$_issue_url")" + +_issue_body="$(jq -r ".body" <<< "$_issue")" +_issue_title="$(jq -r ".title" <<< "$_issue")" + +_system_prompt="$($_generate_sys_prompt_script)" +_input_prompt="$($_generate_prompt_script "$_issue_title" "$_issue_body")" + +_updated_prompt_file_content="$( + cat "$_prompt_file" | + yq eval 'del(.testData, .evaluators)' | # drop test data + _system="$_system_prompt" _input="$_input_prompt" yq eval ".messages[0].content = strenv(_system) | .messages[1].content = strenv(_input)" +)" + +gh extension install github/gh-models 2>/dev/null + +_result="$(gh models run --file <(echo "$_updated_prompt_file_content") | cat)" + +if [[ "$_result" != "PASS" && "$_result" != "FAIL" ]]; then + echo "error: expected PASS or FAIL but got an unexpected result: $_result" >&2 + exit 1 +fi + +echo "$_result" diff --git a/.github/workflows/scripts/spam-detection/eval.sh b/.github/workflows/scripts/spam-detection/eval.sh new file mode 100755 index 000000000..f4a5dc0f7 --- /dev/null +++ b/.github/workflows/scripts/spam-detection/eval.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Run the eval tests for the spam detection AI model. +# +# This script must be run from the root directory of the repository. + +set -euo pipefail + +_prompt_file=".github/workflows/scripts/spam-detection/prompt.yml" +_generate_sys_prompt_script=".github/workflows/scripts/spam-detection/generate-sys-prompt.sh" + +_system_prompt="$($_generate_sys_prompt_script)" +_updated_prompt_file="$(_value="$_system_prompt" yq eval '.messages[0].content = strenv(_value)' "$_prompt_file")" + +# We should be able to just run the following command: +# +# ``` +# gh models eval <(echo "$_updated_prompt_file") +# ``` +# +# But since `gh-models` does not throttle the rate of API requests, we need to +# modify the extension code and introduce a deliberate delay between the runs. +# Here, we assume a binary of the `gh-models` extension (with appropriate +# throttling) is available in the root directory of the repository and we're +# calling it directly (not though `gh`). +./gh-models eval <(echo "$_updated_prompt_file") diff --git a/.github/workflows/scripts/spam-detection/generate-prompt.sh b/.github/workflows/scripts/spam-detection/generate-prompt.sh new file mode 100755 index 000000000..9437bf751 --- /dev/null +++ b/.github/workflows/scripts/spam-detection/generate-prompt.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Generate the prompt for the spam detection AI model. The issue title and body +# should be provided as arguments. +# +# This script must be run from the root directory of the repository. + +set -euo pipefail + +_issue_title="$1" +_issue_body="$2" + +_prompt=" +
\n\n```\n❯ gh run view | cat \nrun or job ID required when not running interactively\n\nUsage: gh run view [
\r\n\r\n```\r\n$ GH_DEBUG=api gh extensions install https://github.com/IvanRibakov/gh-workflow-stats --pin f2286ac --force\r\n* Request at 2024-10-15 13:12:09.106395563 +0200 CEST m=+0.050985944\r\n* Request to https://api.github.com/repos/IvanRibakov/gh-workflow-stats/releases/latest\r\n> GET /repos/IvanRibakov/gh-workflow-stats/releases/latest HTTP/1.1\r\n> Host: api.github.com\r\n> Accept: application/vnd.github.merge-info-preview+json, application/vnd.github.nebula-preview\r\n> Authorization: token ████████████████████\r\n> Content-Type: application/json; charset=utf-8\r\n> Time-Zone: Europe/Madrid\r\n> User-Agent: GitHub CLI 2.54.0\r\n> X-Gh-Cache-Ttl: 30s\r\n\r\n⢿< HTTP/2.0 404 Not Found\r\n< Access-Control-Allow-Origin: *\r\n< Access-Control-Expose-Headers: ETag, Link, Location, Retry-After, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Used, X-RateLimit-Resource, X-RateLimit-Reset, X-OAuth-Scopes, X-Accepted-OAuth-Scopes, X-Poll-Interval, X-GitHub-Media-Type, X-GitHub-SSO, X-GitHub-Request-Id, Deprecation, Sunset\r\n< Content-Security-Policy: default-src 'none'\r\n< Content-Type: application/json; charset=utf-8\r\n< Date: Tue, 15 Oct 2024 11:12:14 GMT\r\n< Referrer-Policy: origin-when-cross-origin, strict-origin-when-cross-origin\r\n< Server: github.com\r\n< Strict-Transport-Security: max-age=31536000; includeSubdomains; preload\r\n< Vary: Accept-Encoding, Accept, X-Requested-With\r\n< X-Accepted-Oauth-Scopes: repo\r\n< X-Content-Type-Options: nosniff\r\n< X-Frame-Options: deny\r\n< X-Github-Api-Version-Selected: 2022-11-28\r\n< X-Github-Media-Type: github.v3; param=merge-info-preview.nebula-preview; format=json\r\n< X-Github-Request-Id: 9D5A:37C4BC:D9552DD:DCC2930:670E4E0E\r\n< X-Oauth-Scopes: delete:packages, read:org, repo, workflow, write:packages\r\n< X-Ratelimit-Limit: 5000\r\n< X-Ratelimit-Remaining: 4955\r\n< X-Ratelimit-Reset: 1728991844\r\n< X-Ratelimit-Resource: core\r\n< X-Ratelimit-Used: 45\r\n< X-Xss-Protection: 0\r\n\r\n\r\n{\r\n \"message\": \"Not Found\",\r\n \"documentation_url\": \"https://docs.github.com/rest/releases/releases#get-the-latest-release\",\r\n \"status\": \"404\"\r\n}\r\n\r\n* Request took 5.302112651s\r\n* Request at 2024-10-15 13:12:14.408536288 +0200 CEST m=+5.353126669\r\n* Request to https://api.github.com/repos/IvanRibakov/gh-workflow-stats\r\n> GET /repos/IvanRibakov/gh-workflow-stats HTTP/1.1\r\n> Host: api.github.com\r\n> Accept: application/vnd.github.merge-info-preview+json, application/vnd.github.nebula-preview\r\n> Authorization: token ████████████████████\r\n> Content-Type: application/json; charset=utf-8\r\n> Time-Zone: Europe/Madrid\r\n> User-Agent: GitHub CLI 2.54.0\r\n> X-Gh-Cache-Ttl: 30s\r\n\r\n...\r\n\r\n{\r\n ...\r\n \"name\": \"gh-workflow-stats\",\r\n \"full_name\": \"IvanRibakov/gh-workflow-stats\",\r\n \"private\": false,\r\n ...\r\n}\r\n\r\n* Request took 254.775396ms\r\n* Request at 2024-10-15 13:12:14.663345618 +0200 CEST m=+5.607935989\r\n* Request to https://api.github.com/repos/IvanRibakov/gh-workflow-stats/contents/gh-workflow-stats\r\n> GET /repos/IvanRibakov/gh-workflow-stats/contents/gh-workflow-stats HTTP/1.1\r\n> Host: api.github.com\r\n> Accept: application/vnd.github.merge-info-preview+json, application/vnd.github.nebula-preview\r\n> Authorization: token ████████████████████\r\n> Content-Type: application/json; charset=utf-8\r\n> Time-Zone: Europe/Madrid\r\n> User-Agent: GitHub CLI 2.54.0\r\n> X-Gh-Cache-Ttl: 30s\r\n\r\n⣾< HTTP/2.0 404 Not Found\r\n< Access-Control-Allow-Origin: *\r\n< Access-Control-Expose-Headers: ETag, Link, Location, Retry-After, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Used, X-RateLimit-Resource, X-RateLimit-Reset, X-OAuth-Scopes, X-Accepted-OAuth-Scopes, X-Poll-Interval, X-GitHub-Media-Type, X-GitHub-SSO, X-GitHub-Request-Id, Deprecation, Sunset\r\n< Content-Security-Policy: default-src 'none'\r\n< Content-Type: application/json; charset=utf-8\r\n< Date: Tue, 15 Oct 2024 11:12:14 GMT\r\n< Referrer-Policy: origin-when-cross-origin, strict-origin-when-cross-origin\r\n< Server: github.com\r\n< Strict-Transport-Security: max-age=31536000; includeSubdomains; preload\r\n< Vary: Accept-Encoding, Accept, X-Requested-With\r\n< X-Accepted-Oauth-Scopes: \r\n< X-Content-Type-Options: nosniff\r\n< X-Frame-Options: deny\r\n< X-Github-Api-Version-Selected: 2022-11-28\r\n< X-Github-Media-Type: github.v3; param=merge-info-preview.nebula-preview; format=json\r\n< X-Github-Request-Id: 9D5A:37C4BC:D95549D:DCC2AE7:670E4E0E\r\n< X-Oauth-Scopes: delete:packages, read:org, repo, workflow, write:packages\r\n< X-Ratelimit-Limit: 5000\r\n< X-Ratelimit-Remaining: 4953\r\n< X-Ratelimit-Reset: 1728991844\r\n< X-Ratelimit-Resource: core\r\n< X-Ratelimit-Used: 47\r\n< X-Xss-Protection: 0\r\n\r\n{\r\n \"message\": \"Not Found\",\r\n \"documentation_url\": \"https://docs.github.com/rest/repos/contents#get-repository-content\",\r\n \"status\": \"404\"\r\n}\r\n\r\n* Request took 282.202949ms\r\nextension is not installable: missing executable\r\n```\r\n\r\n
\r\ngit_protocol: the protocol to use for git clone and push operations {https | ssh} (default https) |