21 lines
849 B
Bash
Executable file
21 lines
849 B
Bash
Executable file
#!/bin/bash
|
|
|
|
# Run the eval tests for the spam detection AI model.
|
|
#
|
|
# This script must be run from the root directory of the repository.
|
|
|
|
set -euo pipefail
|
|
|
|
# Determine absolute path to script directory based on where it is called from.
|
|
# This allows the script to be run from any directory.
|
|
SPAM_DIR="$(dirname "$(realpath "$0")")"
|
|
|
|
# Generate dynamic prompts for inference
|
|
_system_prompt="$($SPAM_DIR/generate-sys-prompt.sh)"
|
|
_final_prompt="$(_value="$_system_prompt" yq eval '.messages[0].content = strenv(_value)' $SPAM_DIR/eval-prompts.yml)"
|
|
|
|
# The following `gh models eval` command will fail after 20 requests due to rate limits.
|
|
# We are going to open up an issue in `github/gh-models` to address this.
|
|
#
|
|
# TODO: break up `eval-prompts.yml` file into smaller batches to avoid hitting the rate limit.
|
|
gh models eval <(echo "$_final_prompt")
|