Merge pull request #9782 from cli/andyfeller/testscripts-secret

Add acceptance tests for `secret` commands
2024-10-21 15:33:31 -04:00 · 2024-10-21 15:33:31 -04:00 · 1886d1b44b
commit 1886d1b44b
parent 857854d1b3 b3c0c39f47
5 changed files with 352 additions and 2 deletions
--- a/acceptance/README.md
+++ b/acceptance/README.md
@ -57,10 +57,52 @@ The following custom environment variables are made available to the scripts:
 * `ORG`: Set to the value of the `GH_ACCEPTANCE_ORG` env var provided to `go test`
 * `GH_TOKEN`: Set to the value of the `GH_ACCEPTANCE_TOKEN` env var provided to `go test`
 * `RANDOM_STRING`: Set to a length 10 random string of letters to help isolate globally visible resources
- * `SCRIPT_NAME`: Set to the name of the `testscript` currently running, without extension e.g. `pr-view`
+ * `SCRIPT_NAME`: Set to the name of the `testscript` currently running, without extension and replacing hyphens with underscores e.g. `pr_view`
 * `HOME`: Set to the initial working directory. Required for `git` operations
 * `GH_CONFIG_DIR`: Set to the initial working directory. Required for `gh` operations

+#### Custom Commands
+
+The following custom commands are defined within [`acceptance_test.go`](./acceptance_test.go) to help with writing tests:
+
+- `defer`: register a command to run after the testscript completes
+
+  ```txtar
+  # Defer repo cleanup
+  defer gh repo delete --yes $ORG/$SCRIPT_NAME-$RANDOM_STRING
+  ```
+
+- `env2upper`: set environment variable to the uppercase version of another environment variable
+
+  ```txtar
+  # Prepare organization secret, GitHub Actions uppercases secret names
+  env2upper ORG_SECRET_NAME=$RANDOM_STRING
+  ```
+
+- `replace`: replace placeholders in file with interpolated content provided
+
+  ```txtar
+  env2upper SECRET_NAME=$SCRIPT_NAME_$RANDOM_STRING
+
+  # Modify workflow file to use generated organization secret name
+  mv ../workflow.yml .github/workflows/workflow.yml
+  replace .github/workflows/workflow.yml SECRET_NAME=$SECRET_NAME
+
+  -- workflow.yml --
+  on:
+    workflow_dispatch:
+  env:
+    ORG_SECRET: ${{ secrets.$SECRET_NAME }}
+  ```
+
+- `stdout2env`: set environment variable containing standard output from previous command
+
+  ```txtar
+  # Create the PR
+  exec gh pr create --title 'Feature Title' --body 'Feature Body' --assignee '@me' --label 'bug'
+  stdout2env PR_URL
+  ```
+
 ### Acceptance Test VS Code Support

 Due to the `//go:build acceptance` build constraint, some functionality is limited because `gopls` isn't being informed about the tag. To resolve this, set the following in your `settings.json`:
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@ -90,6 +90,15 @@ func TestRepo(t *testing.T) {
 	testscript.Run(t, testScriptParamsFor(tsEnv, "repo"))
 }

+func TestSecrets(t *testing.T) {
+	var tsEnv testScriptEnv
+	if err := tsEnv.fromEnv(); err != nil {
+		t.Fatal(err)
+	}
+
+	testscript.Run(t, testScriptParamsFor(tsEnv, "secret"))
+}
+
 func testScriptParamsFor(tsEnv testScriptEnv, command string) testscript.Params {
 	var files []string
 	if tsEnv.script != "" {
@ -120,7 +129,11 @@ func sharedSetup(tsEnv testScriptEnv) func(ts *testscript.Env) error {
 		if !ok {
 			ts.T().Fatal("script name not found")
 		}
-		ts.Setenv("SCRIPT_NAME", scriptName)
+
+		// When using script name to uniquely identify where test data comes from,
+		// some places like GitHub Actions secret names don't accept hyphens.
+		// Replace them with underscores until such a time this becomes a problem.
+		ts.Setenv("SCRIPT_NAME", strings.ReplaceAll(scriptName, "-", "_"))
 		ts.Setenv("HOME", ts.Cd)
 		ts.Setenv("GH_CONFIG_DIR", ts.Cd)

@ -164,6 +177,60 @@ func sharedCmds(tsEnv testScriptEnv) map[string]func(ts *testscript.TestScript,
 				}
 			})
 		},
+		"env2upper": func(ts *testscript.TestScript, neg bool, args []string) {
+			if neg {
+				ts.Fatalf("unsupported: ! env2upper")
+			}
+			if len(args) == 0 {
+				ts.Fatalf("usage: env2upper name=value ...")
+			}
+			for _, env := range args {
+				i := strings.Index(env, "=")
+
+				if i < 0 {
+					ts.Fatalf("env2upper: argument does not match name=value")
+				}
+
+				ts.Setenv(env[:i], strings.ToUpper(env[i+1:]))
+			}
+		},
+		"replace": func(ts *testscript.TestScript, neg bool, args []string) {
+			if neg {
+				ts.Fatalf("unsupported: ! replace")
+			}
+			if len(args) < 2 {
+				ts.Fatalf("usage: replace file env...")
+			}
+
+			src := ts.MkAbs(args[0])
+			ts.Logf("replace src: %s", src)
+
+			// Preserve the existing file mode while replacing the contents similar to native cp behavior
+			info, err := os.Stat(src)
+			ts.Check(err)
+			mode := info.Mode() & 0o777
+			data, err := os.ReadFile(src)
+			ts.Check(err)
+
+			for _, arg := range args[1:] {
+				i := strings.Index(arg, "=")
+				if i < 0 {
+					ts.Fatalf("replace: %s argument does not match name=value", arg)
+				}
+
+				name := fmt.Sprintf("$%s", arg[:i])
+				value := arg[i+1:]
+				ts.Logf("replace %s: %s", name, value)
+
+				// `replace` was originally built similar to `cp` and `cmpenv`, expanding environment variables within a file.
+				// However files with content that looks like environments variable such as GitHub Actions workflows
+				// were being modified unexpectedly. Thus `replace` has been designed to using string replacement
+				// looking for `$KEY` specifically.
+				data = []byte(strings.ReplaceAll(string(data), name, value))
+			}
+
+			ts.Check(os.WriteFile(src, data, mode))
+		},
 		"stdout2env": func(ts *testscript.TestScript, neg bool, args []string) {
 			if neg {
 				ts.Fatalf("unsupported: ! stdout2env")
--- a/acceptance/testdata/secret/secret-org.txtar
+++ b/acceptance/testdata/secret/secret-org.txtar
@ -0,0 +1,85 @@
+# Setup environment variables used for testscript
+env REPO=${SCRIPT_NAME}-${RANDOM_STRING}
+env2upper SECRET_NAME=${SCRIPT_NAME}_${RANDOM_STRING}
+
+# Use gh as a credential helper
+exec gh auth setup-git
+
+# Create a repository with a file so it has a default branch
+exec gh repo create $ORG/$REPO --add-readme --private
+
+# Defer repo cleanup
+defer gh repo delete --yes $ORG/$REPO
+
+# Clone the repo
+exec gh repo clone $ORG/$REPO
+cd $REPO
+
+# Confirm organization secret does not exist, will fail admin:org scope missing
+exec gh secret list --org $ORG
+! stdout $SECRET_NAME
+
+# Create an organization secret
+exec gh secret set $SECRET_NAME --org $ORG --body 'just an organization secret' --repos $REPO
+
+# Defer organization secret cleanup
+defer gh secret delete $SECRET_NAME --org $ORG
+
+# Verify new organization secret exists
+exec gh secret list --org $ORG
+stdout $SECRET_NAME
+
+# Commit workflow file creating dispatchable workflow able to verify secret matches
+mkdir .github/workflows
+mv ../workflow.yml .github/workflows/workflow.yml
+replace .github/workflows/workflow.yml SECRET_NAME=$SECRET_NAME
+exec git add .github/workflows/workflow.yml
+exec git commit -m 'Create workflow file'
+exec git push -u origin main
+
+# Sleep because it takes a second for the workflow to register
+sleep 1
+
+# Check the workflow is indeed created
+exec gh workflow list
+stdout 'Test Workflow Name'
+
+# Run the workflow
+exec gh workflow run 'Test Workflow Name'
+
+# It takes some time for a workflow run to register
+sleep 10
+
+# Get the run ID we want to watch & delete
+exec gh run list --json databaseId --jq '.[0].databaseId'
+stdout2env RUN_ID
+
+# Wait for workflow to complete
+exec gh run watch $RUN_ID --exit-status
+
+# Verify secret matched what was set earlier
+exec gh run view $RUN_ID --log
+stdout 'GitHub Actions secret value matches$'
+
+-- workflow.yml --
+# This workflow is intended to assert the value of the GitHub Actions secret was set appropriately
+name: Test Workflow Name
+on:
+  # Allow workflow to be dispatched by gh workflow run
+  workflow_dispatch:
+
+jobs:
+  # This workflow contains a single job called "assert" that should only pass if the GitHub Actions secret value matches
+  assert:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Assert secret value matches
+        env:
+          ORG_SECRET: ${{ secrets.$SECRET_NAME }}
+        run: |
+          if [[ "$ORG_SECRET" == "just an organization secret" ]]; then
+            echo "GitHub Actions secret value matches"
+          else
+            echo "GitHub Actions secret value does not match"
+            exit 1
+          fi
--- a/acceptance/testdata/secret/secret-repo-env.txtar
+++ b/acceptance/testdata/secret/secret-repo-env.txtar
@ -0,0 +1,80 @@
+# Setup environment variables used for testscript
+env REPO=$SCRIPT_NAME-$RANDOM_STRING
+
+# Use gh as a credential helper
+exec gh auth setup-git
+
+# Create a repository with a file so it has a default branch
+exec gh repo create $ORG/$REPO --add-readme --private
+
+# Defer repo cleanup
+defer gh repo delete --yes $ORG/$REPO
+
+# Clone the repo
+exec gh repo clone $ORG/$REPO
+cd $REPO
+
+# Create a repository environment, will fail if organization does not have environment support
+exec gh api /repos/$ORG/$REPO/environments/testscripts -X PUT
+
+# Create a repository environment secret
+exec gh secret set TESTSCRIPTS_ENV --env testscripts --body 'just a repository environment secret'
+
+# Verify new repository secret exists
+exec gh secret list --env testscripts
+stdout 'TESTSCRIPTS_ENV'
+
+# Commit workflow file creating dispatchable workflow able to verify secret matches
+mkdir .github/workflows
+mv ../workflow.yml .github/workflows/workflow.yml
+exec git add .github/workflows/workflow.yml
+exec git commit -m 'Create workflow file'
+exec git push -u origin main
+
+# Sleep because it takes a second for the workflow to register
+sleep 1
+
+# Check the workflow is indeed created
+exec gh workflow list
+stdout 'Test Workflow Name'
+
+# Run the workflow
+exec gh workflow run 'Test Workflow Name'
+
+# It takes some time for a workflow run to register
+sleep 10
+
+# Get the run ID we want to watch & delete
+exec gh run list --json databaseId --jq '.[0].databaseId'
+stdout2env RUN_ID
+
+# Wait for workflow to complete
+exec gh run watch $RUN_ID --exit-status
+
+# Verify secret matched what was set earlier
+exec gh run view $RUN_ID --log
+stdout 'GitHub Actions secret value matches$'
+
+-- workflow.yml --
+# This workflow is intended to assert the value of the GitHub Actions secret was set appropriately
+name: Test Workflow Name
+on:
+  # Allow workflow to be dispatched by gh workflow run
+  workflow_dispatch:
+
+jobs:
+  # This workflow contains a single job called "assert" that should only pass if the GitHub Actions secret value matches
+  assert:
+    runs-on: ubuntu-latest
+    environment: testscripts
+    steps:
+      - name: Assert secret value matches
+        env:
+          TESTSCRIPTS_ENV: ${{ secrets.TESTSCRIPTS_ENV }}
+        run: |
+          if [[ "$TESTSCRIPTS_ENV" == "just a repository environment secret" ]]; then
+            echo "GitHub Actions secret value matches"
+          else
+            echo "GitHub Actions secret value does not match"
+            exit 1
+          fi
--- a/acceptance/testdata/secret/secret-repo.txtar
+++ b/acceptance/testdata/secret/secret-repo.txtar
@ -0,0 +1,76 @@
+# Setup environment variables used for testscript
+env REPO=$SCRIPT_NAME-$RANDOM_STRING
+
+# Use gh as a credential helper
+exec gh auth setup-git
+
+# Create a repository with a file so it has a default branch
+exec gh repo create $ORG/$REPO --add-readme --private
+
+# Defer repo cleanup
+defer gh repo delete --yes $ORG/$REPO
+
+# Clone the repo
+exec gh repo clone $ORG/$REPO
+cd $REPO
+
+# Create a repository secret
+exec gh secret set TESTSCRIPTS --body 'just a repository secret'
+
+# Verify new repository secret exists
+exec gh secret list
+stdout 'TESTSCRIPTS'
+
+# Commit workflow file creating dispatchable workflow able to verify secret matches
+mkdir .github/workflows
+mv ../workflow.yml .github/workflows/workflow.yml
+exec git add .github/workflows/workflow.yml
+exec git commit -m 'Create workflow file'
+exec git push -u origin main
+
+# Sleep because it takes a second for the workflow to register
+sleep 1
+
+# Check the workflow is indeed created
+exec gh workflow list
+stdout 'Test Workflow Name'
+
+# Run the workflow
+exec gh workflow run 'Test Workflow Name'
+
+# It takes some time for a workflow run to register
+sleep 10
+
+# Get the run ID we want to watch & delete
+exec gh run list --json databaseId --jq '.[0].databaseId'
+stdout2env RUN_ID
+
+# Wait for workflow to complete
+exec gh run watch $RUN_ID --exit-status
+
+# Verify secret matched what was set earlier
+exec gh run view $RUN_ID --log
+stdout 'GitHub Actions secret value matches$'
+
+-- workflow.yml --
+# This workflow is intended to assert the value of the GitHub Actions secret was set appropriately
+name: Test Workflow Name
+on:
+  # Allow workflow to be dispatched by gh workflow run
+  workflow_dispatch:
+
+jobs:
+  # This workflow contains a single job called "assert" that should only pass if the GitHub Actions secret value matches
+  assert:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Assert secret value matches
+        env:
+          TESTSCRIPTS: ${{ secrets.TESTSCRIPTS }}
+        run: |
+          if [[ "$TESTSCRIPTS" == "just a repository secret" ]]; then
+            echo "GitHub Actions secret value matches"
+          else
+            echo "GitHub Actions secret value does not match"
+            exit 1
+          fi