From ae37c25d25a6499652026ab0e74089ef46814a0c Mon Sep 17 00:00:00 2001 From: William Martin Date: Mon, 14 Oct 2024 13:06:16 +0200 Subject: [PATCH 1/6] Add acceptance task to makefile --- Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e68b68939..04babcc53 100644 --- a/Makefile +++ b/Makefile @@ -38,11 +38,15 @@ completions: bin/gh$(EXE) bin/gh$(EXE) completion -s fish > ./share/fish/vendor_completions.d/gh.fish bin/gh$(EXE) completion -s zsh > ./share/zsh/site-functions/_gh -# just a convenience task around `go test` +# just convenience tasks around `go test` .PHONY: test test: go test ./... +.PHONY: acceptance +acceptance: + go test -tags acceptance ./acceptance + ## Site-related tasks are exclusively intended for use by the GitHub CLI team and for our release automation. site: From 3b1f06a88f8e185c939590df90bb9bf9deff5299 Mon Sep 17 00:00:00 2001 From: William Martin Date: Mon, 14 Oct 2024 20:27:34 +0200 Subject: [PATCH 2/6] Ensure Acceptance defer failures are debuggable --- acceptance/acceptance_test.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index bcc27faca..e7b2c1313 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -47,6 +47,8 @@ func testScriptParamsFor(tsEnv testScriptEnv, dir string) testscript.Params { } } +var keyT struct{} + func sharedSetup(tsEnv testScriptEnv) func(ts *testscript.Env) error { return func(ts *testscript.Env) error { scriptName, ok := extractScriptName(ts.Vars) @@ -62,6 +64,8 @@ func sharedSetup(tsEnv testScriptEnv) func(ts *testscript.Env) error { ts.Setenv("GH_TOKEN", tsEnv.token) ts.Setenv("RANDOM_STRING", randomString(10)) + + ts.Values[keyT] = ts.T() return nil } } @@ -78,8 +82,21 @@ func sharedCmds(tsEnv testScriptEnv) map[string]func(ts *testscript.TestScript, return } + tt, ok := ts.Value(keyT).(testscript.T) + if !ok { + ts.Fatalf("%v is not a testscript.T", ts.Value(keyT)) + } + ts.Defer(func() { - ts.Check(ts.Exec(args[0], args[1:]...)) + // If you're wondering why we're not using ts.Check here, it's because it raises a panic, and testscript + // only catches the panics directly from commands, not from the deferred functions. So what we do + // instead is grab the `t` in the setup function and store it as a value. It's important that we use + // `t` from the setup function because it represents the subtest created for each individual script, + // rather than each top-level test. + // See: https://github.com/rogpeppe/go-internal/issues/276 + if err := ts.Exec(args[0], args[1:]...); err != nil { + tt.FailNow() + } }) }, "stdout2env": func(ts *testscript.TestScript, neg bool, args []string) { From 5ad4a385bb7d676a220b2858143d07ffbc25eaf9 Mon Sep 17 00:00:00 2001 From: William Martin Date: Mon, 14 Oct 2024 20:36:42 +0200 Subject: [PATCH 3/6] Support GH_ACCEPTANCE_SCRIPT --- acceptance/README.md | 6 ++++++ acceptance/acceptance_test.go | 19 ++++++++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/acceptance/README.md b/acceptance/README.md index 1a7f6a93d..a1cd3f216 100644 --- a/acceptance/README.md +++ b/acceptance/README.md @@ -32,6 +32,12 @@ A full example invocation can be found below: GH_ACCEPTANCE_HOST= GH_ACCEPTANCE_ORG= GH_ACCEPTANCE_TOKEN= go test -tags=acceptance ./acceptance ``` +While writing a new test, it can be useful to target that specific script by providing the `GH_ACCEPTANCE_SCRIPT` env var in combination with the `-run` flag, for example: + +``` +GH_ACCEPTANCE_SCRIPT=pr-view.txtar GH_ACCEPTANCE_HOST= GH_ACCEPTANCE_ORG= GH_ACCEPTANCE_TOKEN= go test -tags=acceptance -run ^TestPullRequests$ ./acceptance +``` + #### Code Coverage To get code coverage, `go test` can be invoked with `coverpkg` and `coverprofile` like so: diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index bcc27faca..3f3fb48ec 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -35,10 +35,20 @@ func TestPullRequests(t *testing.T) { testscript.Run(t, testScriptParamsFor(tsEnv, "pr")) } -func testScriptParamsFor(tsEnv testScriptEnv, dir string) testscript.Params { +func testScriptParamsFor(tsEnv testScriptEnv, command string) testscript.Params { + var files []string + if tsEnv.script != "" { + files = []string{path.Join("testdata", command, tsEnv.script)} + } + + var dir string + if len(files) == 0 { + dir = path.Join("testdata", command) + } + return testscript.Params{ - Dir: path.Join("testdata", dir), - Files: []string{}, + Dir: dir, + Files: files, Setup: sharedSetup(tsEnv), Cmds: sharedCmds(tsEnv), RequireExplicitExec: true, @@ -128,6 +138,8 @@ type testScriptEnv struct { org string token string + script string + skipDefer bool preserveWorkDir bool } @@ -164,6 +176,7 @@ func (e *testScriptEnv) fromEnv() error { e.org = envMap["GH_ACCEPTANCE_ORG"] e.token = envMap["GH_ACCEPTANCE_TOKEN"] + e.script = os.Getenv("GH_ACCEPTANCE_SCRIPT") e.preserveWorkDir = os.Getenv("GH_ACCEPTANCE_PRESERVE_WORK_DIR") == "true" e.skipDefer = os.Getenv("GH_ACCEPTANCE_SKIP_DEFER") == "true" From 6970eb36f134beb88fc0170675b5ff5984005a8e Mon Sep 17 00:00:00 2001 From: William Martin Date: Mon, 14 Oct 2024 20:48:38 +0200 Subject: [PATCH 4/6] Acceptance test issue command --- acceptance/acceptance_test.go | 14 ++++++++++--- acceptance/testdata/issue/issue-comment.txtar | 20 +++++++++++++++++++ .../testdata/issue/issue-create-basic.txtar | 17 ++++++++++++++++ .../issue/issue-create-with-metadata.txtar | 19 ++++++++++++++++++ acceptance/testdata/issue/issue-list.txtar | 16 +++++++++++++++ acceptance/testdata/issue/issue-view.txtar | 17 ++++++++++++++++ 6 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 acceptance/testdata/issue/issue-comment.txtar create mode 100644 acceptance/testdata/issue/issue-create-basic.txtar create mode 100644 acceptance/testdata/issue/issue-create-with-metadata.txtar create mode 100644 acceptance/testdata/issue/issue-list.txtar create mode 100644 acceptance/testdata/issue/issue-view.txtar diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index bcc27faca..429e0850c 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -28,8 +28,16 @@ func TestMain(m *testing.M) { func TestPullRequests(t *testing.T) { var tsEnv testScriptEnv if err := tsEnv.fromEnv(); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) + t.Fatal(err) + } + + testscript.Run(t, testScriptParamsFor(tsEnv, "pr")) +} + +func TestIssues(t *testing.T) { + var tsEnv testScriptEnv + if err := tsEnv.fromEnv(); err != nil { + t.Fatal(err) } testscript.Run(t, testScriptParamsFor(tsEnv, "pr")) @@ -120,7 +128,7 @@ type missingEnvError struct { } func (e missingEnvError) Error() string { - return fmt.Sprintf("environment variables %s must be set and non-empty", strings.Join(e.missingEnvs, ", ")) + return fmt.Sprintf("environment variable(s) %s must be set and non-empty", strings.Join(e.missingEnvs, ", ")) } type testScriptEnv struct { diff --git a/acceptance/testdata/issue/issue-comment.txtar b/acceptance/testdata/issue/issue-comment.txtar new file mode 100644 index 000000000..f47bf619c --- /dev/null +++ b/acceptance/testdata/issue/issue-comment.txtar @@ -0,0 +1,20 @@ +# Create a repository with a file so it has a default branch +exec gh repo create $ORG/$SCRIPT_NAME-$RANDOM_STRING --add-readme --private + +# Defer repo cleanup +defer gh repo delete --yes $ORG/$SCRIPT_NAME-$RANDOM_STRING + +# Clone the repo +exec gh repo clone $ORG/$SCRIPT_NAME-$RANDOM_STRING + +# Create an issue in the repo +cd $SCRIPT_NAME-$RANDOM_STRING +exec gh issue create --title 'Feature Request' --body 'Feature Body' +stdout2env ISSUE_URL + +# Comment on the issue +exec gh issue comment $ISSUE_URL --body 'Looks like a great feature!' + +# View the issue +exec gh issue view $ISSUE_URL --comments +stdout 'Looks like a great feature!' diff --git a/acceptance/testdata/issue/issue-create-basic.txtar b/acceptance/testdata/issue/issue-create-basic.txtar new file mode 100644 index 000000000..ddba28eec --- /dev/null +++ b/acceptance/testdata/issue/issue-create-basic.txtar @@ -0,0 +1,17 @@ +# Create a repository with a file so it has a default branch +exec gh repo create $ORG/$SCRIPT_NAME-$RANDOM_STRING --add-readme --private + +# Defer repo cleanup +defer gh repo delete --yes $ORG/$SCRIPT_NAME-$RANDOM_STRING + +# Clone the repo +exec gh repo clone $ORG/$SCRIPT_NAME-$RANDOM_STRING + +# Create an issue in the repo +cd $SCRIPT_NAME-$RANDOM_STRING +exec gh issue create --title 'Feature Request' --body 'Feature Body' +stdout2env ISSUE_URL + +# Check the issue was created +exec gh issue view $ISSUE_URL +stdout 'title:\tFeature Request$' diff --git a/acceptance/testdata/issue/issue-create-with-metadata.txtar b/acceptance/testdata/issue/issue-create-with-metadata.txtar new file mode 100644 index 000000000..e1ce5a0da --- /dev/null +++ b/acceptance/testdata/issue/issue-create-with-metadata.txtar @@ -0,0 +1,19 @@ +# Create a repository with a file so it has a default branch +exec gh repo create $ORG/$SCRIPT_NAME-$RANDOM_STRING --add-readme --private + +# Defer repo cleanup +defer gh repo delete --yes $ORG/$SCRIPT_NAME-$RANDOM_STRING + +# Clone the repo +exec gh repo clone $ORG/$SCRIPT_NAME-$RANDOM_STRING + +# Create an issue in the repo +cd $SCRIPT_NAME-$RANDOM_STRING +exec gh issue create --title 'Feature Request' --body 'Feature Body' --assignee '@me' --label 'bug' +stdout2env ISSUE_URL + +# Check the issue was create +exec gh issue view $ISSUE_URL +stdout 'title:\tFeature Request$' +stdout 'assignees:\t.*$' +stdout 'labels:\tbug$' diff --git a/acceptance/testdata/issue/issue-list.txtar b/acceptance/testdata/issue/issue-list.txtar new file mode 100644 index 000000000..5a810f5e1 --- /dev/null +++ b/acceptance/testdata/issue/issue-list.txtar @@ -0,0 +1,16 @@ +# Create a repository with a file so it has a default branch +exec gh repo create $ORG/$SCRIPT_NAME-$RANDOM_STRING --add-readme --private + +# Defer repo cleanup +defer gh repo delete --yes $ORG/$SCRIPT_NAME-$RANDOM_STRING + +# Clone the repo +exec gh repo clone $ORG/$SCRIPT_NAME-$RANDOM_STRING + +# Create an issue in the repo +cd $SCRIPT_NAME-$RANDOM_STRING +exec gh issue create --title 'Feature Request' --body 'Feature Body' + +# Check the issue is included in the list output +exec gh issue list +stdout 'OPEN\tFeature Request' diff --git a/acceptance/testdata/issue/issue-view.txtar b/acceptance/testdata/issue/issue-view.txtar new file mode 100644 index 000000000..ddba28eec --- /dev/null +++ b/acceptance/testdata/issue/issue-view.txtar @@ -0,0 +1,17 @@ +# Create a repository with a file so it has a default branch +exec gh repo create $ORG/$SCRIPT_NAME-$RANDOM_STRING --add-readme --private + +# Defer repo cleanup +defer gh repo delete --yes $ORG/$SCRIPT_NAME-$RANDOM_STRING + +# Clone the repo +exec gh repo clone $ORG/$SCRIPT_NAME-$RANDOM_STRING + +# Create an issue in the repo +cd $SCRIPT_NAME-$RANDOM_STRING +exec gh issue create --title 'Feature Request' --body 'Feature Body' +stdout2env ISSUE_URL + +# Check the issue was created +exec gh issue view $ISSUE_URL +stdout 'title:\tFeature Request$' From 7066bdb66f8b85c92dc0b0a75c5c587ee0b2101f Mon Sep 17 00:00:00 2001 From: William Martin Date: Tue, 15 Oct 2024 17:33:18 +0100 Subject: [PATCH 5/6] Add comment to acceptance make target Co-authored-by: Kynan Ware <47394200+BagToad@users.noreply.github.com> --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 04babcc53..32a06df2f 100644 --- a/Makefile +++ b/Makefile @@ -43,6 +43,7 @@ completions: bin/gh$(EXE) test: go test ./... +# For more information, see https://github.com/cli/cli/blob/trunk/acceptance/README.md .PHONY: acceptance acceptance: go test -tags acceptance ./acceptance From e37cf8b6944141059aa02ef7565b5850b8d7fbb0 Mon Sep 17 00:00:00 2001 From: William Martin Date: Tue, 15 Oct 2024 19:11:49 +0100 Subject: [PATCH 6/6] Fix issue creation with metadata regex Co-authored-by: Andy Feller --- acceptance/testdata/issue/issue-create-with-metadata.txtar | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/testdata/issue/issue-create-with-metadata.txtar b/acceptance/testdata/issue/issue-create-with-metadata.txtar index e1ce5a0da..8187f96cf 100644 --- a/acceptance/testdata/issue/issue-create-with-metadata.txtar +++ b/acceptance/testdata/issue/issue-create-with-metadata.txtar @@ -15,5 +15,5 @@ stdout2env ISSUE_URL # Check the issue was create exec gh issue view $ISSUE_URL stdout 'title:\tFeature Request$' -stdout 'assignees:\t.*$' +stdout 'assignees:\t.+$' stdout 'labels:\tbug$'