Update 3rd party license information
This commit is contained in:
Andy Feller
parent
bd24865076
commit
815c557f9a
1023 changed files with 158572 additions and 0 deletions
84
third-party/github.com/letsencrypt/boulder/cmd/contact-auditor/README.md
vendored
Normal file
84
third-party/github.com/letsencrypt/boulder/cmd/contact-auditor/README.md
vendored
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
# Contact-Auditor
|
||||
|
||||
Audits subscriber registrations for e-mail addresses that
|
||||
`notify-mailer` is currently configured to skip.
|
||||
|
||||
# Usage:
|
||||
|
||||
```shell
|
||||
-config string
|
||||
File containing a JSON config.
|
||||
-to-file
|
||||
Write the audit results to a file.
|
||||
-to-stdout
|
||||
Print the audit results to stdout.
|
||||
```
|
||||
|
||||
## Results format:
|
||||
|
||||
```
|
||||
<id> <createdAt> <problem type> "<contact contents or entry>" "<error msg>"
|
||||
```
|
||||
|
||||
## Example output:
|
||||
|
||||
### Successful run with no violations encountered and `--to-file`:
|
||||
|
||||
```
|
||||
I004823 contact-auditor nfWK_gM Running contact-auditor
|
||||
I004823 contact-auditor qJ_zsQ4 Beginning database query
|
||||
I004823 contact-auditor je7V9QM Query completed successfully
|
||||
I004823 contact-auditor 7LzGvQI Audit finished successfully
|
||||
I004823 contact-auditor 5Pbk_QM Audit results were written to: audit-2006-01-02T15:04.tsv
|
||||
```
|
||||
|
||||
### Contact contains entries that violate policy and `--to-stdout`:
|
||||
|
||||
```
|
||||
I004823 contact-auditor nfWK_gM Running contact-auditor
|
||||
I004823 contact-auditor qJ_zsQ4 Beginning database query
|
||||
I004823 contact-auditor je7V9QM Query completed successfully
|
||||
1 2006-01-02 15:04:05 validation "<contact entry>" "<error msg>"
|
||||
...
|
||||
I004823 contact-auditor 2fv7-QY Audit finished successfully
|
||||
```
|
||||
|
||||
### Contact is not valid JSON and `--to-stdout`:
|
||||
|
||||
```
|
||||
I004823 contact-auditor nfWK_gM Running contact-auditor
|
||||
I004823 contact-auditor qJ_zsQ4 Beginning database query
|
||||
I004823 contact-auditor je7V9QM Query completed successfully
|
||||
3 2006-01-02 15:04:05 unmarshal "<contact contents>" "<error msg>"
|
||||
...
|
||||
I004823 contact-auditor 2fv7-QY Audit finished successfully
|
||||
```
|
||||
|
||||
### Audit incomplete, query ended prematurely:
|
||||
|
||||
```
|
||||
I004823 contact-auditor nfWK_gM Running contact-auditor
|
||||
I004823 contact-auditor qJ_zsQ4 Beginning database query
|
||||
...
|
||||
E004823 contact-auditor 8LmTgww [AUDIT] Audit was interrupted, results may be incomplete: <error msg>
|
||||
exit status 1
|
||||
```
|
||||
|
||||
# Configuration file:
|
||||
The path to a database config file like the one below must be provided
|
||||
following the `-config` flag.
|
||||
|
||||
```json
|
||||
{
|
||||
"contactAuditor": {
|
||||
"db": {
|
||||
"dbConnectFile": <string>,
|
||||
"maxOpenConns": <int>,
|
||||
"maxIdleConns": <int>,
|
||||
"connMaxLifetime": <int>,
|
||||
"connMaxIdleTime": <int>
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
```
|
||||
212
third-party/github.com/letsencrypt/boulder/cmd/contact-auditor/main.go
vendored
Normal file
212
third-party/github.com/letsencrypt/boulder/cmd/contact-auditor/main.go
vendored
Normal file
|
|
@ -0,0 +1,212 @@
|
|||
package notmain
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/letsencrypt/boulder/cmd"
|
||||
"github.com/letsencrypt/boulder/db"
|
||||
blog "github.com/letsencrypt/boulder/log"
|
||||
"github.com/letsencrypt/boulder/policy"
|
||||
"github.com/letsencrypt/boulder/sa"
|
||||
)
|
||||
|
||||
type contactAuditor struct {
|
||||
db *db.WrappedMap
|
||||
resultsFile *os.File
|
||||
writeToStdout bool
|
||||
logger blog.Logger
|
||||
}
|
||||
|
||||
type result struct {
|
||||
id int64
|
||||
contacts []string
|
||||
createdAt string
|
||||
}
|
||||
|
||||
func unmarshalContact(contact []byte) ([]string, error) {
|
||||
var contacts []string
|
||||
err := json.Unmarshal(contact, &contacts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return contacts, nil
|
||||
}
|
||||
|
||||
func validateContacts(id int64, createdAt string, contacts []string) error {
|
||||
// Setup a buffer to store any validation problems we encounter.
|
||||
var probsBuff strings.Builder
|
||||
|
||||
// Helper to write validation problems to our buffer.
|
||||
writeProb := func(contact string, prob string) {
|
||||
// Add validation problem to buffer.
|
||||
fmt.Fprintf(&probsBuff, "%d\t%s\tvalidation\t%q\t%q\t%q\n", id, createdAt, contact, prob, contacts)
|
||||
}
|
||||
|
||||
for _, contact := range contacts {
|
||||
if strings.HasPrefix(contact, "mailto:") {
|
||||
err := policy.ValidEmail(strings.TrimPrefix(contact, "mailto:"))
|
||||
if err != nil {
|
||||
writeProb(contact, err.Error())
|
||||
}
|
||||
} else {
|
||||
writeProb(contact, "missing 'mailto:' prefix")
|
||||
}
|
||||
}
|
||||
|
||||
if probsBuff.Len() != 0 {
|
||||
return errors.New(probsBuff.String())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// beginAuditQuery executes the audit query and returns a cursor used to
|
||||
// stream the results.
|
||||
func (c contactAuditor) beginAuditQuery(ctx context.Context) (*sql.Rows, error) {
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT DISTINCT id, contact, createdAt
|
||||
FROM registrations
|
||||
WHERE contact NOT IN ('[]', 'null');`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
func (c contactAuditor) writeResults(result string) {
|
||||
if c.writeToStdout {
|
||||
_, err := fmt.Print(result)
|
||||
if err != nil {
|
||||
c.logger.Errf("Error while writing result to stdout: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
if c.resultsFile != nil {
|
||||
_, err := c.resultsFile.WriteString(result)
|
||||
if err != nil {
|
||||
c.logger.Errf("Error while writing result to file: %s", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// run retrieves a cursor from `beginAuditQuery` and then audits the
|
||||
// `contact` column of all returned rows for abnormalities or policy
|
||||
// violations.
|
||||
func (c contactAuditor) run(ctx context.Context, resChan chan *result) error {
|
||||
c.logger.Infof("Beginning database query")
|
||||
rows, err := c.beginAuditQuery(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
var contact []byte
|
||||
var createdAt string
|
||||
err := rows.Scan(&id, &contact, &createdAt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
contacts, err := unmarshalContact(contact)
|
||||
if err != nil {
|
||||
c.writeResults(fmt.Sprintf("%d\t%s\tunmarshal\t%q\t%q\n", id, createdAt, contact, err))
|
||||
}
|
||||
|
||||
err = validateContacts(id, createdAt, contacts)
|
||||
if err != nil {
|
||||
c.writeResults(err.Error())
|
||||
}
|
||||
|
||||
// Only used for testing.
|
||||
if resChan != nil {
|
||||
resChan <- &result{id, contacts, createdAt}
|
||||
}
|
||||
}
|
||||
// Ensure the query wasn't interrupted before it could complete.
|
||||
err = rows.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
} else {
|
||||
c.logger.Info("Query completed successfully")
|
||||
}
|
||||
|
||||
// Only used for testing.
|
||||
if resChan != nil {
|
||||
close(resChan)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
ContactAuditor struct {
|
||||
DB cmd.DBConfig
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
configFile := flag.String("config", "", "File containing a JSON config.")
|
||||
writeToStdout := flag.Bool("to-stdout", false, "Print the audit results to stdout.")
|
||||
writeToFile := flag.Bool("to-file", false, "Write the audit results to a file.")
|
||||
flag.Parse()
|
||||
|
||||
logger := cmd.NewLogger(cmd.SyslogConfig{StdoutLevel: 7})
|
||||
logger.Info(cmd.VersionString())
|
||||
|
||||
if *configFile == "" {
|
||||
flag.Usage()
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Load config from JSON.
|
||||
configData, err := os.ReadFile(*configFile)
|
||||
cmd.FailOnError(err, fmt.Sprintf("Error reading config file: %q", *configFile))
|
||||
|
||||
var cfg Config
|
||||
err = json.Unmarshal(configData, &cfg)
|
||||
cmd.FailOnError(err, "Couldn't unmarshal config")
|
||||
|
||||
db, err := sa.InitWrappedDb(cfg.ContactAuditor.DB, nil, logger)
|
||||
cmd.FailOnError(err, "Couldn't setup database client")
|
||||
|
||||
var resultsFile *os.File
|
||||
if *writeToFile {
|
||||
resultsFile, err = os.Create(
|
||||
fmt.Sprintf("contact-audit-%s.tsv", time.Now().Format("2006-01-02T15:04")),
|
||||
)
|
||||
cmd.FailOnError(err, "Failed to create results file")
|
||||
}
|
||||
|
||||
// Setup and run contact-auditor.
|
||||
auditor := contactAuditor{
|
||||
db: db,
|
||||
resultsFile: resultsFile,
|
||||
writeToStdout: *writeToStdout,
|
||||
logger: logger,
|
||||
}
|
||||
|
||||
logger.Info("Running contact-auditor")
|
||||
|
||||
err = auditor.run(context.TODO(), nil)
|
||||
cmd.FailOnError(err, "Audit was interrupted, results may be incomplete")
|
||||
|
||||
logger.Info("Audit finished successfully")
|
||||
|
||||
if *writeToFile {
|
||||
logger.Infof("Audit results were written to: %s", resultsFile.Name())
|
||||
resultsFile.Close()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func init() {
|
||||
cmd.RegisterCommand("contact-auditor", main, &cmd.ConfigValidator{Config: &Config{}})
|
||||
}
|
||||
219
third-party/github.com/letsencrypt/boulder/cmd/contact-auditor/main_test.go
vendored
Normal file
219
third-party/github.com/letsencrypt/boulder/cmd/contact-auditor/main_test.go
vendored
Normal file
|
|
@ -0,0 +1,219 @@
|
|||
package notmain
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jmhodges/clock"
|
||||
corepb "github.com/letsencrypt/boulder/core/proto"
|
||||
"github.com/letsencrypt/boulder/db"
|
||||
blog "github.com/letsencrypt/boulder/log"
|
||||
"github.com/letsencrypt/boulder/metrics"
|
||||
"github.com/letsencrypt/boulder/sa"
|
||||
"github.com/letsencrypt/boulder/test"
|
||||
"github.com/letsencrypt/boulder/test/vars"
|
||||
)
|
||||
|
||||
var (
|
||||
regA *corepb.Registration
|
||||
regB *corepb.Registration
|
||||
regC *corepb.Registration
|
||||
regD *corepb.Registration
|
||||
)
|
||||
|
||||
const (
|
||||
emailARaw = "test@example.com"
|
||||
emailBRaw = "example@notexample.com"
|
||||
emailCRaw = "test-example@notexample.com"
|
||||
telNum = "666-666-7777"
|
||||
)
|
||||
|
||||
func TestContactAuditor(t *testing.T) {
|
||||
testCtx := setup(t)
|
||||
defer testCtx.cleanUp()
|
||||
|
||||
// Add some test registrations.
|
||||
testCtx.addRegistrations(t)
|
||||
|
||||
resChan := make(chan *result, 10)
|
||||
err := testCtx.c.run(context.Background(), resChan)
|
||||
test.AssertNotError(t, err, "received error")
|
||||
|
||||
// We should get back A, B, C, and D
|
||||
test.AssertEquals(t, len(resChan), 4)
|
||||
for entry := range resChan {
|
||||
err := validateContacts(entry.id, entry.createdAt, entry.contacts)
|
||||
switch entry.id {
|
||||
case regA.Id:
|
||||
// Contact validation policy sad path.
|
||||
test.AssertDeepEquals(t, entry.contacts, []string{"mailto:test@example.com"})
|
||||
test.AssertError(t, err, "failed to error on a contact that violates our e-mail policy")
|
||||
case regB.Id:
|
||||
// Ensure grace period was respected.
|
||||
test.AssertDeepEquals(t, entry.contacts, []string{"mailto:example@notexample.com"})
|
||||
test.AssertNotError(t, err, "received error for a valid contact entry")
|
||||
case regC.Id:
|
||||
// Contact validation happy path.
|
||||
test.AssertDeepEquals(t, entry.contacts, []string{"mailto:test-example@notexample.com"})
|
||||
test.AssertNotError(t, err, "received error for a valid contact entry")
|
||||
|
||||
// Unmarshal Contact sad path.
|
||||
_, err := unmarshalContact([]byte("[ mailto:test@example.com ]"))
|
||||
test.AssertError(t, err, "failed to error while unmarshaling invalid Contact JSON")
|
||||
|
||||
// Fix our JSON and ensure that the contact field returns
|
||||
// errors for our 2 additional contacts
|
||||
contacts, err := unmarshalContact([]byte(`[ "mailto:test@example.com", "tel:666-666-7777" ]`))
|
||||
test.AssertNotError(t, err, "received error while unmarshaling valid Contact JSON")
|
||||
|
||||
// Ensure Contact validation now fails.
|
||||
err = validateContacts(entry.id, entry.createdAt, contacts)
|
||||
test.AssertError(t, err, "failed to error on 2 invalid Contact entries")
|
||||
case regD.Id:
|
||||
test.AssertDeepEquals(t, entry.contacts, []string{"tel:666-666-7777"})
|
||||
test.AssertError(t, err, "failed to error on an invalid contact entry")
|
||||
default:
|
||||
t.Errorf("ID: %d was not expected", entry.id)
|
||||
}
|
||||
}
|
||||
|
||||
// Load results file.
|
||||
data, err := os.ReadFile(testCtx.c.resultsFile.Name())
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
// Results file should contain 2 newlines, 1 for each result.
|
||||
contentLines := strings.Split(strings.TrimRight(string(data), "\n"), "\n")
|
||||
test.AssertEquals(t, len(contentLines), 2)
|
||||
|
||||
// Each result entry should contain six tab separated columns.
|
||||
for _, line := range contentLines {
|
||||
test.AssertEquals(t, len(strings.Split(line, "\t")), 6)
|
||||
}
|
||||
}
|
||||
|
||||
type testCtx struct {
|
||||
c contactAuditor
|
||||
dbMap *db.WrappedMap
|
||||
ssa *sa.SQLStorageAuthority
|
||||
cleanUp func()
|
||||
}
|
||||
|
||||
func (tc testCtx) addRegistrations(t *testing.T) {
|
||||
emailA := "mailto:" + emailARaw
|
||||
emailB := "mailto:" + emailBRaw
|
||||
emailC := "mailto:" + emailCRaw
|
||||
tel := "tel:" + telNum
|
||||
|
||||
// Every registration needs a unique JOSE key
|
||||
jsonKeyA := []byte(`{
|
||||
"kty":"RSA",
|
||||
"n":"0vx7agoebGcQSuuPiLJXZptN9nndrQmbXEps2aiAFbWhM78LhWx4cbbfAAtVT86zwu1RK7aPFFxuhDR1L6tSoc_BJECPebWKRXjBZCiFV4n3oknjhMstn64tZ_2W-5JsGY4Hc5n9yBXArwl93lqt7_RN5w6Cf0h4QyQ5v-65YGjQR0_FDW2QvzqY368QQMicAtaSqzs8KJZgnYb9c7d0zgdAZHzu6qMQvRL5hajrn1n91CbOpbISD08qNLyrdkt-bFTWhAI4vMQFh6WeZu0fM4lFd2NcRwr3XPksINHaQ-G_xBniIqbw0Ls1jF44-csFCur-kEgU8awapJzKnqDKgw",
|
||||
"e":"AQAB"
|
||||
}`)
|
||||
jsonKeyB := []byte(`{
|
||||
"kty":"RSA",
|
||||
"n":"z8bp-jPtHt4lKBqepeKF28g_QAEOuEsCIou6sZ9ndsQsEjxEOQxQ0xNOQezsKa63eogw8YS3vzjUcPP5BJuVzfPfGd5NVUdT-vSSwxk3wvk_jtNqhrpcoG0elRPQfMVsQWmxCAXCVRz3xbcFI8GTe-syynG3l-g1IzYIIZVNI6jdljCZML1HOMTTW4f7uJJ8mM-08oQCeHbr5ejK7O2yMSSYxW03zY-Tj1iVEebROeMv6IEEJNFSS4yM-hLpNAqVuQxFGetwtwjDMC1Drs1dTWrPuUAAjKGrP151z1_dE74M5evpAhZUmpKv1hY-x85DC6N0hFPgowsanmTNNiV75w",
|
||||
"e":"AAEAAQ"
|
||||
}`)
|
||||
jsonKeyC := []byte(`{
|
||||
"kty":"RSA",
|
||||
"n":"rFH5kUBZrlPj73epjJjyCxzVzZuV--JjKgapoqm9pOuOt20BUTdHqVfC2oDclqM7HFhkkX9OSJMTHgZ7WaVqZv9u1X2yjdx9oVmMLuspX7EytW_ZKDZSzL-sCOFCuQAuYKkLbsdcA3eHBK_lwc4zwdeHFMKIulNvLqckkqYB9s8GpgNXBDIQ8GjR5HuJke_WUNjYHSd8jY1LU9swKWsLQe2YoQUz_ekQvBvBCoaFEtrtRaSJKNLIVDObXFr2TLIiFiM0Em90kK01-eQ7ZiruZTKomll64bRFPoNo4_uwubddg3xTqur2vdF3NyhTrYdvAgTem4uC0PFjEQ1bK_djBQ",
|
||||
"e":"AQAB"
|
||||
}`)
|
||||
jsonKeyD := []byte(`{
|
||||
"kty":"RSA",
|
||||
"n":"rFH5kUBZrlPj73epjJjyCxzVzZuV--JjKgapoqm9pOuOt20BUTdHqVfC2oDclqM7HFhkkX9OSJMTHgZ7WaVqZv9u1X2yjdx9oVmMLuspX7EytW_ZKDZSzL-FCOFCuQAuYKkLbsdcA3eHBK_lwc4zwdeHFMKIulNvLqckkqYB9s8GpgNXBDIQ8GjR5HuJke_WUNjYHSd8jY1LU9swKWsLQe2YoQUz_ekQvBvBCoaFEtrtRaSJKNLIVDObXFr2TLIiFiM0Em90kK01-eQ7ZiruZTKomll64bRFPoNo4_uwubddg3xTqur2vdF3NyhTrYdvAgTem4uC0PFjEQ1bK_djBQ",
|
||||
"e":"AQAB"
|
||||
}`)
|
||||
|
||||
initialIP, err := net.ParseIP("127.0.0.1").MarshalText()
|
||||
test.AssertNotError(t, err, "Couldn't create initialIP")
|
||||
|
||||
regA = &corepb.Registration{
|
||||
Id: 1,
|
||||
Contact: []string{emailA},
|
||||
Key: jsonKeyA,
|
||||
InitialIP: initialIP,
|
||||
}
|
||||
regB = &corepb.Registration{
|
||||
Id: 2,
|
||||
Contact: []string{emailB},
|
||||
Key: jsonKeyB,
|
||||
InitialIP: initialIP,
|
||||
}
|
||||
regC = &corepb.Registration{
|
||||
Id: 3,
|
||||
Contact: []string{emailC},
|
||||
Key: jsonKeyC,
|
||||
InitialIP: initialIP,
|
||||
}
|
||||
// Reg D has a `tel:` contact ACME URL
|
||||
regD = &corepb.Registration{
|
||||
Id: 4,
|
||||
Contact: []string{tel},
|
||||
Key: jsonKeyD,
|
||||
InitialIP: initialIP,
|
||||
}
|
||||
|
||||
// Add the four test registrations
|
||||
ctx := context.Background()
|
||||
regA, err = tc.ssa.NewRegistration(ctx, regA)
|
||||
test.AssertNotError(t, err, "Couldn't store regA")
|
||||
regB, err = tc.ssa.NewRegistration(ctx, regB)
|
||||
test.AssertNotError(t, err, "Couldn't store regB")
|
||||
regC, err = tc.ssa.NewRegistration(ctx, regC)
|
||||
test.AssertNotError(t, err, "Couldn't store regC")
|
||||
regD, err = tc.ssa.NewRegistration(ctx, regD)
|
||||
test.AssertNotError(t, err, "Couldn't store regD")
|
||||
}
|
||||
|
||||
func setup(t *testing.T) testCtx {
|
||||
log := blog.UseMock()
|
||||
|
||||
// Using DBConnSAFullPerms to be able to insert registrations and
|
||||
// certificates
|
||||
dbMap, err := sa.DBMapForTest(vars.DBConnSAFullPerms)
|
||||
if err != nil {
|
||||
t.Fatalf("Couldn't connect to the database: %s", err)
|
||||
}
|
||||
|
||||
// Make temp results file
|
||||
file, err := os.CreateTemp("", fmt.Sprintf("audit-%s", time.Now().Format("2006-01-02T15:04")))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
cleanUp := func() {
|
||||
test.ResetBoulderTestDatabase(t)
|
||||
file.Close()
|
||||
os.Remove(file.Name())
|
||||
}
|
||||
|
||||
db, err := sa.DBMapForTest(vars.DBConnSAMailer)
|
||||
if err != nil {
|
||||
t.Fatalf("Couldn't connect to the database: %s", err)
|
||||
}
|
||||
|
||||
ssa, err := sa.NewSQLStorageAuthority(dbMap, dbMap, nil, 1, 0, clock.New(), log, metrics.NoopRegisterer)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to create SQLStorageAuthority: %s", err)
|
||||
}
|
||||
|
||||
return testCtx{
|
||||
c: contactAuditor{
|
||||
db: db,
|
||||
resultsFile: file,
|
||||
logger: blog.NewMock(),
|
||||
},
|
||||
dbMap: dbMap,
|
||||
ssa: ssa,
|
||||
cleanUp: cleanUp,
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue