diff --git a/examples/allowlist.yml b/examples/allowlist.yml new file mode 100644 index 000000000000..91ebba364ca1 --- /dev/null +++ b/examples/allowlist.yml @@ -0,0 +1,20 @@ +- description: "RSA test keys" + values: + - | + -----BEGIN RSA PRIVATE KEY----- + MIIEpAIBAAKCAQEA... + -----END RSA PRIVATE KEY----- +- description: "Demo API keys for documentation" + values: + - "demo_.*" + - "example-api-key-.*" + - "sk_test_.*" +- description: "Decommissioned secrets (rotated out)" + values: + - "old-prod-key-2023" + - "^legacy-.*-deprecated$" +- description: "Public certificates and well-known test tokens" + values: + - "^-----BEGIN CERTIFICATE-----" + - "password123" + - "token_1234567890abcdef" diff --git a/examples/generic.yml b/examples/generic.yml index 04a226d21f61..98b70847a0d1 100644 --- a/examples/generic.yml +++ b/examples/generic.yml @@ -13,3 +13,13 @@ detectors: regex: # borrowing the gitleaks generic-api-key regex generic-api-key: "(?i)(?:key|api|token|secret|client|passwd|password|auth|access)(?:[0-9a-z\\-_\\t .]{0,20})(?:[\\s|']|[\\s|\"]){0,3}(?:=|>|:{1,3}=|\\|\\|:|<=|=>|:|\\?=)(?:'|\"|\\s|=|\\x60){0,5}([0-9a-z\\-_.=]{10,150})(?:['|\"|\\n|\\r|\\s|\\x60|;]|$)" +allowlists: + - description: "Test allowlist" + values: + - "https://user:password@example.com" + - | + -----BEGIN OPENSSH PRIVATE KEY----- + b3BlbnNzaC1rZXktdjEAAAAACm + - description: "Ignore AWS access keys" + values: + - "AKIA*" diff --git a/main.go b/main.go index 567b160f5742..bc629c3ee5fd 100644 --- a/main.go +++ b/main.go @@ -65,6 +65,7 @@ var ( allowVerificationOverlap = cli.Flag("allow-verification-overlap", "Allow verification of similar credentials across detectors").Bool() filterUnverified = cli.Flag("filter-unverified", "Only output first unverified result per chunk per detector if there are more than one results.").Bool() filterEntropy = cli.Flag("filter-entropy", "Filter unverified results with Shannon entropy. Start with 3.0.").Float64() + allowlistSecretsFile = cli.Flag("allowlist-secrets-file", "Path to YAML file with secrets to allowlist. See examples/allowlist.yml for format.").String() scanEntireChunk = cli.Flag("scan-entire-chunk", "Scan the entire chunk for secrets.").Hidden().Default("false").Bool() compareDetectionStrategies = cli.Flag("compare-detection-strategies", "Compare different detection strategies for matching spans").Hidden().Default("false").Bool() configFilename = cli.Flag("config", "Path to configuration file.").ExistingFile() @@ -525,6 +526,25 @@ func run(state overseer.State) { verificationCacheMetrics := verificationcache.InMemoryMetrics{} + // Load allowlisted secrets if specified + var allowlistedSecrets []detectors.AllowlistEntry + if *allowlistSecretsFile != "" { + allowlistedSecrets, err = detectors.LoadAllowlistedSecrets(*allowlistSecretsFile) + if err != nil { + logFatal(err, "failed to load allowlisted secrets") + } + } + allowListedSecrets := append(allowlistedSecrets, conf.Allowlists...) + compiledAllowlist := detectors.CompileAllowlistPatterns(allowListedSecrets) + + logger.Info( + "loaded allowlisted secrets", + "exact_matches", len(compiledAllowlist.ExactMatches), + "regex_patterns", len(compiledAllowlist.CompiledRegexes), + "total", len(compiledAllowlist.ExactMatches)+len(compiledAllowlist.CompiledRegexes), + "file", *allowlistSecretsFile, + ) + engConf := engine.Config{ Concurrency: *concurrency, ConfiguredSources: conf.Sources, @@ -541,6 +561,7 @@ func run(state overseer.State) { Dispatcher: engine.NewPrinterDispatcher(printer), FilterUnverified: *filterUnverified, FilterEntropy: *filterEntropy, + AllowlistedSecrets: compiledAllowlist, VerificationOverlap: *allowVerificationOverlap, Results: parsedResults, PrintAvgDetectorTime: *printAvgDetectorTime, diff --git a/pkg/config/config.go b/pkg/config/config.go index 47fe3868c5d8..13d40aea2930 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -23,8 +23,9 @@ import ( // Config holds user supplied configuration. type Config struct { - Sources []sources.ConfiguredSource - Detectors []detectors.Detector + Sources []sources.ConfiguredSource + Detectors []detectors.Detector + Allowlists []detectors.AllowlistEntry } // Read parses a given filename into a Config. @@ -66,9 +67,19 @@ func NewYAML(input []byte) (*Config, error) { sourceConfigs = append(sourceConfigs, src) } + // Convert allowlist entries to Go structs. + var allowlistConfigs []detectors.AllowlistEntry + for _, pbAllowlist := range inputYAML.Allowlists { + allowlistConfigs = append(allowlistConfigs, detectors.AllowlistEntry{ + Description: pbAllowlist.GetDescription(), + Values: pbAllowlist.GetValues(), + }) + } + return &Config{ - Detectors: detectorConfigs, - Sources: sourceConfigs, + Detectors: detectorConfigs, + Sources: sourceConfigs, + Allowlists: allowlistConfigs, }, nil } diff --git a/pkg/detectors/falsepositives.go b/pkg/detectors/falsepositives.go index 316d1d752163..3b1caa0dd935 100644 --- a/pkg/detectors/falsepositives.go +++ b/pkg/detectors/falsepositives.go @@ -3,14 +3,20 @@ package detectors import ( _ "embed" "fmt" + "io" "math" + "os" + "regexp" + "slices" "strings" "unicode" "unicode/utf8" ahocorasick "github.com/BobuSumisu/aho-corasick" + "gopkg.in/yaml.v3" "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/log" ) var ( @@ -29,6 +35,19 @@ type CustomFalsePositiveChecker interface { IsFalsePositive(result Result) (bool, string) } +// AllowlistEntry represents an allowlist entry in the YAML config +type AllowlistEntry struct { + Description string `yaml:"description,omitempty"` // Optional description for the allowlist + Values []string `yaml:"values"` // List of secret patterns/regexes to allowlist +} + +// CompiledAllowlist holds both exact string matches and compiled regex patterns for efficient matching +type CompiledAllowlist struct { + ExactMatches map[string]struct{} // For exact string matching (O(1) lookup) + CompiledRegexes []*regexp.Regexp // Pre-compiled regex patterns + RegexPatterns []string // Original regex patterns (for logging/debugging) +} + var ( filter *ahocorasick.Trie @@ -199,3 +218,117 @@ func FilterKnownFalsePositives(ctx context.Context, detector Detector, results [ return filteredResults } + +// FilterAllowlistedSecrets filters out results that match allowlisted secrets. +// This allows users to specify known safe secrets that should not be reported. +// Supports regex patterns. +func FilterAllowlistedSecrets(ctx context.Context, results []Result, allowlist *CompiledAllowlist) []Result { + if allowlist == nil || (len(allowlist.ExactMatches) == 0 && len(allowlist.CompiledRegexes) == 0) { + return results + } + + return slices.DeleteFunc(results, func(result Result) bool { + if len(result.Raw) == 0 { + return false // Keep results with empty Raw + } + + // Check if the raw secret matches any allowlisted secret + rawSecret := string(result.Raw) + log.RedactGlobally(rawSecret) + if isAllowlisted, matchReason := isSecretAllowlisted(rawSecret, allowlist); isAllowlisted { + ctx.Logger().V(4).Info("Skipping result: allowlisted secret", "result", rawSecret, "reason", matchReason) + return true // Delete this result + } + + // Also check RawV2 if present + if result.RawV2 != nil { + rawV2Secret := string(result.RawV2) + if isAllowlisted, matchReason := isSecretAllowlisted(rawV2Secret, allowlist); isAllowlisted { + ctx.Logger().V(4).Info("Skipping result: allowlisted secret", "result", rawV2Secret, "reason", matchReason) + return true // Delete this result + } + } + + return false // Keep this result + }) +} + +// LoadAllowlistedSecrets loads secrets from a YAML file that should be allowlisted. +// The YAML format supports multiline secrets and includes optional descriptions. +// Returns a CompiledAllowlist with pre-compiled regex patterns for efficient matching. +func LoadAllowlistedSecrets(yamlFile string) ([]AllowlistEntry, error) { + file, err := os.Open(yamlFile) + if err != nil { + return nil, fmt.Errorf("failed to open allowlist file: %w", err) + } + defer file.Close() + + // Read the entire file content + content, err := io.ReadAll(file) + if err != nil { + return nil, fmt.Errorf("failed to read allowlist file: %w", err) + } + + var allowList []AllowlistEntry + if err := yaml.Unmarshal(content, &allowList); err != nil { + return nil, fmt.Errorf("failed to parse YAML allowlist file: %w", err) + } + + return allowList, nil +} + +// CompileAllowlistPatterns compiles a list of patterns into a CompiledAllowlist. +// All patterns are first attempted to be compiled as regex. If compilation fails, +// they are treated as exact string matches. +func CompileAllowlistPatterns(allowList []AllowlistEntry) *CompiledAllowlist { + compiledAllowlist := &CompiledAllowlist{ + ExactMatches: make(map[string]struct{}, 0), + CompiledRegexes: make([]*regexp.Regexp, 0), + RegexPatterns: make([]string, 0), + } + + for _, entry := range allowList { + for _, pattern := range entry.Values { + pattern = strings.TrimSpace(pattern) + if pattern == "" { + continue // Skip empty patterns + } + + // Always try to compile as regex first + if compiledRegex, err := regexp.Compile(pattern); err == nil { + // Successfully compiled as regex + compiledAllowlist.CompiledRegexes = append(compiledAllowlist.CompiledRegexes, compiledRegex) + compiledAllowlist.RegexPatterns = append(compiledAllowlist.RegexPatterns, pattern) + } else { + // Invalid regex, treat as exact string match + compiledAllowlist.ExactMatches[pattern] = struct{}{} + } + } + } + + return compiledAllowlist +} + +// isSecretAllowlisted checks if a secret matches any allowlisted pattern (exact string or regex) +func isSecretAllowlisted(secret string, allowlist *CompiledAllowlist) (bool, string) { + if allowlist == nil { + return false, "" + } + + // Trim all whitespace (spaces, tabs, newlines, carriage returns) from the secret + secret = strings.TrimSpace(secret) + + // First, try exact string matching for performance (O(1) lookup) + if _, isAllowlisted := allowlist.ExactMatches[secret]; isAllowlisted { + return true, "exact match" + } + + // Try pre-compiled regex patterns + for i, compiledRegex := range allowlist.CompiledRegexes { + if compiledRegex.MatchString(secret) { + return true, "regex match: " + allowlist.RegexPatterns[i] + } + } + + return false, "" +} diff --git a/pkg/detectors/falsepositives_test.go b/pkg/detectors/falsepositives_test.go index 7d3cedafc479..f6b5b7302b0c 100644 --- a/pkg/detectors/falsepositives_test.go +++ b/pkg/detectors/falsepositives_test.go @@ -3,10 +3,14 @@ package detectors import ( "context" _ "embed" + "os" + "regexp" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/google/go-cmp/cmp" logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -183,9 +187,633 @@ func TestStringShannonEntropy(t *testing.T) { } } +func TestFilterAllowlistedSecrets(t *testing.T) { + ctx := logContext.Background() + + tests := []struct { + name string + results []Result + allowlistedSecrets []AllowlistEntry + expected []Result + }{ + { + name: "exact string match", + results: []Result{ + {Raw: []byte("secret123")}, + {Raw: []byte("password456")}, + {Raw: []byte("token789")}, + }, + allowlistedSecrets: []AllowlistEntry{ + { + Values: []string{ + "secret123", + }, + }, + }, + expected: []Result{ + {Raw: []byte("password456")}, + {Raw: []byte("token789")}, + }, + }, + { + name: "regex pattern match", + results: []Result{ + {Raw: []byte("test-api-key-12345")}, + {Raw: []byte("prod-api-key-67890")}, + {Raw: []byte("dev-token-abcdef")}, + {Raw: []byte("random-secret")}, + }, + allowlistedSecrets: []AllowlistEntry{ + { + Values: []string{ + `^test-.*`, + `.*-token-.*`, + }, + }, + }, + expected: []Result{ + {Raw: []byte("prod-api-key-67890")}, + {Raw: []byte("random-secret")}, + }, + }, + { + name: "mixed exact and regex patterns", + results: []Result{ + {Raw: []byte("exact-match")}, + {Raw: []byte("dev-key-123")}, + {Raw: []byte("prod-key-456")}, + {Raw: []byte("another-secret")}, + }, + allowlistedSecrets: []AllowlistEntry{ + { + Values: []string{ + "exact-match", + `^dev-.*`, + }, + }, + }, + expected: []Result{ + {Raw: []byte("prod-key-456")}, + {Raw: []byte("another-secret")}, + }, + }, + { + name: "invalid regex treated as literal string", + results: []Result{ + {Raw: []byte("[invalid")}, + {Raw: []byte("valid-secret")}, + }, + allowlistedSecrets: []AllowlistEntry{ + { + Values: []string{ + "[invalid", + }, + }, + }, + expected: []Result{ + {Raw: []byte("valid-secret")}, + }, + }, + { + name: "case sensitive regex", + results: []Result{ + {Raw: []byte("Secret123")}, + {Raw: []byte("secret456")}, + }, + allowlistedSecrets: []AllowlistEntry{ + { + Values: []string{ + `^secret.*`, + }, + }, + }, + expected: []Result{ + {Raw: []byte("Secret123")}, + }, + }, + { + name: "case insensitive regex", + results: []Result{ + {Raw: []byte("Secret123")}, + {Raw: []byte("secret456")}, + {Raw: []byte("other789")}, + }, + allowlistedSecrets: []AllowlistEntry{ + { + Values: []string{ + `(?i)^secret.*`, + }, + }, + }, + expected: []Result{ + {Raw: []byte("other789")}, + }, + }, + { + name: "RawV2 field testing", + results: []Result{ + { + Raw: []byte("primary-secret"), + RawV2: []byte("secondary-secret"), + }, + }, + allowlistedSecrets: []AllowlistEntry{ + { + Values: []string{ + "secondary-secret", + }, + }, + }, + expected: []Result{}, // should be filtered out due to RawV2 match + }, + { + name: "empty results", + results: []Result{}, + allowlistedSecrets: []AllowlistEntry{ + { + Values: []string{ + "any-pattern", + }, + }, + }, + expected: []Result{}, + }, + { + name: "empty allowlist", + results: []Result{ + {Raw: []byte("secret123")}, + }, + allowlistedSecrets: []AllowlistEntry{ + { + Values: []string{}, + }, + }, + expected: []Result{ + {Raw: []byte("secret123")}, + }, + }, + { + name: "nil allowlist", + results: []Result{ + {Raw: []byte("secret123")}, + }, + allowlistedSecrets: nil, + expected: []Result{ + {Raw: []byte("secret123")}, + }, + }, + { + name: "complex regex patterns", + results: []Result{ + {Raw: []byte("api-key-12345")}, + {Raw: []byte("token-67890")}, + {Raw: []byte("secret-abcdef")}, + {Raw: []byte("random-secret-123")}, + }, + allowlistedSecrets: []AllowlistEntry{ + { + Values: []string{ + `^api-key-\d+$`, + `^token-\d+$`, + `^secret-[a-f]+$`, + }, + }, + }, + expected: []Result{ + {Raw: []byte("random-secret-123")}, // This doesn't match any pattern + }, + }, + { + name: "hexadecimal pattern matching", + results: []Result{ + {Raw: []byte("abcdef1234567890")}, // 16-char hex + {Raw: []byte("123456789abcdef0123")}, // 19-char hex + {Raw: []byte("ghijklmnop123456")}, // not hex + {Raw: []byte("ABC123DEF456")}, // mixed case hex + }, + allowlistedSecrets: []AllowlistEntry{ + { + Values: []string{ + `^[a-f0-9]{16}$`, + `^[A-F0-9a-f]{12}$`, + }, + }, + }, + expected: []Result{ + {Raw: []byte("123456789abcdef0123")}, // 19 chars, doesn't match 16-char pattern + {Raw: []byte("ghijklmnop123456")}, // contains non-hex chars + }, + }, + { + name: "multiline RSA private key exact match", + results: []Result{ + {Raw: []byte(`-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA7YQU7gTBJOfGJ4NlMJOtL... +-----END RSA PRIVATE KEY-----`)}, + {Raw: []byte("single-line-secret")}, + {Raw: []byte(`-----BEGIN CERTIFICATE----- +MIIDXTCCAkWgAwIBAgIJAKoK/heBjcOuMA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV +-----END CERTIFICATE-----`)}, + }, + allowlistedSecrets: []AllowlistEntry{ + { + Values: []string{ + `-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA7YQU7gTBJOfGJ4NlMJOtL... +-----END RSA PRIVATE KEY-----`, + }, + }, + }, + expected: []Result{ + {Raw: []byte("single-line-secret")}, + {Raw: []byte(`-----BEGIN CERTIFICATE----- +MIIDXTCCAkWgAwIBAgIJAKoK/heBjcOuMA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV +-----END CERTIFICATE-----`)}, + }, + }, + { + name: "multiline regex pattern matching", + results: []Result{ + {Raw: []byte(`-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA... +-----END RSA PRIVATE KEY-----`)}, + {Raw: []byte(`-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC... +-----END PRIVATE KEY-----`)}, + {Raw: []byte(`-----BEGIN CERTIFICATE----- +MIIDXTCCAkWgAwIBAgIJAKoK/heBjcOuMA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV +-----END CERTIFICATE-----`)}, + {Raw: []byte("some-other-secret")}, + }, + allowlistedSecrets: []AllowlistEntry{ + { + Values: []string{ + `(?s)-----BEGIN.*PRIVATE KEY-----.*-----END.*PRIVATE KEY-----`, + }, + }, + }, + expected: []Result{ + {Raw: []byte(`-----BEGIN CERTIFICATE----- +MIIDXTCCAkWgAwIBAgIJAKoK/heBjcOuMA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV +-----END CERTIFICATE-----`)}, + {Raw: []byte("some-other-secret")}, + }, + }, + { + name: "multiline patterns that shouldn't match", + results: []Result{ + {Raw: []byte(`-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA... +-----END RSA PRIVATE KEY-----`)}, + {Raw: []byte("production-api-key-12345")}, + }, + allowlistedSecrets: []AllowlistEntry{ + { + Values: []string{ + `(?s)-----BEGIN.*CERTIFICATE-----.*-----END.*CERTIFICATE-----`, + `^test-.*`, + }, + }, + }, + expected: []Result{ + {Raw: []byte(`-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA... +-----END RSA PRIVATE KEY-----`)}, + {Raw: []byte("production-api-key-12345")}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + compiledAllowlist := CompileAllowlistPatterns(tt.allowlistedSecrets) + result := FilterAllowlistedSecrets(ctx, tt.results, compiledAllowlist) + assert.ElementsMatch(t, tt.expected, result) + }) + } +} + +func TestIsSecretAllowlisted(t *testing.T) { + tests := []struct { + name string + secret string + allowlistedSecrets []AllowlistEntry + expectedMatch bool + expectedReason string + }{ + { + name: "simple string compiled as regex", + secret: "exact-secret", + allowlistedSecrets: []AllowlistEntry{ + { + Values: []string{"exact-secret"}, + }, + }, + expectedMatch: true, + expectedReason: "regex match: exact-secret", + }, + { + name: "regex pattern match", + secret: "test-key-12345", + allowlistedSecrets: []AllowlistEntry{ + { + Values: []string{`^test-.*`}, + }, + }, + expectedMatch: true, + expectedReason: "regex match: ^test-.*", + }, + { + name: "no match", + secret: "random-secret", + allowlistedSecrets: []AllowlistEntry{ + { + Values: []string{ + "different-secret", + `^test-.*`, + }, + }, + }, + expectedMatch: false, + expectedReason: "", + }, + { + name: "simple string and regex pattern both work", + secret: "test-key", + allowlistedSecrets: []AllowlistEntry{ + { + Values: []string{ + "test-key", + `^test-.*`, + }, + }, + }, + expectedMatch: true, + expectedReason: "regex match: test-key", // simple strings are compiled as regex first + }, + { + name: "invalid regex treated as literal", + secret: "[invalid", + allowlistedSecrets: []AllowlistEntry{ + { + Values: []string{ + "[invalid", + }, + }, + }, + expectedMatch: true, + expectedReason: "exact match", + }, + { + name: "multiline RSA key compiled as regex", + secret: `-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA7YQU7gTBJOfGJ4NlMJOtL... +-----END RSA PRIVATE KEY-----`, + allowlistedSecrets: []AllowlistEntry{ + { + Values: []string{ + `-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA7YQU7gTBJOfGJ4NlMJOtL... +-----END RSA PRIVATE KEY-----`, + }, + }, + }, + expectedMatch: true, + expectedReason: "regex match: -----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA7YQU7gTBJOfGJ4NlMJOtL...\n-----END RSA PRIVATE KEY-----", + }, + { + name: "multiline private key regex match", + secret: `-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA... +-----END RSA PRIVATE KEY-----`, + allowlistedSecrets: []AllowlistEntry{ + { + Values: []string{ + `(?s)-----BEGIN.*PRIVATE KEY-----.*-----END.*PRIVATE KEY-----`, + }, + }, + }, + expectedMatch: true, + expectedReason: "regex match: (?s)-----BEGIN.*PRIVATE KEY-----.*-----END.*PRIVATE KEY-----", + }, + { + name: "multiline certificate not matching private key pattern", + secret: `-----BEGIN CERTIFICATE----- +MIIDXTCCAkWgAwIBAgIJAKoK/heBjcOuMA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV +-----END CERTIFICATE-----`, + allowlistedSecrets: []AllowlistEntry{ + { + Values: []string{ + `(?s)-----BEGIN.*PRIVATE KEY-----.*-----END.*PRIVATE KEY-----`, + }, + }, + }, + expectedMatch: false, + expectedReason: "", + }, + { + name: "multiline secret with multiple patterns", + secret: `-----BEGIN RSA PRIVATE KEY----- +test-content +-----END RSA PRIVATE KEY-----`, + allowlistedSecrets: []AllowlistEntry{ + { + Values: []string{ + `-----BEGIN RSA PRIVATE KEY----- +test-content +-----END RSA PRIVATE KEY-----`, + `(?s)-----BEGIN.*PRIVATE KEY-----.*-----END.*PRIVATE KEY-----`, + }, + }, + }, + expectedMatch: true, + expectedReason: "", // Don't check specific reason since map iteration order is non-deterministic + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + compiledAllowlist := CompileAllowlistPatterns(tt.allowlistedSecrets) + match, reason := isSecretAllowlisted(tt.secret, compiledAllowlist) + assert.Equal(t, tt.expectedMatch, match) + if tt.expectedReason != "" { + assert.Equal(t, tt.expectedReason, reason) + } + }) + } +} + +func BenchmarkFilterallowlistedSecrets(b *testing.B) { + ctx := logContext.Background() + + results := []Result{ + {Raw: []byte("secret1")}, + {Raw: []byte("test-api-key-12345")}, + {Raw: []byte("prod-token-abcdef")}, + {Raw: []byte("random-secret-123")}, + {Raw: []byte("dev-key-67890")}, + } + + allowlistedSecrets := []AllowlistEntry{ + { + Values: []string{ + "secret1", // exact match + `^test-.*`, // regex + `.*-token-.*`, // regex + }, + }, + } + + compiledAllowlist := CompileAllowlistPatterns(allowlistedSecrets) + b.ResetTimer() + for i := 0; i < b.N; i++ { + FilterAllowlistedSecrets(ctx, results, compiledAllowlist) + } +} + +func BenchmarkIsSecretAllowlisted(b *testing.B) { + allowlistedSecrets := []AllowlistEntry{ + { + Values: []string{ + "exact-secret", + `^test-.*`, + `.*-token-.*`, + `^[a-f0-9]{32}$`, + }, + }, + } + + secrets := []string{ + "exact-secret", // exact match + "test-key-12345", // regex match + "random-token-abc", // regex match + "abcdef1234567890abcdef1234567890", // regex match + "no-match-secret", // no match + } + + compiledAllowlist := CompileAllowlistPatterns(allowlistedSecrets) + b.ResetTimer() + for i := 0; i < b.N; i++ { + for _, secret := range secrets { + isSecretAllowlisted(secret, compiledAllowlist) + } + } +} + func BenchmarkDefaultIsKnownFalsePositive(b *testing.B) { for i := 0; i < b.N; i++ { // Use a string that won't be found in any dictionary for the worst case check. IsKnownFalsePositive("aoeuaoeuaoeuaoeuaoeuaoeu", DefaultFalsePositives, true) } } + +func TestLoadallowlistedSecrets(t *testing.T) { + tests := []struct { + name string + yamlContent string + expected []AllowlistEntry + wantErr bool + }{ + { + name: "basic patterns with descriptions", + yamlContent: `- description: "Used in tests" + values: + - "^dev-.*" + - "^stage.*" +- description: "Legacy API keys" + values: + - "legacy-key-123" + - "old-token-.*"`, + expected: []AllowlistEntry{ + { + Values: []string{"^dev-.*", "^stage.*"}, + }, + { + Values: []string{"legacy-key-123", "old-token-.*"}, + }, + }, + wantErr: false, + }, + { + name: "multiline RSA key", + yamlContent: `- description: "Test RSA keys" + values: + - | + -----BEGIN RSA PRIVATE KEY----- + MIIEpAIBAAKCAQEA7YQU7gTBJOfGJ4NlMJOtL... + -----END RSA PRIVATE KEY-----`, + expected: []AllowlistEntry{ + { + Values: []string{"-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA7YQU7gTBJOfGJ4NlMJOtL...\n-----END RSA PRIVATE KEY-----"}, + }, + }, + wantErr: false, + }, + { + name: "entry without description field", + yamlContent: `- values: + - "no-description-pattern" + - "another-pattern"`, + expected: []AllowlistEntry{ + { + Values: []string{"no-description-pattern", "another-pattern"}, + }, + }, + wantErr: false, + }, + { + name: "empty values filtered out", + yamlContent: `- description: "Test filtering" + values: + - "valid-pattern" + - "" + - " " + - "another-valid"`, + expected: []AllowlistEntry{ + { + Values: []string{"valid-pattern", "another-valid"}, + }, + }, + wantErr: false, + }, + { + name: "invalid YAML", + yamlContent: `invalid yaml [content`, + expected: nil, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create temporary file + tmpFile, err := os.CreateTemp("", "allowlist-test-*.yaml") + require.NoError(t, err) + defer os.Remove(tmpFile.Name()) + + // Write test content + _, err = tmpFile.WriteString(tt.yamlContent) + require.NoError(t, err) + require.NoError(t, tmpFile.Close()) + + // Test the function + result, err := LoadAllowlistedSecrets(tmpFile.Name()) + + if tt.wantErr { + assert.Error(t, err) + return + } + + require.NoError(t, err) + want := CompileAllowlistPatterns(tt.expected) + got := CompileAllowlistPatterns(result) + assert.True(t, cmp.Equal(want, got, cmp.AllowUnexported(regexp.Regexp{})), "CompiledAllowlist structures should be functionally equivalent") + }) + } +} + +func TestLoadAllowlistedSecretsFileNotFound(t *testing.T) { + _, err := LoadAllowlistedSecrets("nonexistent-file.yaml") + assert.Error(t, err) + assert.Contains(t, err.Error(), "failed to open allowlist file") +} diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index ffc69e91b2c0..d00c0cc5c71f 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -120,7 +120,9 @@ type Config struct { FilterEntropy float64 // FilterUnverified sets the filterUnverified flag on the engine. If set to // true, the engine will only return the first unverified result for a chunk for a detector. - FilterUnverified bool + FilterUnverified bool + // AllowlistedSecrets contains secrets that should be ignored during scanning. + AllowlistedSecrets *detectors.CompiledAllowlist ShouldScanEntireChunk bool Dispatcher ResultsDispatcher @@ -174,6 +176,7 @@ type Engine struct { filterUnverified bool // entropyFilter is used to filter out unverified results using Shannon entropy. filterEntropy float64 + allowlistedSecrets *detectors.CompiledAllowlist notifyVerifiedResults bool notifyUnverifiedResults bool notifyUnknownResults bool @@ -236,6 +239,7 @@ func NewEngine(ctx context.Context, cfg *Config) (*Engine, error) { verify: cfg.Verify, filterUnverified: cfg.FilterUnverified, filterEntropy: cfg.FilterEntropy, + allowlistedSecrets: cfg.AllowlistedSecrets, printAvgDetectorTime: cfg.PrintAvgDetectorTime, retainFalsePositives: cfg.LogFilteredUnverified, verificationOverlap: cfg.VerificationOverlap, @@ -1167,6 +1171,10 @@ func (e *Engine) filterResults( results = detectors.FilterResultsWithEntropy(ctx, results, e.filterEntropy, e.retainFalsePositives) } + if e.allowlistedSecrets != nil && (len(e.allowlistedSecrets.ExactMatches) > 0 || len(e.allowlistedSecrets.CompiledRegexes) > 0) { + results = detectors.FilterAllowlistedSecrets(ctx, results, e.allowlistedSecrets) + } + return results } diff --git a/pkg/engine/engine_test.go b/pkg/engine/engine_test.go index 4e2888e99918..86138ab281e5 100644 --- a/pkg/engine/engine_test.go +++ b/pkg/engine/engine_test.go @@ -1318,3 +1318,247 @@ def test_something(): }) } } + +func TestEngine_AllowlistedSecrets(t *testing.T) { + tests := []struct { + name string + content string + allowlistedSecrets []detectors.AllowlistEntry + expectedFindings int + }{ + { + name: "exact string allowlist match", + content: ` +aws_access_key_id = AKIAQYLPMN5HHHFPZAM2 +aws_secret_access_key = 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 +deepseek_api_key = sk-abc123def456ghi789jkl012mno345pq +openai_api_key = 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 +`, + allowlistedSecrets: []detectors.AllowlistEntry{ + { + Values: []string{ + "AKIAQYLPMN5HHHFPZAM2", + }, + }, + }, + expectedFindings: 2, + }, + { + name: "regex pattern allowlist", + content: ` +aws_access_key_id = AKIAQYLPMN5HHHFPZAM2 +aws_secret_access_key = 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 +deepseek_api_key = sk-abc123def456ghi789jkl012mno345pq +openai_api_key = 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 + `, + allowlistedSecrets: []detectors.AllowlistEntry{ + { + Values: []string{ + `^sk-[a-z0-9]{32}$`, + }, + }, + }, + expectedFindings: 2, + }, + { + name: "mixed exact and regex allowlist", + content: ` +aws_access_key_id = AKIAQYLPMN5HHHFPZAM2 +aws_secret_access_key = 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 +deepseek_api_key = sk-abc123def456ghi789jkl012mno345pq +openai_api_key = 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 + `, + allowlistedSecrets: []detectors.AllowlistEntry{ + { + Values: []string{ + "AKIAQYLPMN5HHHFPZAM2", // exact string + `^sk-[a-z0-9]{32}$`, // regex pattern + }, + }, + }, + expectedFindings: 1, + }, + { + name: "case sensitivity test", + content: ` +aws_access_key_id = AKIAQYLPMN5HHHFPZAM2 +aws_secret_access_key = 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 +deepseek_api_key = sk-abc123def456ghi789jkl012mno345pq +openai_api_key = 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 + `, + allowlistedSecrets: []detectors.AllowlistEntry{ + { + Values: []string{ + `^SK-[a-z0-9]{32}$`, // case sensitive - only matches uppercase SK + }, + }, + }, + expectedFindings: 3, + }, + { + name: "case insensitive regex", + content: ` +aws_access_key_id = AKIAQYLPMN5HHHFPZAM2 +aws_secret_access_key = 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 +deepseek_api_key = sk-abc123def456ghi789jkl012mno345pq +openai_api_key = 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 + `, + allowlistedSecrets: []detectors.AllowlistEntry{ + { + Values: []string{ + `(?i)^SK-[a-z0-9]{32}$`, // case insensitive + }, + }, + }, + expectedFindings: 2, + }, + { + name: "no allowlist", + content: ` +aws_access_key_id = AKIAQYLPMN5HHHFPZAM2 +aws_secret_access_key = 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 +deepseek_api_key = sk-abc123def456ghi789jkl012mno345pq +openai_api_key = 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 + `, + allowlistedSecrets: []detectors.AllowlistEntry{}, + expectedFindings: 3, + }, + { + name: "invalid regex patterns allowlist", + content: ` +aws_access_key_id = AKIAQYLPMN5HHHFPZAM2 +aws_secret_access_key = 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 +deepseek_api_key = sk-abc123def456ghi789jkl012mno345pq +openai_api_key = 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 + `, + allowlistedSecrets: []detectors.AllowlistEntry{ + { + Values: []string{ + "[AKIAQYLPMN5HHHFPZAM2", // invalid regex, treated as exact string + }, + }, + }, + expectedFindings: 3, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + tmpFile, err := os.CreateTemp("", "test_allowlist") + assert.NoError(t, err) + defer os.Remove(tmpFile.Name()) + + err = os.WriteFile(tmpFile.Name(), []byte(tt.content), os.ModeAppend) + assert.NoError(t, err) + + const defaultOutputBufferSize = 64 + opts := []func(*sources.SourceManager){ + sources.WithSourceUnits(), + sources.WithBufferedOutput(defaultOutputBufferSize), + } + + sourceManager := sources.NewManager(opts...) + + conf := Config{ + Concurrency: 1, + Decoders: decoders.DefaultDecoders(), + Detectors: defaults.DefaultDetectors(), + Verify: false, + SourceManager: sourceManager, + Dispatcher: NewPrinterDispatcher(new(discardPrinter)), + AllowlistedSecrets: detectors.CompileAllowlistPatterns(tt.allowlistedSecrets), + } + + eng, err := NewEngine(ctx, &conf) + assert.NoError(t, err) + + eng.Start(ctx) + + cfg := sources.FilesystemConfig{Paths: []string{tmpFile.Name()}} + _, err = eng.ScanFileSystem(ctx, cfg) + assert.NoError(t, err) + + assert.NoError(t, eng.Finish(ctx)) + assert.Equal(t, tt.expectedFindings, int(eng.GetMetrics().UnverifiedSecretsFound), + "Expected %d findings but got %d", tt.expectedFindings, eng.GetMetrics().UnverifiedSecretsFound) + }) + } +} + +func TestEngine_allowlistedSecretsPerformance(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // Create a large test file with many secrets + content := "" + // Add 1000 secrets with various patterns + for i := 0; i < 1000; i++ { + content += fmt.Sprintf("aws_access_key_id_%d=AKIAQYLPMN5HHHF%05d{\n", i, i) + content += fmt.Sprintf("deepseek_api_key_%d=sk-abc123def456ghi789jkl012mno%05d{\n", i, i) + content += fmt.Sprintf("openai_api_key_%d=sk-SDAPGGZUyVr7SYJpSODgT3BlbkFJM1fIItFASvyIsaC%05d{\n", i, i) + } + + tmpFile, err := os.CreateTemp("", "test_performance") + assert.NoError(t, err) + defer os.Remove(tmpFile.Name()) + + err = os.WriteFile(tmpFile.Name(), []byte(content), os.ModeAppend) + assert.NoError(t, err) + + // Define allowlist with both exact strings and regex patterns + allowlistedSecrets := []detectors.AllowlistEntry{ + { + Values: []string{ + // Some exact matches + "AKIAQYLPMN5HHHF00001", + "sk-abc123def456ghi789jkl012mno00005", + "SDAPGGZUyVr7SYJpSODgT3BlbkFJM1fIItFASvyIsaC00003", + // Some regex patterns + `^AKIAQYLPMN5HHHF[0-4][0-9]{4}$`, // keys 0-49 + `^sk-abc123def456ghi789jkl012mno[5-9][0-9]{4}$`, // tokens 50-99 + `^sk-SDAPGGZUyVr7SYJpSODgT3BlbkFJM1fIItFASvyIsaC[1-9][0-9]{4}$`, // keys 100-999 + }, + }, + } + + const defaultOutputBufferSize = 64 + opts := []func(*sources.SourceManager){ + sources.WithSourceUnits(), + sources.WithBufferedOutput(defaultOutputBufferSize), + } + + sourceManager := sources.NewManager(opts...) + + start := time.Now() + + conf := Config{ + Concurrency: 4, // Use multiple threads for realistic performance test + Decoders: decoders.DefaultDecoders(), + Detectors: defaults.DefaultDetectors(), + Verify: false, + SourceManager: sourceManager, + Dispatcher: NewPrinterDispatcher(new(discardPrinter)), + AllowlistedSecrets: detectors.CompileAllowlistPatterns(allowlistedSecrets), + } + + eng, err := NewEngine(ctx, &conf) + assert.NoError(t, err) + + eng.Start(ctx) + + cfg := sources.FilesystemConfig{Paths: []string{tmpFile.Name()}} + _, err = eng.ScanFileSystem(ctx, cfg) + assert.NoError(t, err) + assert.NoError(t, eng.Finish(ctx)) + + elapsed := time.Since(start) + + assert.Less(t, elapsed, 10*time.Second, "Allowlisting performance test took too long") + assert.Greater(t, int(eng.GetMetrics().UnverifiedSecretsFound), 0, "Should find some secrets") + assert.Less(t, int(eng.GetMetrics().UnverifiedSecretsFound), 3000, "Should filter out many secrets") +} diff --git a/pkg/pb/allowlistspb/allowlists.pb.go b/pkg/pb/allowlistspb/allowlists.pb.go new file mode 100644 index 000000000000..1bd913988ab7 --- /dev/null +++ b/pkg/pb/allowlistspb/allowlists.pb.go @@ -0,0 +1,164 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.33.0 +// protoc v4.25.3 +// source: allowlists.proto + +package allowlistspb + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// AllowlistEntry represents an allowlist entry with optional description +// and a list of patterns (exact strings or regex) to allowlist +type AllowlistEntry struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Optional description for this allowlist entry (for documentation/organization) + Description string `protobuf:"bytes,1,opt,name=description,proto3" json:"description,omitempty"` + // List of secret patterns/regexes to allowlist + // Each value can be either: + // - An exact string match + // - A regular expression pattern + // The implementation will automatically determine which type it is + Values []string `protobuf:"bytes,2,rep,name=values,proto3" json:"values,omitempty"` +} + +func (x *AllowlistEntry) Reset() { + *x = AllowlistEntry{} + if protoimpl.UnsafeEnabled { + mi := &file_allowlists_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *AllowlistEntry) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AllowlistEntry) ProtoMessage() {} + +func (x *AllowlistEntry) ProtoReflect() protoreflect.Message { + mi := &file_allowlists_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AllowlistEntry.ProtoReflect.Descriptor instead. +func (*AllowlistEntry) Descriptor() ([]byte, []int) { + return file_allowlists_proto_rawDescGZIP(), []int{0} +} + +func (x *AllowlistEntry) GetDescription() string { + if x != nil { + return x.Description + } + return "" +} + +func (x *AllowlistEntry) GetValues() []string { + if x != nil { + return x.Values + } + return nil +} + +var File_allowlists_proto protoreflect.FileDescriptor + +var file_allowlists_proto_rawDesc = []byte{ + 0x0a, 0x10, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x6c, 0x69, 0x73, 0x74, 0x73, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x12, 0x0a, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x6c, 0x69, 0x73, 0x74, 0x73, 0x22, 0x4a, + 0x0a, 0x0e, 0x41, 0x6c, 0x6c, 0x6f, 0x77, 0x6c, 0x69, 0x73, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x42, 0x3e, 0x5a, 0x3c, 0x67, 0x69, + 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, + 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, + 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x61, 0x6c, + 0x6c, 0x6f, 0x77, 0x6c, 0x69, 0x73, 0x74, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, +} + +var ( + file_allowlists_proto_rawDescOnce sync.Once + file_allowlists_proto_rawDescData = file_allowlists_proto_rawDesc +) + +func file_allowlists_proto_rawDescGZIP() []byte { + file_allowlists_proto_rawDescOnce.Do(func() { + file_allowlists_proto_rawDescData = protoimpl.X.CompressGZIP(file_allowlists_proto_rawDescData) + }) + return file_allowlists_proto_rawDescData +} + +var file_allowlists_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_allowlists_proto_goTypes = []interface{}{ + (*AllowlistEntry)(nil), // 0: allowlists.AllowlistEntry +} +var file_allowlists_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_allowlists_proto_init() } +func file_allowlists_proto_init() { + if File_allowlists_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_allowlists_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*AllowlistEntry); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_allowlists_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_allowlists_proto_goTypes, + DependencyIndexes: file_allowlists_proto_depIdxs, + MessageInfos: file_allowlists_proto_msgTypes, + }.Build() + File_allowlists_proto = out.File + file_allowlists_proto_rawDesc = nil + file_allowlists_proto_goTypes = nil + file_allowlists_proto_depIdxs = nil +} diff --git a/pkg/pb/allowlistspb/allowlists.pb.validate.go b/pkg/pb/allowlistspb/allowlists.pb.validate.go new file mode 100644 index 000000000000..096fd60d7931 --- /dev/null +++ b/pkg/pb/allowlistspb/allowlists.pb.validate.go @@ -0,0 +1,138 @@ +// Code generated by protoc-gen-validate. DO NOT EDIT. +// source: allowlists.proto + +package allowlistspb + +import ( + "bytes" + "errors" + "fmt" + "net" + "net/mail" + "net/url" + "regexp" + "sort" + "strings" + "time" + "unicode/utf8" + + "google.golang.org/protobuf/types/known/anypb" +) + +// ensure the imports are used +var ( + _ = bytes.MinRead + _ = errors.New("") + _ = fmt.Print + _ = utf8.UTFMax + _ = (*regexp.Regexp)(nil) + _ = (*strings.Reader)(nil) + _ = net.IPv4len + _ = time.Duration(0) + _ = (*url.URL)(nil) + _ = (*mail.Address)(nil) + _ = anypb.Any{} + _ = sort.Sort +) + +// Validate checks the field values on AllowlistEntry with the rules defined in +// the proto definition for this message. If any rules are violated, the first +// error encountered is returned, or nil if there are no violations. +func (m *AllowlistEntry) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on AllowlistEntry with the rules defined +// in the proto definition for this message. If any rules are violated, the +// result is a list of violation errors wrapped in AllowlistEntryMultiError, +// or nil if none found. +func (m *AllowlistEntry) ValidateAll() error { + return m.validate(true) +} + +func (m *AllowlistEntry) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for Description + + if len(errors) > 0 { + return AllowlistEntryMultiError(errors) + } + + return nil +} + +// AllowlistEntryMultiError is an error wrapping multiple validation errors +// returned by AllowlistEntry.ValidateAll() if the designated constraints +// aren't met. +type AllowlistEntryMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m AllowlistEntryMultiError) Error() string { + var msgs []string + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m AllowlistEntryMultiError) AllErrors() []error { return m } + +// AllowlistEntryValidationError is the validation error returned by +// AllowlistEntry.Validate if the designated constraints aren't met. +type AllowlistEntryValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e AllowlistEntryValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e AllowlistEntryValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e AllowlistEntryValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e AllowlistEntryValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e AllowlistEntryValidationError) ErrorName() string { return "AllowlistEntryValidationError" } + +// Error satisfies the builtin error interface +func (e AllowlistEntryValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sAllowlistEntry.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = AllowlistEntryValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = AllowlistEntryValidationError{} diff --git a/pkg/pb/configpb/config.pb.go b/pkg/pb/configpb/config.pb.go index cc760fcba20a..5db9f180c40b 100644 --- a/pkg/pb/configpb/config.pb.go +++ b/pkg/pb/configpb/config.pb.go @@ -7,6 +7,7 @@ package configpb import ( + allowlistspb "github.com/trufflesecurity/trufflehog/v3/pkg/pb/allowlistspb" custom_detectorspb "github.com/trufflesecurity/trufflehog/v3/pkg/pb/custom_detectorspb" sourcespb "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" protoreflect "google.golang.org/protobuf/reflect/protoreflect" @@ -27,8 +28,9 @@ type Config struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Sources []*sourcespb.LocalSource `protobuf:"bytes,9,rep,name=sources,proto3" json:"sources,omitempty"` - Detectors []*custom_detectorspb.CustomRegex `protobuf:"bytes,13,rep,name=detectors,proto3" json:"detectors,omitempty"` + Sources []*sourcespb.LocalSource `protobuf:"bytes,9,rep,name=sources,proto3" json:"sources,omitempty"` + Detectors []*custom_detectorspb.CustomRegex `protobuf:"bytes,13,rep,name=detectors,proto3" json:"detectors,omitempty"` + Allowlists []*allowlistspb.AllowlistEntry `protobuf:"bytes,14,rep,name=allowlists,proto3" json:"allowlists,omitempty"` } func (x *Config) Reset() { @@ -77,25 +79,37 @@ func (x *Config) GetDetectors() []*custom_detectorspb.CustomRegex { return nil } +func (x *Config) GetAllowlists() []*allowlistspb.AllowlistEntry { + if x != nil { + return x.Allowlists + } + return nil +} + var File_config_proto protoreflect.FileDescriptor var file_config_proto_rawDesc = []byte{ 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x1a, 0x0d, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x16, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x64, 0x65, - 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x75, 0x0a, - 0x06, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x2e, 0x0a, 0x07, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x73, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x07, - 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x3b, 0x0a, 0x09, 0x64, 0x65, 0x74, 0x65, 0x63, - 0x74, 0x6f, 0x72, 0x73, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x63, 0x75, 0x73, - 0x74, 0x6f, 0x6d, 0x5f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x2e, 0x43, 0x75, - 0x73, 0x74, 0x6f, 0x6d, 0x52, 0x65, 0x67, 0x65, 0x78, 0x52, 0x09, 0x64, 0x65, 0x74, 0x65, 0x63, - 0x74, 0x6f, 0x72, 0x73, 0x42, 0x3a, 0x5a, 0x38, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, - 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, - 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x70, 0x62, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x10, 0x61, + 0x6c, 0x6c, 0x6f, 0x77, 0x6c, 0x69, 0x73, 0x74, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, + 0xb1, 0x01, 0x0a, 0x06, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x2e, 0x0a, 0x07, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x73, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x53, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x52, 0x07, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x3b, 0x0a, 0x09, 0x64, 0x65, + 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, + 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, + 0x2e, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x52, 0x65, 0x67, 0x65, 0x78, 0x52, 0x09, 0x64, 0x65, + 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x12, 0x3a, 0x0a, 0x0a, 0x61, 0x6c, 0x6c, 0x6f, 0x77, + 0x6c, 0x69, 0x73, 0x74, 0x73, 0x18, 0x0e, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x61, 0x6c, + 0x6c, 0x6f, 0x77, 0x6c, 0x69, 0x73, 0x74, 0x73, 0x2e, 0x41, 0x6c, 0x6c, 0x6f, 0x77, 0x6c, 0x69, + 0x73, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x6c, 0x69, + 0x73, 0x74, 0x73, 0x42, 0x3a, 0x5a, 0x38, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, + 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, + 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x70, 0x62, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -115,15 +129,17 @@ var file_config_proto_goTypes = []interface{}{ (*Config)(nil), // 0: config.Config (*sourcespb.LocalSource)(nil), // 1: sources.LocalSource (*custom_detectorspb.CustomRegex)(nil), // 2: custom_detectors.CustomRegex + (*allowlistspb.AllowlistEntry)(nil), // 3: allowlists.AllowlistEntry } var file_config_proto_depIdxs = []int32{ 1, // 0: config.Config.sources:type_name -> sources.LocalSource 2, // 1: config.Config.detectors:type_name -> custom_detectors.CustomRegex - 2, // [2:2] is the sub-list for method output_type - 2, // [2:2] is the sub-list for method input_type - 2, // [2:2] is the sub-list for extension type_name - 2, // [2:2] is the sub-list for extension extendee - 0, // [0:2] is the sub-list for field type_name + 3, // 2: config.Config.allowlists:type_name -> allowlists.AllowlistEntry + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name } func init() { file_config_proto_init() } diff --git a/pkg/pb/configpb/config.pb.validate.go b/pkg/pb/configpb/config.pb.validate.go index e02545a95e0a..cd9174c366f0 100644 --- a/pkg/pb/configpb/config.pb.validate.go +++ b/pkg/pb/configpb/config.pb.validate.go @@ -124,6 +124,40 @@ func (m *Config) validate(all bool) error { } + for idx, item := range m.GetAllowlists() { + _, _ = idx, item + + if all { + switch v := interface{}(item).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, ConfigValidationError{ + field: fmt.Sprintf("Allowlists[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, ConfigValidationError{ + field: fmt.Sprintf("Allowlists[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(item).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return ConfigValidationError{ + field: fmt.Sprintf("Allowlists[%v]", idx), + reason: "embedded message failed validation", + cause: err, + } + } + } + + } + if len(errors) > 0 { return ConfigMultiError(errors) } diff --git a/proto/allowlists.proto b/proto/allowlists.proto new file mode 100644 index 000000000000..75fe9631fa9e --- /dev/null +++ b/proto/allowlists.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +package allowlists; + +option go_package = "github.com/trufflesecurity/trufflehog/v3/pkg/pb/allowlistspb"; + +// AllowlistEntry represents an allowlist entry with optional description +// and a list of patterns (exact strings or regex) to allowlist +message AllowlistEntry { + // Optional description for this allowlist entry (for documentation/organization) + string description = 1; + + // List of secret patterns/regexes to allowlist + // Each value can be either: + // - An exact string match + // - A regular expression pattern + // The implementation will automatically determine which type it is + repeated string values = 2; +} diff --git a/proto/config.proto b/proto/config.proto index b74bbd54169e..7483141a376f 100644 --- a/proto/config.proto +++ b/proto/config.proto @@ -6,8 +6,10 @@ option go_package = "github.com/trufflesecurity/trufflehog/v3/pkg/pb/configpb"; import "sources.proto"; import "custom_detectors.proto"; +import "allowlists.proto"; message Config { repeated sources.LocalSource sources = 9; repeated custom_detectors.CustomRegex detectors = 13; + repeated allowlists.AllowlistEntry allowlists = 14; }