added sentry org token detector (#3985)

Author: kashifkhan0771

pkg/detectors/sentryorgtoken/sentryorgtoken.go

@@ -0,0 +1,101 @@
+package sentryorgtoken
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+
+	regexp "github.com/wasilibs/go-re2"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct {
+	client *http.Client
+}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
+	orgAuthTokenPat = regexp.MustCompile(`\b(sntrys_eyJ[a-zA-Z0-9=_+/]{197})\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"sntrys_eyJ"}
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_SentryOrgToken
+}
+
+func (s Scanner) Description() string {
+	return "Sentry is an error tracking service that helps developers monitor and fix crashes in real time. Sentry Organization Auth Tokens can be used in many places to interact with Sentry programmatically. For example, they can be used for sentry-cli, bundler plugins, or similar use cases."
+}
+
+// FromData will find and optionally verify SentryToken secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	// find all unique org auth tokens
+	var uniqueOrgTokens = make(map[string]struct{})
+
+	for _, orgToken := range orgAuthTokenPat.FindAllStringSubmatch(dataStr, -1) {
+		uniqueOrgTokens[orgToken[1]] = struct{}{}
+	}
+
+	for orgToken := range uniqueOrgTokens {
+		s1 := detectors.Result{
+			DetectorType: detectorspb.DetectorType_SentryOrgToken,
+			Raw:          []byte(orgToken),
+		}
+
+		if verify {
+			if s.client == nil {
+				s.client = common.SaneHttpClient()
+			}
+
+			isVerified, verificationErr := verifySentryOrgToken(ctx, s.client, orgToken)
+			s1.Verified = isVerified
+			s1.SetVerificationError(verificationErr, orgToken)
+		}
+
+		results = append(results, s1)
+	}
+
+	return results, nil
+}
+
+// docs: https://docs.sentry.io/account/auth-tokens/#organization-auth-tokens
+func verifySentryOrgToken(ctx context.Context, client *http.Client, token string) (bool, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://sentry.io/api/0/auth/validate", nil)
+	if err != nil {
+		return false, err
+	}
+
+	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return false, err
+	}
+	defer func() {
+		_, _ = io.Copy(io.Discard, resp.Body)
+		_ = resp.Body.Close()
+	}()
+
+	switch resp.StatusCode {
+	case http.StatusOK:
+		return true, nil
+	case http.StatusForbidden, http.StatusUnauthorized:
+		return false, nil
+	default:
+		return false, fmt.Errorf("unexpected HTTP response status %d", resp.StatusCode)
+	}
+}

pkg/detectors/sentryorgtoken/sentryorgtoken_integration_test.go

@@ -0,0 +1,125 @@
+//go:build detectors
+// +build detectors
+
+package sentryorgtoken
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSentryOrgToken_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("SENTRY_ORG_TOKEN")
+	inactiveSecret := testSecrets.MustGetField("SENTRY_ORG_TOKEN_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a sentry secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_SentryOrgToken,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a sentry secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_SentryOrgToken,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("SentryOrgToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatal("no raw secret present")
+				}
+				if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+					t.Fatalf("wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError())
+				}
+			}
+			opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+			if diff := cmp.Diff(got, tt.want, opts); diff != "" {
+				t.Errorf("SentryOrgToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/sentryorgtoken/sentryorgtoken_test.go

@@ -0,0 +1,89 @@
+package sentryorgtoken
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
+
+var (
+	validPattern = `
+	sentry_token := sntrys_eyJFAKEiOjE3NDIzNjM1NTIuNTAzMzA5LCJ1cmwiOiJodHRwczovL3NlbnRyeS5pbyIsInJlZ2lvbl91cmwiOiJodHRwczovL3VzLnNlbnRyeS5pbfakem9yZyI6InRydWZmbGUtc2VjdXJpdHktamQifQ==_+zqSnKjs87cicc3FAK08vmZs5cWx9C5EARKHFtW5lqI
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", sentry_token))
+	`
+	invalidPattern = "sntrys_eyJFAKE-OjE3NDIzNjM1NTIuNTAzMzA5LCJ1cmwiOiJodHRwczovL3NlbnRyeS5pbyIsInJlZ2lvbl91cmwiOiJodHRwczovL3VzLnNlbnRyeS5pbfakem9yZyI6InRydWZmbGUtc2VjdXJpdHktamQifQ==_+zqSnKjs87cicc3FAK08vmZs5cWx9C5EARKHFtW5lqI"
+	token          = "sntrys_eyJFAKEiOjE3NDIzNjM1NTIuNTAzMzA5LCJ1cmwiOiJodHRwczovL3NlbnRyeS5pbyIsInJlZ2lvbl91cmwiOiJodHRwczovL3VzLnNlbnRyeS5pbfakem9yZyI6InRydWZmbGUtc2VjdXJpdHktamQifQ==_+zqSnKjs87cicc3FAK08vmZs5cWx9C5EARKHFtW5lqI"
+)
+
+func TestSentryToken_Pattern(t *testing.T) {
+	d := Scanner{}
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
+	tests := []struct {
+		name  string
+		input string
+		want  []string
+	}{
+		{
+			name:  "valid pattern - with keyword sentry org token",
+			input: validPattern,
+			want:  []string{token},
+		},
+		{
+			name:  "valid pattern - ignore duplicate",
+			input: fmt.Sprintf("token = '%s' | '%s'", validPattern, validPattern),
+			want:  []string{token},
+		},
+		{
+			name:  "invalid pattern",
+			input: invalidPattern,
+			want:  []string{},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+			if len(matchedDetectors) == 0 {
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
+				return
+			}
+
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
+			if err != nil {
+				t.Errorf("error = %v", err)
+				return
+			}
+
+			if len(results) != len(test.want) {
+				if len(results) == 0 {
+					t.Errorf("did not receive result")
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+			}
+		})
+	}
+}

pkg/engine/defaults/defaults.go

@@ -629,6 +629,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sendbirdorganizationapi"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sendgrid"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sendinbluev2"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sentryorgtoken"
 	sentrytokenv1 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sentrytoken/v1"
 	sentrytokenv2 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sentrytoken/v2"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/serphouse"
@@ -1480,6 +1481,7 @@ func buildDetectorList() []detectors.Detector {
 		&sendinbluev2.Scanner{},
 		&sentrytokenv1.Scanner{},
 		&sentrytokenv2.Scanner{},
+		&sentryorgtoken.Scanner{},
 		&serphouse.Scanner{},
 		&serpstack.Scanner{},
 		&sheety.Scanner{},

pkg/pb/detectorspb/detectors.pb.go

@@ -1026,6 +1026,7 @@ enum DetectorType {
   AirtableOAuth = 1014;
   AirtablePersonalAccessToken = 1015;
   StoryblokPersonalAccessToken = 1016;
+  SentryOrgToken = 1017;
 }
 
 message Result {