Skip to content

Commit 56db30d

Browse files
kashifkhan0771kashifkhan0771
authored
Merge branch 'main' into update/source/add-pagination-and-rate-limit-retry-in-docker-registries
2 parents 167a1e9 + b6389e2 commit 56db30d

27 files changed

+1724
-309
lines changed

‎CODEOWNERS‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ pkg/giturl/ @trufflesecurity/Scanning @trufflesecurity/OSS
2727
pkg/handlers/ @trufflesecurity/Scanning @trufflesecurity/OSS
2828
pkg/iobuf/ @trufflesecurity/Scanning @trufflesecurity/OSS
2929
pkg/sanitizer/ @trufflesecurity/Scanning @trufflesecurity/OSS
30-
proto/ @trufflesecurity/Scanning @trufflesecurity/OSS
30+
proto/ @trufflesecurity/Scanning @trufflesecurity/Integrations
3131

3232
# OSS
3333
pkg/detectors/ @trufflesecurity/OSS

‎pkg/common/http.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -133,8 +133,8 @@ func (t *InstrumentedTransport) RoundTrip(req *http.Request) (*http.Response, er
133133
}
134134

135135
if resp != nil {
136-
// record latency and increment counter for non-200 status code
137-
recordHTTPResponse(sanitizedURL, resp.StatusCode, duration.Seconds())
136+
// record latency, response size and increment counter for non-200 status code
137+
recordHTTPResponse(sanitizedURL, resp.StatusCode, duration.Seconds(), resp.ContentLength)
138138
}
139139

140140
return resp, err
@@ -198,7 +198,7 @@ func WithRetryWaitMax(wait time.Duration) ClientOption {
198198
func PinnedRetryableHttpClient() *http.Client {
199199
httpClient := retryablehttp.NewClient()
200200
httpClient.Logger = nil
201-
httpClient.HTTPClient.Transport = NewCustomTransport(&http.Transport{
201+
httpClient.HTTPClient.Transport = NewInstrumentedTransport(NewCustomTransport(&http.Transport{
202202
TLSClientConfig: &tls.Config{
203203
RootCAs: PinnedCertPool(),
204204
},
@@ -212,15 +212,15 @@ func PinnedRetryableHttpClient() *http.Client {
212212
IdleConnTimeout: 90 * time.Second,
213213
TLSHandshakeTimeout: 10 * time.Second,
214214
ExpectContinueTimeout: 1 * time.Second,
215-
})
215+
}))
216216
return httpClient.StandardClient()
217217
}
218218

219219
func RetryableHTTPClient(opts ...ClientOption) *http.Client {
220220
httpClient := retryablehttp.NewClient()
221221
httpClient.RetryMax = 3
222222
httpClient.Logger = nil
223-
httpClient.HTTPClient.Transport = NewCustomTransport(nil)
223+
httpClient.HTTPClient.Transport = NewInstrumentedTransport(NewCustomTransport(nil))
224224

225225
for _, opt := range opts {
226226
opt(httpClient)
@@ -234,7 +234,7 @@ func RetryableHTTPClientTimeout(timeOutSeconds int64, opts ...ClientOption) *htt
234234
httpClient.RetryMax = 3
235235
httpClient.Logger = nil
236236
httpClient.HTTPClient.Timeout = time.Duration(timeOutSeconds) * time.Second
237-
httpClient.HTTPClient.Transport = NewCustomTransport(nil)
237+
httpClient.HTTPClient.Transport = NewInstrumentedTransport(NewCustomTransport(nil))
238238

239239
for _, opt := range opts {
240240
opt(httpClient)

‎pkg/common/http_metrics.go‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,17 @@ var (
3939
},
4040
[]string{"url", "status_code"},
4141
)
42+
43+
httpResponseBodySizeBytes = promauto.NewHistogramVec(
44+
prometheus.HistogramOpts{
45+
Namespace: MetricsNamespace,
46+
Subsystem: "http_client",
47+
Name: "response_body_size_bytes",
48+
Help: "Size of HTTP response bodies in bytes, labeled by URL.",
49+
Buckets: prometheus.ExponentialBuckets(100, 10, 5), // [100B, 1KB, 10KB, 100KB, 1MB]
50+
},
51+
[]string{"url"},
52+
)
4253
)
4354

4455
// sanitizeURL sanitizes a URL to avoid high cardinality metrics.
@@ -92,14 +103,19 @@ func recordHTTPRequest(sanitizedURL string) {
92103
}
93104

94105
// recordHTTPResponse records metrics for an HTTP response.
95-
func recordHTTPResponse(sanitizedURL string, statusCode int, durationSeconds float64) {
106+
func recordHTTPResponse(sanitizedURL string, statusCode int, durationSeconds float64, contentLength int64) {
96107
// Record latency
97108
httpRequestDuration.WithLabelValues(sanitizedURL).Observe(durationSeconds)
98109

99110
// Record non-200 responses
100111
if statusCode != 200 {
101112
httpNon200ResponsesTotal.WithLabelValues(sanitizedURL, strconv.Itoa(statusCode)).Inc()
102113
}
114+
115+
// Record response body size if known
116+
if contentLength >= 0 {
117+
httpResponseBodySizeBytes.WithLabelValues(sanitizedURL).Observe(float64(contentLength))
118+
}
103119
}
104120

105121
// recordNetworkError records metrics for failed HTTP response

‎pkg/common/http_test.go‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -405,6 +405,73 @@ func TestSaneHttpClientMetrics(t *testing.T) {
405405
}
406406
}
407407

408+
func TestRetryableHttpClientMetrics(t *testing.T) {
409+
// Create a test server that returns different status codes
410+
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
411+
switch r.URL.Path {
412+
case "/success":
413+
w.WriteHeader(http.StatusOK)
414+
_, _ = w.Write([]byte("success"))
415+
case "/error":
416+
w.WriteHeader(http.StatusInternalServerError)
417+
_, _ = w.Write([]byte("error"))
418+
case "/notfound":
419+
w.WriteHeader(http.StatusNotFound)
420+
_, _ = w.Write([]byte("not found"))
421+
default:
422+
w.WriteHeader(http.StatusOK)
423+
_, _ = w.Write([]byte("default"))
424+
}
425+
}))
426+
defer server.Close()
427+
428+
// Create a RetryableHttpClient
429+
client := RetryableHTTPClient()
430+
431+
testCases := []struct {
432+
name string
433+
path string
434+
expectedStatusCode int
435+
}{
436+
{
437+
name: "successful request",
438+
path: "/success",
439+
expectedStatusCode: 200,
440+
},
441+
{
442+
name: "not found request",
443+
path: "/notfound",
444+
expectedStatusCode: 404,
445+
},
446+
}
447+
448+
for _, tc := range testCases {
449+
t.Run(tc.name, func(t *testing.T) {
450+
var requestURL string
451+
if strings.HasPrefix(tc.path, "http") {
452+
requestURL = tc.path
453+
} else {
454+
requestURL = server.URL + tc.path
455+
}
456+
457+
// Get initial metric values
458+
sanitizedURL := sanitizeURL(requestURL)
459+
initialRequestsTotal := testutil.ToFloat64(httpRequestsTotal.WithLabelValues(sanitizedURL))
460+
461+
// Make the request
462+
resp, err := client.Get(requestURL)
463+
464+
require.NoError(t, err)
465+
defer resp.Body.Close()
466+
assert.Equal(t, tc.expectedStatusCode, resp.StatusCode)
467+
468+
// Check that request counter was incremented
469+
requestsTotal := testutil.ToFloat64(httpRequestsTotal.WithLabelValues(sanitizedURL))
470+
assert.Equal(t, initialRequestsTotal+1, requestsTotal)
471+
})
472+
}
473+
}
474+
408475
func TestInstrumentedTransport(t *testing.T) {
409476
// Create a mock transport that we can control
410477
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

‎pkg/detectors/gitlab/v1/gitlab_integration_test.go‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -297,6 +297,100 @@ func TestGitlab_FromChunk_WithV2Secrets(t *testing.T) {
297297
}
298298
}
299299

300+
// This test ensures gitlab v1 detector does not work on gitlab v3 secrets
301+
func TestGitlab_FromChunk_WithV3Secrets(t *testing.T) {
302+
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
303+
defer cancel()
304+
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6")
305+
if err != nil {
306+
t.Fatalf("could not get test secrets from GCP: %s", err)
307+
}
308+
secret := testSecrets.MustGetField("GITLABV3")
309+
secretInactive := testSecrets.MustGetField("GITLABV3_INACTIVE")
310+
type args struct {
311+
ctx context.Context
312+
data []byte
313+
verify bool
314+
}
315+
tests := []struct {
316+
name string
317+
s Scanner
318+
args args
319+
want []detectors.Result
320+
wantErr bool
321+
wantVerificationErr bool
322+
}{
323+
{
324+
name: "verified v3 secret, not found",
325+
s: Scanner{},
326+
args: args{
327+
ctx: context.Background(),
328+
data: []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
329+
verify: true,
330+
},
331+
want: nil,
332+
wantErr: false,
333+
},
334+
{
335+
name: "verified v3 secret, not found",
336+
s: Scanner{},
337+
args: args{
338+
ctx: context.Background(),
339+
data: []byte(fmt.Sprintf("gitlab %s", secret)),
340+
verify: true,
341+
},
342+
want: nil,
343+
wantErr: false,
344+
},
345+
{
346+
name: "unverified v3 secret, not found",
347+
s: Scanner{},
348+
args: args{
349+
ctx: context.Background(),
350+
data: []byte(fmt.Sprintf("You can find a gitlab secret %s within", secretInactive)),
351+
verify: true,
352+
},
353+
want: nil,
354+
wantErr: false,
355+
},
356+
{
357+
name: "not found",
358+
s: Scanner{},
359+
args: args{
360+
ctx: context.Background(),
361+
data: []byte("You cannot find the secret within"),
362+
verify: true,
363+
},
364+
want: nil,
365+
wantErr: false,
366+
},
367+
}
368+
for _, tt := range tests {
369+
t.Run(tt.name, func(t *testing.T) {
370+
tt.s.SetCloudEndpoint("https://gitlab.com")
371+
tt.s.UseCloudEndpoint(true)
372+
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
373+
if (err != nil) != tt.wantErr {
374+
t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
375+
return
376+
}
377+
for i := range got {
378+
if len(got[i].Raw) == 0 {
379+
t.Fatal("no raw secret present")
380+
}
381+
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
382+
t.Fatalf(" wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError())
383+
}
384+
got[i].AnalysisInfo = nil
385+
}
386+
opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError", "primarySecret")
387+
if diff := cmp.Diff(got, tt.want, opts); diff != "" {
388+
t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
389+
}
390+
})
391+
}
392+
}
393+
300394
func BenchmarkV2FromData(benchmark *testing.B) {
301395
ctx := context.Background()
302396
s := Scanner{}

‎pkg/detectors/gitlab/v2/gitlab_integration_test.go‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,100 @@ func TestGitlabV2_FromChunk_WithV1Secrets(t *testing.T) {
111111
}
112112
}
113113

114+
// This test ensures gitlab v2 detector does not work on gitlab v3 secrets
115+
func TestGitlabV2_FromChunk_WithV3Secrets(t *testing.T) {
116+
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
117+
defer cancel()
118+
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6")
119+
if err != nil {
120+
t.Fatalf("could not get test secrets from GCP: %s", err)
121+
}
122+
secret := testSecrets.MustGetField("GITLABV3")
123+
secretInactive := testSecrets.MustGetField("GITLABV3_INACTIVE")
124+
type args struct {
125+
ctx context.Context
126+
data []byte
127+
verify bool
128+
}
129+
tests := []struct {
130+
name string
131+
s Scanner
132+
args args
133+
want []detectors.Result
134+
wantErr bool
135+
wantVerificationErr bool
136+
}{
137+
{
138+
name: "verified v3 secret, not found",
139+
s: Scanner{},
140+
args: args{
141+
ctx: context.Background(),
142+
data: []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
143+
verify: true,
144+
},
145+
want: nil,
146+
wantErr: false,
147+
},
148+
{
149+
name: "verified v3 secret, not found",
150+
s: Scanner{},
151+
args: args{
152+
ctx: context.Background(),
153+
data: []byte(fmt.Sprintf("gitlab %s", secret)),
154+
verify: true,
155+
},
156+
want: nil,
157+
wantErr: false,
158+
},
159+
{
160+
name: "unverified v3 secret, not found",
161+
s: Scanner{},
162+
args: args{
163+
ctx: context.Background(),
164+
data: []byte(fmt.Sprintf("You can find a gitlab secret %s within", secretInactive)),
165+
verify: true,
166+
},
167+
want: nil,
168+
wantErr: false,
169+
},
170+
{
171+
name: "not found",
172+
s: Scanner{},
173+
args: args{
174+
ctx: context.Background(),
175+
data: []byte("You cannot find the secret within"),
176+
verify: true,
177+
},
178+
want: nil,
179+
wantErr: false,
180+
},
181+
}
182+
for _, tt := range tests {
183+
t.Run(tt.name, func(t *testing.T) {
184+
tt.s.SetCloudEndpoint("https://gitlab.com")
185+
tt.s.UseCloudEndpoint(true)
186+
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
187+
if (err != nil) != tt.wantErr {
188+
t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
189+
return
190+
}
191+
for i := range got {
192+
if len(got[i].Raw) == 0 {
193+
t.Fatal("no raw secret present")
194+
}
195+
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
196+
t.Fatalf(" wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError())
197+
}
198+
got[i].AnalysisInfo = nil
199+
}
200+
opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError", "primarySecret")
201+
if diff := cmp.Diff(got, tt.want, opts); diff != "" {
202+
t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
203+
}
204+
})
205+
}
206+
}
207+
114208
func BenchmarkFromData(benchmark *testing.B) {
115209
ctx := context.Background()
116210
s := Scanner{}

0 commit comments

Comments
 (0)