From db46e5db8678e1ccbb76379901ff8377ed95f354 Mon Sep 17 00:00:00 2001 From: Mustansir Muzaffar Date: Fri, 5 Dec 2025 17:14:38 +0500 Subject: [PATCH 1/2] abort scan if monthly request limit crosses 80% --- pkg/sources/postman/postman.go | 12 ++++ pkg/sources/postman/postman_client.go | 81 ++++++++++++++++++++------- pkg/sources/postman/postman_test.go | 65 +++++++++++++++++++++ 3 files changed, 137 insertions(+), 21 deletions(-) diff --git a/pkg/sources/postman/postman.go b/pkg/sources/postman/postman.go index db8e81b26ef4..84ed8a781530 100644 --- a/pkg/sources/postman/postman.go +++ b/pkg/sources/postman/postman.go @@ -186,6 +186,9 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ . for _, workspaceID := range s.conn.Workspaces { w, err := s.client.GetWorkspace(ctx, workspaceID) if err != nil { + if errors.Is(err, errAbortScanDueToAPIRateLimit) { + return err + } // Log and move on, because sometimes the Postman API seems to give us workspace IDs // that we don't have access to, so we don't want to kill the scan because of it. ctx.Logger().Error(err, "error getting workspace %s", workspaceID) @@ -206,6 +209,9 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ . collection, err := s.client.GetCollection(ctx, collectionID) if err != nil { + if errors.Is(err, errAbortScanDueToAPIRateLimit) { + return err + } // Log and move on, because sometimes the Postman API seems to give us collection IDs // that we don't have access to, so we don't want to kill the scan because of it. ctx.Logger().Error(err, "error getting collection %s", collectionID) @@ -279,6 +285,9 @@ func (s *Source) scanWorkspace(ctx context.Context, chunksChan chan *sources.Chu for _, envID := range workspace.Environments { envVars, err := s.client.GetEnvironmentVariables(ctx, envID.Uid) if err != nil { + if errors.Is(err, errAbortScanDueToAPIRateLimit) { + return err + } ctx.Logger().Error(err, "could not get env variables", "environment_uuid", envID.Uid) continue } @@ -316,6 +325,9 @@ func (s *Source) scanWorkspace(ctx context.Context, chunksChan chan *sources.Chu } collection, err := s.client.GetCollection(ctx, collectionID.Uid) if err != nil { + if errors.Is(err, errAbortScanDueToAPIRateLimit) { + return err + } // Log and move on, because sometimes the Postman API seems to give us collection IDs // that we don't have access to, so we don't want to kill the scan because of it. ctx.Logger().Error(err, "error getting collection %s", collectionID) diff --git a/pkg/sources/postman/postman_client.go b/pkg/sources/postman/postman_client.go index ef8c3b132dda..74c9414a7925 100644 --- a/pkg/sources/postman/postman_client.go +++ b/pkg/sources/postman/postman_client.go @@ -18,13 +18,16 @@ import ( ) const ( - WORKSPACE_URL = "https://api.getpostman.com/workspaces/%s" - ENVIRONMENTS_URL = "https://api.getpostman.com/environments/%s" - COLLECTIONS_URL = "https://api.getpostman.com/collections/%s" - userAgent = "PostmanRuntime/7.26.8" - defaultContentType = "*" + WORKSPACE_URL = "https://api.getpostman.com/workspaces/%s" + ENVIRONMENTS_URL = "https://api.getpostman.com/environments/%s" + COLLECTIONS_URL = "https://api.getpostman.com/collections/%s" + userAgent = "PostmanRuntime/7.26.8" + defaultContentType = "*" + abortScanAPIReqLimitThreshold = 0.8 // Abort scan if 80% of monthly api request limit is reached ) +var errAbortScanDueToAPIRateLimit = fmt.Errorf("aborting scan due to Postman API monthly requests limit being used over %f%%", abortScanAPIReqLimitThreshold*100) + type Workspace struct { Id string `json:"id"` Name string `json:"name"` @@ -282,22 +285,9 @@ func (c *Client) getPostmanResponseBodyBytes(ctx trContext.Context, urlString st c.Metrics.apiRequests.WithLabelValues(urlString).Inc() - rateLimitRemainingMonthValue := resp.Header.Get("RateLimit-Remaining-Month") - if rateLimitRemainingMonthValue == "" { - rateLimitRemainingMonthValue = resp.Header.Get("X-RateLimit-Remaining-Month") - } - - if rateLimitRemainingMonthValue != "" { - rateLimitRemainingMonth, err := strconv.Atoi(rateLimitRemainingMonthValue) - if err != nil { - ctx.Logger().Error(err, "Couldn't convert RateLimit-Remaining-Month to an int", - "header_value", rateLimitRemainingMonthValue, - ) - } else { - c.Metrics.apiMonthlyRequestsRemaining.WithLabelValues().Set( - float64(rateLimitRemainingMonth), - ) - } + err = c.handleRateLimits(ctx, resp) + if err != nil { + return nil, err } body, err := io.ReadAll(resp.Body) @@ -313,6 +303,55 @@ func (c *Client) getPostmanResponseBodyBytes(ctx trContext.Context, urlString st return body, nil } +// handleRateLimits processes the rate limit headers from the Postman API response +// and updates the client's metrics accordingly. If the monthly rate limit usage exceeds +// the set threshold, an error is returned to abort further processing. +func (c *Client) handleRateLimits(ctx trContext.Context, resp *http.Response) error { + rateLimitRemainingMonthValue := resp.Header.Get("RateLimit-Remaining-Month") + if rateLimitRemainingMonthValue == "" { + rateLimitRemainingMonthValue = resp.Header.Get("X-RateLimit-Remaining-Month") + } + if rateLimitRemainingMonthValue == "" { + ctx.Logger().V(2).Info("RateLimit-Remaining-Month header not found in response") + return nil + } + + rateLimitRemainingMonth, err := strconv.Atoi(rateLimitRemainingMonthValue) + if err != nil { + ctx.Logger().Error(err, "Couldn't convert RateLimit-Remaining-Month to an int", + "header_value", rateLimitRemainingMonthValue, + ) + return nil + } + c.Metrics.apiMonthlyRequestsRemaining.WithLabelValues().Set( + float64(rateLimitRemainingMonth), + ) + + rateLimitTotalMonthValue := resp.Header.Get("RateLimit-Limit-Month") + if rateLimitTotalMonthValue == "" { + rateLimitTotalMonthValue = resp.Header.Get("X-RateLimit-Limit-Month") + } + if rateLimitTotalMonthValue == "" { + ctx.Logger().V(2).Info("RateLimit-Limit-Month header not found in response") + return nil + } + rateLimitTotalMonth, err := strconv.Atoi(rateLimitTotalMonthValue) + if err != nil { + ctx.Logger().Error(err, "Couldn't convert RateLimit-Limit-Month to an int", + "header_value", rateLimitTotalMonthValue, + ) + return nil + } + + // failsafe to abandon scan if we are over the threshold of monthly API requests used + percentageUsed := float64(rateLimitTotalMonth-rateLimitRemainingMonth) / float64(rateLimitTotalMonth) + if percentageUsed > abortScanAPIReqLimitThreshold { + return errAbortScanDueToAPIRateLimit + } + + return nil +} + // EnumerateWorkspaces returns the workspaces for a given user (both private, public, team and personal). // Consider adding additional flags to support filtering. func (c *Client) EnumerateWorkspaces(ctx trContext.Context) ([]Workspace, error) { diff --git a/pkg/sources/postman/postman_test.go b/pkg/sources/postman/postman_test.go index 9e88d39b035c..599993b97290 100644 --- a/pkg/sources/postman/postman_test.go +++ b/pkg/sources/postman/postman_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/go-errors/errors" "github.com/stretchr/testify/assert" "github.com/trufflesecurity/trufflehog/v3/pkg/context" @@ -689,3 +690,67 @@ func TestSource_HeadersScanning(t *testing.T) { t.Logf("Generated %d chunks from the mock data", chunksReceived) } } + +func TestSource_AbortScanRateLimitCrossedThreshold(t *testing.T) { + defer gock.Off() + // Mock the API response for getting collection + gock.New("https://api.getpostman.com"). + Get("/collections/1234-abc1"). + Reply(200). + AddHeader("X-RateLimit-Remaining-Month", "19"). + AddHeader("X-RateLimit-Limit-Month", "100"). + BodyString(`{"collection":{"info":{"_postman_id":"abc1","name":"test-collection-1","schema":"https://schema.postman.com/json/collection/v2.1.0/collection.json", + "updatedAt":"2025-03-21T17:39:25.000Z","createdAt":"2025-03-21T17:37:13.000Z","lastUpdatedBy":"1234","uid":"1234-abc1"}, + "item":[]}}`) + + ctx := context.Background() + s, conn := createTestSource(&sourcespb.Postman{ + Credential: &sourcespb.Postman_Token{ + Token: "super-secret-token", + }, + }) + + err := s.Init(ctx, "test - postman", 0, 1, false, conn, 1) + if err != nil { + t.Fatalf("init error: %v", err) + } + gock.InterceptClient(s.client.HTTPClient) + defer gock.RestoreClient(s.client.HTTPClient) + + _, err = s.client.GetCollection(ctx, "1234-abc1") + if !errors.Is(err, errAbortScanDueToAPIRateLimit) { + t.Errorf("expected abort scan error due to rate limit, got: %v", err) + } +} + +func TestSource_AbortScanRateLimitBelowThreshold(t *testing.T) { + defer gock.Off() + // Mock the API response for getting collection + gock.New("https://api.getpostman.com"). + Get("/collections/1234-abc1"). + Reply(200). + AddHeader("X-RateLimit-Remaining-Month", "90"). + AddHeader("X-RateLimit-Limit-Month", "100"). + BodyString(`{"collection":{"info":{"_postman_id":"abc1","name":"test-collection-1","schema":"https://schema.postman.com/json/collection/v2.1.0/collection.json", + "updatedAt":"2025-03-21T17:39:25.000Z","createdAt":"2025-03-21T17:37:13.000Z","lastUpdatedBy":"1234","uid":"1234-abc1"}, + "item":[]}}`) + + ctx := context.Background() + s, conn := createTestSource(&sourcespb.Postman{ + Credential: &sourcespb.Postman_Token{ + Token: "super-secret-token", + }, + }) + + err := s.Init(ctx, "test - postman", 0, 1, false, conn, 1) + if err != nil { + t.Fatalf("init error: %v", err) + } + gock.InterceptClient(s.client.HTTPClient) + defer gock.RestoreClient(s.client.HTTPClient) + + _, err = s.client.GetCollection(ctx, "1234-abc1") + if err != nil { + t.Errorf("expected no error, got: %v", err) + } +} From c26f1ff1eefa6369810275f99fadf015beea500e Mon Sep 17 00:00:00 2001 From: Mustansir Muzaffar Date: Fri, 5 Dec 2025 18:50:28 +0500 Subject: [PATCH 2/2] add check for rateLimiTotalMonth == 0 --- pkg/sources/postman/postman_client.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/sources/postman/postman_client.go b/pkg/sources/postman/postman_client.go index 68b87bedc95f..e41ee84405ec 100644 --- a/pkg/sources/postman/postman_client.go +++ b/pkg/sources/postman/postman_client.go @@ -345,6 +345,11 @@ func (c *Client) handleRateLimits(ctx trContext.Context, resp *http.Response) er return nil } + if rateLimitTotalMonth == 0 { + ctx.Logger().V(2).Info("RateLimit-Limit-Month is zero, cannot compute usage percentage") + return nil + } + // failsafe to abandon scan if we are over the threshold of monthly API requests used percentageUsed := float64(rateLimitTotalMonth-rateLimitRemainingMonth) / float64(rateLimitTotalMonth) if percentageUsed > abortScanAPIReqLimitThreshold {