Skip to content

Commit 244c6f4

Browse files
mcombuechenmcombuechen
committed
fix: handle Snyk API rate limit
Closes #83.
1 parent 6f99e54 commit 244c6f4

File tree

7 files changed

+115
-10
lines changed

7 files changed

+115
-10
lines changed

‎go.mod‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ require (
77
github.com/deepmap/oapi-codegen v1.12.4
88
github.com/edoardottt/depsdev v0.0.3
99
github.com/google/uuid v1.3.0
10+
github.com/hashicorp/go-retryablehttp v0.7.7
1011
github.com/jarcoal/httpmock v1.3.0
1112
github.com/package-url/packageurl-go v0.1.2
1213
github.com/remeh/sizedwaitgroup v1.0.0
@@ -23,11 +24,12 @@ require (
2324
github.com/avast/retry-go v3.0.0+incompatible // indirect
2425
github.com/davecgh/go-spew v1.1.1 // indirect
2526
github.com/fsnotify/fsnotify v1.6.0 // indirect
27+
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
2628
github.com/hashicorp/hcl v1.0.0 // indirect
2729
github.com/inconshreveable/mousetrap v1.1.0 // indirect
2830
github.com/magiconair/properties v1.8.7 // indirect
2931
github.com/mattn/go-colorable v0.1.13 // indirect
30-
github.com/mattn/go-isatty v0.0.16 // indirect
32+
github.com/mattn/go-isatty v0.0.20 // indirect
3133
github.com/mitchellh/mapstructure v1.5.0 // indirect
3234
github.com/pelletier/go-toml/v2 v2.0.6 // indirect
3335
github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -36,7 +38,7 @@ require (
3638
github.com/spf13/jwalterweatherman v1.1.0 // indirect
3739
github.com/spf13/pflag v1.0.5 // indirect
3840
github.com/subosito/gotenv v1.4.2 // indirect
39-
golang.org/x/sys v0.3.0 // indirect
41+
golang.org/x/sys v0.20.0 // indirect
4042
golang.org/x/text v0.5.0 // indirect
4143
gopkg.in/ini.v1 v1.67.0 // indirect
4244
gopkg.in/yaml.v3 v3.0.1 // indirect

‎go.sum‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
7373
github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
7474
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
7575
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
76+
github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
7677
github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
7778
github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
7879
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
@@ -140,6 +141,11 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
140141
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
141142
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
142143
github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
144+
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
145+
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
146+
github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
147+
github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
148+
github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
143149
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
144150
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
145151
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
@@ -168,8 +174,9 @@ github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb
168174
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
169175
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
170176
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
171-
github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
172177
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
178+
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
179+
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
173180
github.com/maxatome/go-testdeep v1.12.0 h1:Ql7Go8Tg0C1D/uMMX59LAoYK7LffeJQ6X2T04nTH68g=
174181
github.com/maxatome/go-testdeep v1.12.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM=
175182
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
@@ -371,8 +378,9 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
371378
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
372379
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
373380
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
374-
golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
375-
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
381+
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
382+
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
383+
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
376384
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
377385
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
378386
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

‎lib/snyk/enrich_cyclonedx.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ func enrichCycloneDX(cfg *Config, bom *cdx.BOM, logger *zerolog.Logger) *cdx.BOM
107107
for _, enrichFunc := range cdxEnrichers {
108108
enrichFunc(cfg, component, &purl)
109109
}
110-
resp, err := GetPackageVulnerabilities(cfg, &purl, auth, orgID)
110+
resp, err := GetPackageVulnerabilities(cfg, &purl, auth, orgID, logger)
111111
if err != nil {
112112
l.Err(err).
113113
Str("purl", purl.ToString()).

‎lib/snyk/enrich_spdx.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ func enrichSPDX(cfg *Config, bom *spdx.Document, logger *zerolog.Logger) *spdx.D
112112
for _, enrichFn := range spdxEnrichers {
113113
enrichFn(cfg, pkg, purl)
114114
}
115-
resp, err := GetPackageVulnerabilities(cfg, purl, auth, orgID)
115+
resp, err := GetPackageVulnerabilities(cfg, purl, auth, orgID, logger)
116116
if err != nil {
117117
l.Err(err).
118118
Str("purl", purl.ToString()).

‎lib/snyk/package.go‎

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,14 @@ import (
2020
"context"
2121
"fmt"
2222
"net/http"
23+
"strconv"
24+
"time"
2325

2426
"github.com/deepmap/oapi-codegen/pkg/securityprovider"
2527
"github.com/google/uuid"
28+
"github.com/hashicorp/go-retryablehttp"
2629
"github.com/package-url/packageurl-go"
30+
"github.com/rs/zerolog"
2731

2832
"github.com/snyk/parlay/snyk/issues"
2933
)
@@ -82,8 +86,11 @@ func SnykVulnURL(cfg *Config, purl *packageurl.PackageURL) string {
8286
return url
8387
}
8488

85-
func GetPackageVulnerabilities(cfg *Config, purl *packageurl.PackageURL, auth *securityprovider.SecurityProviderApiKey, orgID *uuid.UUID) (*issues.FetchIssuesPerPurlResponse, error) {
86-
client, err := issues.NewClientWithResponses(cfg.SnykAPIURL, issues.WithRequestEditorFn(auth.Intercept))
89+
func GetPackageVulnerabilities(cfg *Config, purl *packageurl.PackageURL, auth *securityprovider.SecurityProviderApiKey, orgID *uuid.UUID, logger *zerolog.Logger) (*issues.FetchIssuesPerPurlResponse, error) {
90+
client, err := issues.NewClientWithResponses(
91+
cfg.SnykAPIURL,
92+
issues.WithRequestEditorFn(auth.Intercept),
93+
issues.WithHTTPClient(getRetryClient(logger)))
8794
if err != nil {
8895
return nil, err
8996
}
@@ -100,3 +107,31 @@ func GetPackageVulnerabilities(cfg *Config, purl *packageurl.PackageURL, auth *s
100107

101108
return resp, nil
102109
}
110+
111+
func getRetryClient(logger *zerolog.Logger) *http.Client {
112+
rc := retryablehttp.NewClient()
113+
rc.Logger = nil
114+
rc.Backoff = func(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration {
115+
if sleep, ok := parseRateLimitHeader(resp.Header.Get("X-RateLimit-Reset")); ok {
116+
logger.Warn().
117+
Dur("Retry-After", sleep).
118+
Msg("Getting rate-limited, waiting...")
119+
return sleep
120+
}
121+
return retryablehttp.DefaultBackoff(min, max, attemptNum, resp)
122+
}
123+
124+
return rc.StandardClient()
125+
}
126+
127+
func parseRateLimitHeader(v string) (time.Duration, bool) {
128+
if v == "" {
129+
return 0, false
130+
}
131+
132+
if sec, err := strconv.ParseInt(v, 10, 64); err == nil {
133+
return time.Duration(sec) * time.Second, true
134+
}
135+
136+
return 0, false
137+
}

‎lib/snyk/package_test.go‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
/*
2+
* © 2023 Snyk Limited All rights reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package snyk
18+
19+
import (
20+
"net/http"
21+
"net/http/httptest"
22+
"testing"
23+
24+
"github.com/google/uuid"
25+
"github.com/package-url/packageurl-go"
26+
"github.com/rs/zerolog"
27+
"github.com/stretchr/testify/assert"
28+
"github.com/stretchr/testify/require"
29+
)
30+
31+
func TestGetPackageVulnerabilities_RetryRateLimited(t *testing.T) {
32+
logger := zerolog.Nop()
33+
var numRequests int
34+
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
35+
numRequests++
36+
if numRequests == 1 {
37+
w.Header().Set("X-RateLimit-Reset", "1")
38+
w.WriteHeader(http.StatusTooManyRequests)
39+
return
40+
}
41+
w.Header().Set("Content-Type", "application/vnd.json+api")
42+
_, err := w.Write([]byte(`{"data":[{"type":"issues","id":"VULN-ID"}]}`))
43+
require.NoError(t, err)
44+
}))
45+
cfg := DefaultConfig()
46+
cfg.SnykAPIURL = srv.URL
47+
48+
auth, err := AuthFromToken("asdf")
49+
require.NoError(t, err)
50+
51+
purl, err := packageurl.FromString("pkg:golang/github.com/snyk/parlay")
52+
require.NoError(t, err)
53+
54+
orgID := uuid.New()
55+
issues, err := GetPackageVulnerabilities(cfg, &purl, auth, &orgID, &logger)
56+
require.NoError(t, err)
57+
58+
assert.NotZero(t, numRequests, "retries failed requests")
59+
assert.NotNil(t, issues, "should retrieve issues")
60+
}

‎lib/snyk/service.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ func (svc *serviceImpl) GetPackageVulnerabilities(purl *packageurl.PackageURL) (
4141
return nil, err
4242
}
4343

44-
return GetPackageVulnerabilities(svc.cfg, purl, auth, orgID)
44+
return GetPackageVulnerabilities(svc.cfg, purl, auth, orgID, svc.logger)
4545
}
4646

4747
func (svc *serviceImpl) getAuth() (*securityprovider.SecurityProviderApiKey, error) {

0 commit comments

Comments
 (0)