Skip to content

Commit 2b35d68

Browse files
mustansir14mustansir14
authored
Add Gitlab V3 Detector (#4563)
* add gitlab v3 detector * add detectors build tag to test * add gitlab v3 detector to engine defaults * enhance regex * add comment to link gitlab's PR
1 parent 674f626 commit 2b35d68

File tree

6 files changed

+750
-0
lines changed

6 files changed

+750
-0
lines changed

‎pkg/detectors/gitlab/v1/gitlab_integration_test.go‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -297,6 +297,100 @@ func TestGitlab_FromChunk_WithV2Secrets(t *testing.T) {
297297
}
298298
}
299299

300+
// This test ensures gitlab v1 detector does not work on gitlab v3 secrets
301+
func TestGitlab_FromChunk_WithV3Secrets(t *testing.T) {
302+
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
303+
defer cancel()
304+
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6")
305+
if err != nil {
306+
t.Fatalf("could not get test secrets from GCP: %s", err)
307+
}
308+
secret := testSecrets.MustGetField("GITLABV3")
309+
secretInactive := testSecrets.MustGetField("GITLABV3_INACTIVE")
310+
type args struct {
311+
ctx context.Context
312+
data []byte
313+
verify bool
314+
}
315+
tests := []struct {
316+
name string
317+
s Scanner
318+
args args
319+
want []detectors.Result
320+
wantErr bool
321+
wantVerificationErr bool
322+
}{
323+
{
324+
name: "verified v3 secret, not found",
325+
s: Scanner{},
326+
args: args{
327+
ctx: context.Background(),
328+
data: []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
329+
verify: true,
330+
},
331+
want: nil,
332+
wantErr: false,
333+
},
334+
{
335+
name: "verified v3 secret, not found",
336+
s: Scanner{},
337+
args: args{
338+
ctx: context.Background(),
339+
data: []byte(fmt.Sprintf("gitlab %s", secret)),
340+
verify: true,
341+
},
342+
want: nil,
343+
wantErr: false,
344+
},
345+
{
346+
name: "unverified v3 secret, not found",
347+
s: Scanner{},
348+
args: args{
349+
ctx: context.Background(),
350+
data: []byte(fmt.Sprintf("You can find a gitlab secret %s within", secretInactive)),
351+
verify: true,
352+
},
353+
want: nil,
354+
wantErr: false,
355+
},
356+
{
357+
name: "not found",
358+
s: Scanner{},
359+
args: args{
360+
ctx: context.Background(),
361+
data: []byte("You cannot find the secret within"),
362+
verify: true,
363+
},
364+
want: nil,
365+
wantErr: false,
366+
},
367+
}
368+
for _, tt := range tests {
369+
t.Run(tt.name, func(t *testing.T) {
370+
tt.s.SetCloudEndpoint("https://gitlab.com")
371+
tt.s.UseCloudEndpoint(true)
372+
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
373+
if (err != nil) != tt.wantErr {
374+
t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
375+
return
376+
}
377+
for i := range got {
378+
if len(got[i].Raw) == 0 {
379+
t.Fatal("no raw secret present")
380+
}
381+
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
382+
t.Fatalf(" wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError())
383+
}
384+
got[i].AnalysisInfo = nil
385+
}
386+
opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError", "primarySecret")
387+
if diff := cmp.Diff(got, tt.want, opts); diff != "" {
388+
t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
389+
}
390+
})
391+
}
392+
}
393+
300394
func BenchmarkV2FromData(benchmark *testing.B) {
301395
ctx := context.Background()
302396
s := Scanner{}

‎pkg/detectors/gitlab/v2/gitlab_integration_test.go‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,100 @@ func TestGitlabV2_FromChunk_WithV1Secrets(t *testing.T) {
111111
}
112112
}
113113

114+
// This test ensures gitlab v2 detector does not work on gitlab v3 secrets
115+
func TestGitlabV2_FromChunk_WithV3Secrets(t *testing.T) {
116+
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
117+
defer cancel()
118+
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6")
119+
if err != nil {
120+
t.Fatalf("could not get test secrets from GCP: %s", err)
121+
}
122+
secret := testSecrets.MustGetField("GITLABV3")
123+
secretInactive := testSecrets.MustGetField("GITLABV3_INACTIVE")
124+
type args struct {
125+
ctx context.Context
126+
data []byte
127+
verify bool
128+
}
129+
tests := []struct {
130+
name string
131+
s Scanner
132+
args args
133+
want []detectors.Result
134+
wantErr bool
135+
wantVerificationErr bool
136+
}{
137+
{
138+
name: "verified v3 secret, not found",
139+
s: Scanner{},
140+
args: args{
141+
ctx: context.Background(),
142+
data: []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)),
143+
verify: true,
144+
},
145+
want: nil,
146+
wantErr: false,
147+
},
148+
{
149+
name: "verified v3 secret, not found",
150+
s: Scanner{},
151+
args: args{
152+
ctx: context.Background(),
153+
data: []byte(fmt.Sprintf("gitlab %s", secret)),
154+
verify: true,
155+
},
156+
want: nil,
157+
wantErr: false,
158+
},
159+
{
160+
name: "unverified v3 secret, not found",
161+
s: Scanner{},
162+
args: args{
163+
ctx: context.Background(),
164+
data: []byte(fmt.Sprintf("You can find a gitlab secret %s within", secretInactive)),
165+
verify: true,
166+
},
167+
want: nil,
168+
wantErr: false,
169+
},
170+
{
171+
name: "not found",
172+
s: Scanner{},
173+
args: args{
174+
ctx: context.Background(),
175+
data: []byte("You cannot find the secret within"),
176+
verify: true,
177+
},
178+
want: nil,
179+
wantErr: false,
180+
},
181+
}
182+
for _, tt := range tests {
183+
t.Run(tt.name, func(t *testing.T) {
184+
tt.s.SetCloudEndpoint("https://gitlab.com")
185+
tt.s.UseCloudEndpoint(true)
186+
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
187+
if (err != nil) != tt.wantErr {
188+
t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
189+
return
190+
}
191+
for i := range got {
192+
if len(got[i].Raw) == 0 {
193+
t.Fatal("no raw secret present")
194+
}
195+
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
196+
t.Fatalf(" wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError())
197+
}
198+
got[i].AnalysisInfo = nil
199+
}
200+
opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError", "primarySecret")
201+
if diff := cmp.Diff(got, tt.want, opts); diff != "" {
202+
t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
203+
}
204+
})
205+
}
206+
}
207+
114208
func BenchmarkFromData(benchmark *testing.B) {
115209
ctx := context.Background()
116210
s := Scanner{}

‎pkg/detectors/gitlab/v3/gitlab_v3.go‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
package gitlab
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"maps"
7+
"net/http"
8+
"strings"
9+
10+
regexp "github.com/wasilibs/go-re2"
11+
12+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
13+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
14+
v1 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitlab/v1"
15+
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
16+
)
17+
18+
type Scanner struct {
19+
client *http.Client
20+
detectors.EndpointSetter
21+
}
22+
23+
// Ensure the Scanner satisfies the interfaces at compile time.
24+
var _ detectors.Detector = (*Scanner)(nil)
25+
var _ detectors.EndpointCustomizer = (*Scanner)(nil)
26+
var _ detectors.Versioner = (*Scanner)(nil)
27+
28+
func (Scanner) Version() int { return 3 }
29+
func (Scanner) CloudEndpoint() string { return "https://gitlab.com" }
30+
31+
var (
32+
defaultClient = common.SaneHttpClient()
33+
// pattern taken from gitlab's PR for the format change: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/169322
34+
keyPat = regexp.MustCompile(`\b(glpat-[a-zA-Z0-9\-=_]{27,300}.[0-9a-z]{2}.[a-z0-9]{9})\b`)
35+
)
36+
37+
func (s Scanner) getClient() *http.Client {
38+
if s.client != nil {
39+
return s.client
40+
}
41+
42+
return defaultClient
43+
}
44+
45+
// Keywords are used for efficiently pre-filtering chunks.
46+
// Use identifiers in the secret preferably, or the provider name.
47+
func (s Scanner) Keywords() []string { return []string{"glpat-"} }
48+
49+
func (s Scanner) Type() detectorspb.DetectorType {
50+
return detectorspb.DetectorType_Gitlab
51+
}
52+
53+
func (s Scanner) Description() string {
54+
return "GitLab is a web-based DevOps lifecycle tool that provides a Git repository manager providing wiki, issue-tracking, and CI/CD pipeline features. GitLab Personal Access Tokens (PATs) can be used to authenticate and access GitLab resources."
55+
}
56+
57+
// FromData will find and optionally verify Gitlab secrets in a given set of bytes.
58+
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
59+
dataStr := string(data)
60+
61+
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
62+
for _, match := range matches {
63+
resMatch := strings.TrimSpace(match[1])
64+
65+
for _, endpoint := range s.Endpoints() {
66+
s1 := detectors.Result{
67+
DetectorType: detectorspb.DetectorType_Gitlab,
68+
Raw: []byte(resMatch),
69+
RawV2: []byte(resMatch + endpoint),
70+
ExtraData: map[string]string{
71+
"rotation_guide": "https://howtorotate.com/docs/tutorials/gitlab/",
72+
"version": fmt.Sprintf("%d", s.Version()),
73+
},
74+
}
75+
76+
if verify {
77+
78+
isVerified, extraData, verificationErr := v1.VerifyGitlab(ctx, s.getClient(), endpoint, resMatch)
79+
s1.Verified = isVerified
80+
maps.Copy(s1.ExtraData, extraData)
81+
82+
s1.SetVerificationError(verificationErr)
83+
84+
// for verified keys set the analysis info
85+
if s1.Verified {
86+
s1.AnalysisInfo = map[string]string{
87+
"key": resMatch,
88+
"host": endpoint,
89+
}
90+
91+
// if secret is verified with one endpoint, break the loop to continue to next secret
92+
results = append(results, s1)
93+
break
94+
}
95+
}
96+
97+
results = append(results, s1)
98+
}
99+
}
100+
101+
return results, nil
102+
}

0 commit comments

Comments
 (0)