feat: allow test files to be scanned (#1542)

* feat: allow test files to be scanned

* docs: update docs

docs/_data/bearer_scan.yaml

@@ -162,6 +162,11 @@ options:
       Specify the comma-separated ids of the rules you would like to skip. Runs all other rules.
     environment_variables:
       - BEARER_SKIP_RULE
+  - name: skip-test
+    default_value: "true"
+    usage: Disable automatic skipping of test files
+    environment_variables:
+      - BEARER_SKIP_TEST
 example: |4-
       # Scan a local project, including language-specific files
       $ bearer scan /path/to/your_project

e2e/flags/.snapshots/TestInitCommand

@@ -26,4 +26,5 @@ scan:
     scanner:
         - sast
     skip-path: []
+    skip-test: true
 

e2e/flags/.snapshots/TestMetadataFlags-help-scan

@@ -36,6 +36,7 @@ Scan Flags
       --quiet                                Suppress non-essential messages
       --scanner strings                      Specify which scanner to use e.g. --scanner=secrets, --scanner=secrets,sast (default [sast])
       --skip-path strings                    Specify the comma separated files and directories to skip. Supports * syntax, e.g. --skip-path users/*.go,users/admin.sql
+      --skip-test                            Disable automatic skipping of test files (default true)
 
 General Flags
       --api-key string          Use your Bearer API Key to send the report to Bearer.

e2e/flags/.snapshots/TestMetadataFlags-scan-help

@@ -36,6 +36,7 @@ Scan Flags
       --quiet                                Suppress non-essential messages
       --scanner strings                      Specify which scanner to use e.g. --scanner=secrets, --scanner=secrets,sast (default [sast])
       --skip-path strings                    Specify the comma separated files and directories to skip. Supports * syntax, e.g. --skip-path users/*.go,users/admin.sql
+      --skip-test                            Disable automatic skipping of test files (default true)
 
 General Flags
       --api-key string          Use your Bearer API Key to send the report to Bearer.

e2e/flags/.snapshots/TestReportFlagsShouldFail-invalid-context-flag

@@ -37,6 +37,7 @@ Scan Flags
       --quiet                                Suppress non-essential messages
       --scanner strings                      Specify which scanner to use e.g. --scanner=secrets, --scanner=secrets,sast (default [sast])
       --skip-path strings                    Specify the comma separated files and directories to skip. Supports * syntax, e.g. --skip-path users/*.go,users/admin.sql
+      --skip-test                            Disable automatic skipping of test files (default true)
 
 General Flags
       --api-key string          Use your Bearer API Key to send the report to Bearer.

e2e/flags/.snapshots/TestReportFlagsShouldFail-invalid-format-flag-privacy

@@ -37,6 +37,7 @@ Scan Flags
       --quiet                                Suppress non-essential messages
       --scanner strings                      Specify which scanner to use e.g. --scanner=secrets, --scanner=secrets,sast (default [sast])
       --skip-path strings                    Specify the comma separated files and directories to skip. Supports * syntax, e.g. --skip-path users/*.go,users/admin.sql
+      --skip-test                            Disable automatic skipping of test files (default true)
 
 General Flags
       --api-key string          Use your Bearer API Key to send the report to Bearer.

e2e/flags/.snapshots/TestReportFlagsShouldFail-invalid-format-flag-security

@@ -37,6 +37,7 @@ Scan Flags
       --quiet                                Suppress non-essential messages
       --scanner strings                      Specify which scanner to use e.g. --scanner=secrets, --scanner=secrets,sast (default [sast])
       --skip-path strings                    Specify the comma separated files and directories to skip. Supports * syntax, e.g. --skip-path users/*.go,users/admin.sql
+      --skip-test                            Disable automatic skipping of test files (default true)
 
 General Flags
       --api-key string          Use your Bearer API Key to send the report to Bearer.

e2e/flags/.snapshots/TestReportFlagsShouldFail-invalid-report-flag

@@ -37,6 +37,7 @@ Scan Flags
       --quiet                                Suppress non-essential messages
       --scanner strings                      Specify which scanner to use e.g. --scanner=secrets, --scanner=secrets,sast (default [sast])
       --skip-path strings                    Specify the comma separated files and directories to skip. Supports * syntax, e.g. --skip-path users/*.go,users/admin.sql
+      --skip-test                            Disable automatic skipping of test files (default true)
 
 General Flags
       --api-key string          Use your Bearer API Key to send the report to Bearer.

internal/commands/process/orchestrator/worker/worker.go

@@ -34,11 +34,13 @@ type Worker struct {
 	classifer       *classification.Classifier
 	enabledScanners []string
 	sastScanner     *scanner.Scanner
+	skipTest        bool
 }
 
 func (worker *Worker) Setup(config config.Config) error {
 	worker.debug = config.Debug
 	worker.enabledScanners = config.Scan.Scanner
+	worker.skipTest = config.Scan.SkipTest
 
 	if slices.Contains(worker.enabledScanners, "sast") {
 		classifier, err := classification.NewClassifier(&classification.Config{Config: config})
@@ -86,6 +88,7 @@ func (worker *Worker) Scan(ctx context.Context, scanRequest work.ProcessRequest)
 		fileStats,
 		worker.enabledScanners,
 		worker.sastScanner,
+		worker.skipTest,
 	)
 
 	if ctx.Err() != nil {

internal/detectors/detectors.go

@@ -140,6 +140,7 @@ func Extract(
 	fileStats *stats.FileStats,
 	enabledScanners []string,
 	sastScanner *scanner.Scanner,
+	skipTest bool,
 ) error {
 	return ExtractWithDetectors(
 		ctx,
@@ -149,6 +150,7 @@ func Extract(
 		fileStats,
 		Registrations(enabledScanners),
 		sastScanner,
+		skipTest,
 	)
 }
 
@@ -160,13 +162,15 @@ func ExtractWithDetectors(
 	fileStats *stats.FileStats,
 	allDetectors []InitializedDetector,
 	sastScanner *scanner.Scanner,
+	skipTest bool,
 ) error {
 
 	activeDetectors := make(map[InitializedDetector]activeDetector)
 
 	if err := file.IterateFilesList(
 		rootDir,
 		[]string{filename},
+		skipTest,
 		func(dir *file.Path) (bool, error) {
 			for _, detector := range allDetectors {
 				active, isActive := activeDetectors[detector]

internal/detectors/internal/testhelper/testhelper.go

@@ -53,7 +53,7 @@ func Extract(
 	}
 
 	for _, filename := range files {
-		err = detectors.ExtractWithDetectors(context.Background(), path, filename, &report, nil, registrations, nil)
+		err = detectors.ExtractWithDetectors(context.Background(), path, filename, &report, nil, registrations, nil, true)
 		if !assert.Nil(t, err) {
 			t.Errorf("report has errored %s", err)
 		}

internal/flag/scan_flags.go

@@ -34,6 +34,12 @@ var (
 		Value:      []string{},
 		Usage:      "Specify the comma separated files and directories to skip. Supports * syntax, e.g. --skip-path users/*.go,users/admin.sql",
 	})
+	SkipTestFlag = ScanFlagGroup.add(flagtypes.Flag{
+		Name:       "skip-test",
+		ConfigName: "scan.skip-test",
+		Value:      true,
+		Usage:      "Disable automatic skipping of test files",
+	})
 	DisableDomainResolutionFlag = ScanFlagGroup.add(flagtypes.Flag{
 		Name:       "disable-domain-resolution",
 		ConfigName: "scan.disable-domain-resolution",
@@ -162,6 +168,7 @@ func (scanFlagGroup) SetOptions(options *flagtypes.Options, args []string) error
 
 	options.ScanOptions = flagtypes.ScanOptions{
 		SkipPath:                getStringSlice(SkipPathFlag),
+		SkipTest:                getBool(SkipTestFlag),
 		DisableDomainResolution: getBool(DisableDomainResolutionFlag),
 		DomainResolutionTimeout: getDuration(DomainResolutionTimeoutFlag),
 		InternalDomains:         getStringSlice(InternalDomainsFlag),

internal/flag/types/types.go

@@ -60,6 +60,7 @@ type Options struct {
 
 type ScanOptions struct {
 	Target                  string        `mapstructure:"target" json:"target" yaml:"target"`
+	SkipTest                bool          `mapstructure:"skip-test" json:"skip-test" yaml:"skip-test"`
 	SkipPath                []string      `mapstructure:"skip-path" json:"skip-path" yaml:"skip-path"`
 	DisableDomainResolution bool          `mapstructure:"disable-domain-resolution" json:"disable-domain-resolution" yaml:"disable-domain-resolution"`
 	DomainResolutionTimeout time.Duration `mapstructure:"domain-resolution-timeout" json:"domain-resolution-timeout" yaml:"domain-resolution-timeout"`

internal/util/file/file.go

@@ -27,13 +27,16 @@ type Line struct {
 	Strip      bool
 }
 
-var ignoredFilenames = []*regexp.Regexp{
-	regexp.MustCompile(`(^|/)\.git/`),
+var ignoreTestFiles = []*regexp.Regexp{
 	regexp.MustCompile(`(^|/)(?i:_*tests?_*)/`),
 	regexp.MustCompile(`(^|/)specs?/`),
-	regexp.MustCompile(`(^|/)testing/`),
 	regexp.MustCompile(`(^|/|[_-])(spec|test)s?\.`),
 	regexp.MustCompile(`(?i:unit[-_]?tests?)`),
+}
+
+var ignoredFilenames = []*regexp.Regexp{
+	regexp.MustCompile(`(^|/)\.git/`),
+	regexp.MustCompile(`(^|/)testing/`),
 	regexp.MustCompile(`(^|/)_*mocks?_*`),
 	regexp.MustCompile(`(^|/)fixtures/`),
 	regexp.MustCompile(`\.log$`),
@@ -110,7 +113,13 @@ func (path *Path) Exists() bool {
 	return true
 }
 
-func IterateFilesList(rootDir string, files []string, allowDir AllowDirFunction, visitFile VisitFileFunction) error {
+func IterateFilesList(
+	rootDir string,
+	files []string,
+	skipTest bool,
+	allowDir AllowDirFunction,
+	visitFile VisitFileFunction,
+) error {
 	gitIgnore := getGitIgnore(rootDir)
 
 	rootDir, err := filepath.Abs(rootDir)
@@ -145,6 +154,12 @@ func IterateFilesList(rootDir string, files []string, allowDir AllowDirFunction,
 		}
 
 		if regex.AnyMatch(ignoredFilenames, relativePath) {
+			log.Debug().Msgf("%s: skipping due to filename: other", path)
+			continue
+		}
+
+		if skipTest && regex.AnyMatch(ignoreTestFiles, relativePath) {
+			log.Debug().Msgf("%s: skipping due to filename: test", path)
 			continue
 		}