From 2ca3ac06bf4492645d6d7c2f6a034e5b4973cd29 Mon Sep 17 00:00:00 2001 From: Nikita Pivkin Date: Wed, 22 Nov 2023 00:02:57 +0300 Subject: [PATCH] feat(terraform): add an option to skip cached modules (#55) --- pkg/scanners/terraform/options.go | 8 ++++ pkg/scanners/terraform/parser/evaluator.go | 26 ++++++----- pkg/scanners/terraform/parser/load_module.go | 3 +- pkg/scanners/terraform/parser/option.go | 9 ++++ pkg/scanners/terraform/parser/parser.go | 44 +++++++++++-------- .../parser/parser_integration_test.go | 4 +- .../terraform/parser/resolvers/cache.go | 2 +- .../terraform/parser/resolvers/options.go | 2 +- .../terraform/scanner_integration_test.go | 2 + 9 files changed, 64 insertions(+), 36 deletions(-) diff --git a/pkg/scanners/terraform/options.go b/pkg/scanners/terraform/options.go index cef6bb7d..ddc4089e 100644 --- a/pkg/scanners/terraform/options.go +++ b/pkg/scanners/terraform/options.go @@ -194,6 +194,14 @@ func ScannerWithDownloadsAllowed(allowed bool) options.ScannerOption { } } +func ScannerWithSkipCachedModules(b bool) options.ScannerOption { + return func(s options.ConfigurableScanner) { + if tf, ok := s.(ConfigurableTerraformScanner); ok { + tf.AddParserOptions(parser.OptionWithDownloads(b)) + } + } +} + func ScannerWithConfigsFileSystem(fsys fs.FS) options.ScannerOption { return func(s options.ConfigurableScanner) { if tf, ok := s.(ConfigurableTerraformScanner); ok { diff --git a/pkg/scanners/terraform/parser/evaluator.go b/pkg/scanners/terraform/parser/evaluator.go index 251291c7..41a65690 100644 --- a/pkg/scanners/terraform/parser/evaluator.go +++ b/pkg/scanners/terraform/parser/evaluator.go @@ -25,18 +25,19 @@ const ( ) type evaluator struct { - filesystem fs.FS - ctx *tfcontext.Context - blocks terraform.Blocks - inputVars map[string]cty.Value - moduleMetadata *modulesMetadata - projectRootPath string // root of the current scan - modulePath string - moduleName string - ignores terraform.Ignores - parentParser *Parser - debug debug.Logger - allowDownloads bool + filesystem fs.FS + ctx *tfcontext.Context + blocks terraform.Blocks + inputVars map[string]cty.Value + moduleMetadata *modulesMetadata + projectRootPath string // root of the current scan + modulePath string + moduleName string + ignores terraform.Ignores + parentParser *Parser + debug debug.Logger + allowDownloads bool + skipCachedModules bool } func newEvaluator( @@ -53,6 +54,7 @@ func newEvaluator( ignores []terraform.Ignore, logger debug.Logger, allowDownloads bool, + skipCachedModules bool, ) *evaluator { // create a context to store variables and make functions available diff --git a/pkg/scanners/terraform/parser/load_module.go b/pkg/scanners/terraform/parser/load_module.go index d53e6541..afd37ab0 100644 --- a/pkg/scanners/terraform/parser/load_module.go +++ b/pkg/scanners/terraform/parser/load_module.go @@ -159,8 +159,9 @@ func (e *evaluator) loadExternalModule(ctx context.Context, b *terraform.Block, ModulePath: e.modulePath, DebugLogger: e.debug.Extend("resolver"), AllowDownloads: e.allowDownloads, - AllowCache: e.allowDownloads, + SkipCache: e.skipCachedModules, } + filesystem, prefix, path, err := resolveModule(ctx, e.filesystem, opt) if err != nil { return nil, err diff --git a/pkg/scanners/terraform/parser/option.go b/pkg/scanners/terraform/parser/option.go index 7d76589d..a37e20da 100644 --- a/pkg/scanners/terraform/parser/option.go +++ b/pkg/scanners/terraform/parser/option.go @@ -12,6 +12,7 @@ type ConfigurableTerraformParser interface { SetStopOnHCLError(bool) SetWorkspaceName(string) SetAllowDownloads(bool) + SetSkipCachedModules(bool) SetConfigsFS(fsys fs.FS) } @@ -49,6 +50,14 @@ func OptionWithDownloads(allowed bool) options.ParserOption { } } +func OptionWithSkipCachedModules(b bool) options.ParserOption { + return func(p options.ConfigurableParser) { + if tf, ok := p.(ConfigurableTerraformParser); ok { + tf.SetSkipCachedModules(b) + } + } +} + func OptionWithConfigsFS(fsys fs.FS) options.ParserOption { return func(s options.ConfigurableParser) { if p, ok := s.(ConfigurableTerraformParser); ok { diff --git a/pkg/scanners/terraform/parser/parser.go b/pkg/scanners/terraform/parser/parser.go index 4ae379b6..a7bf358b 100644 --- a/pkg/scanners/terraform/parser/parser.go +++ b/pkg/scanners/terraform/parser/parser.go @@ -43,25 +43,26 @@ var _ ConfigurableTerraformParser = (*Parser)(nil) // Parser is a tool for parsing terraform templates at a given file system location type Parser struct { - projectRoot string - moduleName string - modulePath string - moduleSource string - moduleFS fs.FS - moduleBlock *terraform.Block - files []sourceFile - tfvarsPaths []string - stopOnHCLError bool - workspaceName string - underlying *hclparse.Parser - children []*Parser - metrics Metrics - options []options.ParserOption - debug debug.Logger - allowDownloads bool - fsMap map[string]fs.FS - skipRequired bool - configsFS fs.FS + projectRoot string + moduleName string + modulePath string + moduleSource string + moduleFS fs.FS + moduleBlock *terraform.Block + files []sourceFile + tfvarsPaths []string + stopOnHCLError bool + workspaceName string + underlying *hclparse.Parser + children []*Parser + metrics Metrics + options []options.ParserOption + debug debug.Logger + allowDownloads bool + skipCachedModules bool + fsMap map[string]fs.FS + skipRequired bool + configsFS fs.FS } func (p *Parser) SetDebugWriter(writer io.Writer) { @@ -84,6 +85,10 @@ func (p *Parser) SetAllowDownloads(b bool) { p.allowDownloads = b } +func (p *Parser) SetSkipCachedModules(b bool) { + p.skipCachedModules = b +} + func (p *Parser) SetSkipRequiredCheck(b bool) { p.skipRequired = b } @@ -303,6 +308,7 @@ func (p *Parser) EvaluateAll(ctx context.Context) (terraform.Modules, cty.Value, ignores, p.debug.Extend("evaluator"), p.allowDownloads, + p.skipCachedModules, ) modules, fsMap, parseDuration := evaluator.EvaluateAll(ctx) p.metrics.Counts.Modules = len(modules) diff --git a/pkg/scanners/terraform/parser/parser_integration_test.go b/pkg/scanners/terraform/parser/parser_integration_test.go index d7f4f39a..ba7dd82c 100644 --- a/pkg/scanners/terraform/parser/parser_integration_test.go +++ b/pkg/scanners/terraform/parser/parser_integration_test.go @@ -20,7 +20,7 @@ module "registry" { `, }) - parser := New(fs, "", OptionStopOnHCLError(true)) + parser := New(fs, "", OptionStopOnHCLError(true), OptionWithSkipCachedModules(true)) if err := parser.ParseFS(context.TODO(), "code"); err != nil { t.Fatal(err) } @@ -41,7 +41,7 @@ module "registry" { `, }) - parser := New(fs, "", OptionStopOnHCLError(true)) + parser := New(fs, "", OptionStopOnHCLError(true), OptionWithSkipCachedModules(true)) if err := parser.ParseFS(context.TODO(), "code"); err != nil { t.Fatal(err) } diff --git a/pkg/scanners/terraform/parser/resolvers/cache.go b/pkg/scanners/terraform/parser/resolvers/cache.go index 482a595e..1314d538 100644 --- a/pkg/scanners/terraform/parser/resolvers/cache.go +++ b/pkg/scanners/terraform/parser/resolvers/cache.go @@ -35,7 +35,7 @@ func locateCacheDir() (string, error) { } func (r *cacheResolver) Resolve(_ context.Context, _ fs.FS, opt Options) (filesystem fs.FS, prefix string, downloadPath string, applies bool, err error) { - if !opt.AllowCache { + if opt.SkipCache { opt.Debug("Cache is disabled.") return nil, "", "", false, nil } diff --git a/pkg/scanners/terraform/parser/resolvers/options.go b/pkg/scanners/terraform/parser/resolvers/options.go index ef7847d1..61f720e8 100644 --- a/pkg/scanners/terraform/parser/resolvers/options.go +++ b/pkg/scanners/terraform/parser/resolvers/options.go @@ -10,7 +10,7 @@ type Options struct { Source, OriginalSource, Version, OriginalVersion, WorkingDir, Name, ModulePath string DebugLogger debug.Logger AllowDownloads bool - AllowCache bool + SkipCache bool RelativePath string } diff --git a/pkg/scanners/terraform/scanner_integration_test.go b/pkg/scanners/terraform/scanner_integration_test.go index f7bdcc2b..94dffbb4 100644 --- a/pkg/scanners/terraform/scanner_integration_test.go +++ b/pkg/scanners/terraform/scanner_integration_test.go @@ -54,6 +54,7 @@ deny[res] { options.ScannerWithEmbeddedLibraries(false), options.ScannerWithRegoOnly(true), ScannerWithAllDirectories(true), + ScannerWithSkipCachedModules(true), ) results, err := scanner.ScanFS(context.TODO(), fs, ".") @@ -117,6 +118,7 @@ deny[res] { options.ScannerWithEmbeddedLibraries(false), options.ScannerWithRegoOnly(true), ScannerWithAllDirectories(true), + ScannerWithSkipCachedModules(true), ) results, err := scanner.ScanFS(context.TODO(), fs, ".")