From 16169913985074f8b9944dca2a5c338713ef3e93 Mon Sep 17 00:00:00 2001 From: "Thibault \"bui\" Koechlin" Date: Mon, 4 Nov 2024 10:02:55 +0100 Subject: [PATCH] Alert context appsec (#3288) --- .../modules/appsec/appsec_runner.go | 4 +- pkg/acquisition/modules/appsec/utils.go | 163 +++++------------- pkg/alertcontext/alertcontext.go | 154 ++++++++++++----- pkg/alertcontext/alertcontext_test.go | 162 +++++++++++++++++ pkg/types/appsec_event.go | 8 +- pkg/types/event.go | 9 + 6 files changed, 337 insertions(+), 163 deletions(-) diff --git a/pkg/acquisition/modules/appsec/appsec_runner.go b/pkg/acquisition/modules/appsec/appsec_runner.go index de34b62d704..90d23f63543 100644 --- a/pkg/acquisition/modules/appsec/appsec_runner.go +++ b/pkg/acquisition/modules/appsec/appsec_runner.go @@ -249,7 +249,7 @@ func (r *AppsecRunner) handleInBandInterrupt(request *appsec.ParsedRequest) { // Should the in band match trigger an overflow ? if r.AppsecRuntime.Response.SendAlert { - appsecOvlfw, err := AppsecEventGeneration(evt) + appsecOvlfw, err := AppsecEventGeneration(evt, request.HTTPRequest) if err != nil { r.logger.Errorf("unable to generate appsec event : %s", err) return @@ -293,7 +293,7 @@ func (r *AppsecRunner) handleOutBandInterrupt(request *appsec.ParsedRequest) { // Should the match trigger an overflow ? if r.AppsecRuntime.Response.SendAlert { - appsecOvlfw, err := AppsecEventGeneration(evt) + appsecOvlfw, err := AppsecEventGeneration(evt, request.HTTPRequest) if err != nil { r.logger.Errorf("unable to generate appsec event : %s", err) return diff --git a/pkg/acquisition/modules/appsec/utils.go b/pkg/acquisition/modules/appsec/utils.go index 4fb1a979d14..b4b66897516 100644 --- a/pkg/acquisition/modules/appsec/utils.go +++ b/pkg/acquisition/modules/appsec/utils.go @@ -1,10 +1,10 @@ package appsecacquisition import ( + "errors" "fmt" "net" - "slices" - "strconv" + "net/http" "time" "github.com/oschwald/geoip2-golang" @@ -22,29 +22,44 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" ) -var appsecMetaKeys = []string{ - "id", - "name", - "method", - "uri", - "matched_zones", - "msg", -} +func AppsecEventGenerationGeoIPEnrich(src *models.Source) error { -func appendMeta(meta models.Meta, key string, value string) models.Meta { - if value == "" { - return meta + if src == nil || src.Scope == nil || *src.Scope != types.Ip { + return errors.New("source is nil or not an IP") } - meta = append(meta, &models.MetaItems0{ - Key: key, - Value: value, - }) + //GeoIP enrich + asndata, err := exprhelpers.GeoIPASNEnrich(src.IP) + + if err != nil { + return err + } else if asndata != nil { + record := asndata.(*geoip2.ASN) + src.AsName = record.AutonomousSystemOrganization + src.AsNumber = fmt.Sprintf("%d", record.AutonomousSystemNumber) + } - return meta + cityData, err := exprhelpers.GeoIPEnrich(src.IP) + if err != nil { + return err + } else if cityData != nil { + record := cityData.(*geoip2.City) + src.Cn = record.Country.IsoCode + src.Latitude = float32(record.Location.Latitude) + src.Longitude = float32(record.Location.Longitude) + } + + rangeData, err := exprhelpers.GeoIPRangeEnrich(src.IP) + if err != nil { + return err + } else if rangeData != nil { + record := rangeData.(*net.IPNet) + src.Range = record.String() + } + return nil } -func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) { +func AppsecEventGeneration(inEvt types.Event, request *http.Request) (*types.Event, error) { // if the request didnd't trigger inband rules, we don't want to generate an event to LAPI/CAPI if !inEvt.Appsec.HasInBandMatches { return nil, nil @@ -60,34 +75,12 @@ func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) { Scope: ptr.Of(types.Ip), } - asndata, err := exprhelpers.GeoIPASNEnrich(sourceIP) - - if err != nil { - log.Errorf("Unable to enrich ip '%s' for ASN: %s", sourceIP, err) - } else if asndata != nil { - record := asndata.(*geoip2.ASN) - source.AsName = record.AutonomousSystemOrganization - source.AsNumber = fmt.Sprintf("%d", record.AutonomousSystemNumber) - } - - cityData, err := exprhelpers.GeoIPEnrich(sourceIP) - if err != nil { - log.Errorf("Unable to enrich ip '%s' for geo data: %s", sourceIP, err) - } else if cityData != nil { - record := cityData.(*geoip2.City) - source.Cn = record.Country.IsoCode - source.Latitude = float32(record.Location.Latitude) - source.Longitude = float32(record.Location.Longitude) - } - - rangeData, err := exprhelpers.GeoIPRangeEnrich(sourceIP) - if err != nil { - log.Errorf("Unable to enrich ip '%s' for range: %s", sourceIP, err) - } else if rangeData != nil { - record := rangeData.(*net.IPNet) - source.Range = record.String() + // Enrich source with GeoIP data + if err := AppsecEventGenerationGeoIPEnrich(&source); err != nil { + log.Errorf("unable to enrich source with GeoIP data : %s", err) } + // Build overflow evt.Overflow.Sources = make(map[string]models.Source) evt.Overflow.Sources[sourceIP] = source @@ -95,83 +88,11 @@ func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) { alert.Capacity = ptr.Of(int32(1)) alert.Events = make([]*models.Event, len(evt.Appsec.GetRuleIDs())) - now := ptr.Of(time.Now().UTC().Format(time.RFC3339)) - - tmpAppsecContext := make(map[string][]string) - - for _, matched_rule := range inEvt.Appsec.MatchedRules { - evtRule := models.Event{} - - evtRule.Timestamp = now - - evtRule.Meta = make(models.Meta, 0) - - for _, key := range appsecMetaKeys { - if tmpAppsecContext[key] == nil { - tmpAppsecContext[key] = make([]string, 0) - } - - switch value := matched_rule[key].(type) { - case string: - evtRule.Meta = appendMeta(evtRule.Meta, key, value) - - if value != "" && !slices.Contains(tmpAppsecContext[key], value) { - tmpAppsecContext[key] = append(tmpAppsecContext[key], value) - } - case int: - val := strconv.Itoa(value) - evtRule.Meta = appendMeta(evtRule.Meta, key, val) - - if val != "" && !slices.Contains(tmpAppsecContext[key], val) { - tmpAppsecContext[key] = append(tmpAppsecContext[key], val) - } - case []string: - for _, v := range value { - evtRule.Meta = appendMeta(evtRule.Meta, key, v) - - if v != "" && !slices.Contains(tmpAppsecContext[key], v) { - tmpAppsecContext[key] = append(tmpAppsecContext[key], v) - } - } - case []int: - for _, v := range value { - val := strconv.Itoa(v) - evtRule.Meta = appendMeta(evtRule.Meta, key, val) - - if val != "" && !slices.Contains(tmpAppsecContext[key], val) { - tmpAppsecContext[key] = append(tmpAppsecContext[key], val) - } - } - default: - val := fmt.Sprintf("%v", value) - evtRule.Meta = appendMeta(evtRule.Meta, key, val) - - if val != "" && !slices.Contains(tmpAppsecContext[key], val) { - tmpAppsecContext[key] = append(tmpAppsecContext[key], val) - } - } - } - - alert.Events = append(alert.Events, &evtRule) - } - - metas := make([]*models.MetaItems0, 0) - - for key, values := range tmpAppsecContext { - if len(values) == 0 { - continue - } - - valueStr, err := alertcontext.TruncateContext(values, alertcontext.MaxContextValueLen) - if err != nil { - log.Warning(err.Error()) - } - - meta := models.MetaItems0{ - Key: key, - Value: valueStr, + metas, errors := alertcontext.AppsecEventToContext(inEvt.Appsec, request) + if len(errors) > 0 { + for _, err := range errors { + log.Errorf("failed to generate appsec context: %s", err) } - metas = append(metas, &meta) } alert.Meta = metas diff --git a/pkg/alertcontext/alertcontext.go b/pkg/alertcontext/alertcontext.go index 16ebc6d0ac2..0c60dea4292 100644 --- a/pkg/alertcontext/alertcontext.go +++ b/pkg/alertcontext/alertcontext.go @@ -3,6 +3,7 @@ package alertcontext import ( "encoding/json" "fmt" + "net/http" "slices" "strconv" @@ -30,7 +31,10 @@ type Context struct { func ValidateContextExpr(key string, expressions []string) error { for _, expression := range expressions { - _, err := expr.Compile(expression, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) + _, err := expr.Compile(expression, exprhelpers.GetExprOptions(map[string]interface{}{ + "evt": &types.Event{}, + "match": &types.MatchedRule{}, + "req": &http.Request{}})...) if err != nil { return fmt.Errorf("compilation of '%s' failed: %w", expression, err) } @@ -72,7 +76,10 @@ func NewAlertContext(contextToSend map[string][]string, valueLength int) error { } for _, value := range values { - valueCompiled, err := expr.Compile(value, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) + valueCompiled, err := expr.Compile(value, exprhelpers.GetExprOptions(map[string]interface{}{ + "evt": &types.Event{}, + "match": &types.MatchedRule{}, + "req": &http.Request{}})...) if err != nil { return fmt.Errorf("compilation of '%s' context value failed: %w", value, err) } @@ -85,6 +92,32 @@ func NewAlertContext(contextToSend map[string][]string, valueLength int) error { return nil } +// Truncate the context map to fit in the context value length +func TruncateContextMap(contextMap map[string][]string, contextValueLen int) ([]*models.MetaItems0, []error) { + metas := make([]*models.MetaItems0, 0) + errors := make([]error, 0) + + for key, values := range contextMap { + if len(values) == 0 { + continue + } + + valueStr, err := TruncateContext(values, alertContext.ContextValueLen) + if err != nil { + errors = append(errors, fmt.Errorf("error truncating content for %s: %w", key, err)) + continue + } + + meta := models.MetaItems0{ + Key: key, + Value: valueStr, + } + metas = append(metas, &meta) + } + return metas, errors +} + +// Truncate an individual []string to fit in the context value length func TruncateContext(values []string, contextValueLen int) (string, error) { valueByte, err := json.Marshal(values) if err != nil { @@ -116,61 +149,104 @@ func TruncateContext(values []string, contextValueLen int) (string, error) { return ret, nil } -func EventToContext(events []types.Event) (models.Meta, []error) { +func EvalAlertContextRules(evt *types.Event, match *types.MatchedRule, request *http.Request, tmpContext map[string][]string) []error { + var errors []error - metas := make([]*models.MetaItems0, 0) - tmpContext := make(map[string][]string) + //if we're evaluating context for appsec event, match and request will be present. + //otherwise, only evt will be. + if evt == nil { + evt = types.NewEvent() + } + if match == nil { + match = types.NewMatchedRule() + } + if request == nil { + request = &http.Request{} + } - for _, evt := range events { - for key, values := range alertContext.ContextToSendCompiled { - if _, ok := tmpContext[key]; !ok { - tmpContext[key] = make([]string, 0) - } + for key, values := range alertContext.ContextToSendCompiled { - for _, value := range values { - var val string + if _, ok := tmpContext[key]; !ok { + tmpContext[key] = make([]string, 0) + } - output, err := expr.Run(value, map[string]interface{}{"evt": evt}) - if err != nil { - errors = append(errors, fmt.Errorf("failed to get value for %s: %w", key, err)) - continue - } + for _, value := range values { + var val string - switch out := output.(type) { - case string: - val = out - case int: - val = strconv.Itoa(out) - default: - errors = append(errors, fmt.Errorf("unexpected return type for %s: %T", key, output)) - continue + output, err := expr.Run(value, map[string]interface{}{"match": match, "evt": evt, "req": request}) + if err != nil { + errors = append(errors, fmt.Errorf("failed to get value for %s: %w", key, err)) + continue + } + switch out := output.(type) { + case string: + val = out + if val != "" && !slices.Contains(tmpContext[key], val) { + tmpContext[key] = append(tmpContext[key], val) } - + case []string: + for _, v := range out { + if v != "" && !slices.Contains(tmpContext[key], v) { + tmpContext[key] = append(tmpContext[key], v) + } + } + case int: + val = strconv.Itoa(out) + if val != "" && !slices.Contains(tmpContext[key], val) { + tmpContext[key] = append(tmpContext[key], val) + } + case []int: + for _, v := range out { + val = strconv.Itoa(v) + if val != "" && !slices.Contains(tmpContext[key], val) { + tmpContext[key] = append(tmpContext[key], val) + } + } + default: + val := fmt.Sprintf("%v", output) if val != "" && !slices.Contains(tmpContext[key], val) { tmpContext[key] = append(tmpContext[key], val) } } } } + return errors +} - for key, values := range tmpContext { - if len(values) == 0 { - continue - } +// Iterate over the individual appsec matched rules to create the needed alert context. +func AppsecEventToContext(event types.AppsecEvent, request *http.Request) (models.Meta, []error) { + var errors []error - valueStr, err := TruncateContext(values, alertContext.ContextValueLen) - if err != nil { - log.Warning(err.Error()) - } + tmpContext := make(map[string][]string) - meta := models.MetaItems0{ - Key: key, - Value: valueStr, - } - metas = append(metas, &meta) + for _, matched_rule := range event.MatchedRules { + tmpErrors := EvalAlertContextRules(nil, &matched_rule, request, tmpContext) + errors = append(errors, tmpErrors...) } + metas, truncErrors := TruncateContextMap(tmpContext, alertContext.ContextValueLen) + errors = append(errors, truncErrors...) + + ret := models.Meta(metas) + + return ret, errors +} + +// Iterate over the individual events to create the needed alert context. +func EventToContext(events []types.Event) (models.Meta, []error) { + var errors []error + + tmpContext := make(map[string][]string) + + for _, evt := range events { + tmpErrors := EvalAlertContextRules(&evt, nil, nil, tmpContext) + errors = append(errors, tmpErrors...) + } + + metas, truncErrors := TruncateContextMap(tmpContext, alertContext.ContextValueLen) + errors = append(errors, truncErrors...) + ret := models.Meta(metas) return ret, errors diff --git a/pkg/alertcontext/alertcontext_test.go b/pkg/alertcontext/alertcontext_test.go index c111d1bbcfb..dc752ba8b09 100644 --- a/pkg/alertcontext/alertcontext_test.go +++ b/pkg/alertcontext/alertcontext_test.go @@ -2,6 +2,7 @@ package alertcontext import ( "fmt" + "net/http" "testing" "github.com/stretchr/testify/assert" @@ -9,6 +10,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/crowdsecurity/go-cs-lib/ptr" ) func TestNewAlertContext(t *testing.T) { @@ -200,3 +202,163 @@ func TestEventToContext(t *testing.T) { assert.ElementsMatch(t, test.expectedResult, metas) } } + +func TestValidateContextExpr(t *testing.T) { + tests := []struct { + name string + key string + exprs []string + expectedErr *string + }{ + { + name: "basic config", + key: "source_ip", + exprs: []string{ + "evt.Parsed.source_ip", + }, + expectedErr: nil, + }, + { + name: "basic config with non existent field", + key: "source_ip", + exprs: []string{ + "evt.invalid.source_ip", + }, + expectedErr: ptr.Of("compilation of 'evt.invalid.source_ip' failed: type types.Event has no field invalid"), + }, + } + for _, test := range tests { + fmt.Printf("Running test '%s'\n", test.name) + err := ValidateContextExpr(test.key, test.exprs) + if test.expectedErr == nil { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, *test.expectedErr) + } + } +} + +func TestAppsecEventToContext(t *testing.T) { + + tests := []struct { + name string + contextToSend map[string][]string + match types.AppsecEvent + req *http.Request + expectedResult models.Meta + expectedErrLen int + }{ + { + name: "basic test on match", + contextToSend: map[string][]string{ + "id": {"match.id"}, + }, + match: types.AppsecEvent{ + MatchedRules: types.MatchedRules{ + { + "id": "test", + }, + }, + }, + req: &http.Request{}, + expectedResult: []*models.MetaItems0{ + { + Key: "id", + Value: "[\"test\"]", + }, + }, + expectedErrLen: 0, + }, + { + name: "basic test on req", + contextToSend: map[string][]string{ + "ua": {"req.UserAgent()"}, + }, + match: types.AppsecEvent{ + MatchedRules: types.MatchedRules{ + { + "id": "test", + }, + }, + }, + req: &http.Request{ + Header: map[string][]string{ + "User-Agent": {"test"}, + }, + }, + expectedResult: []*models.MetaItems0{ + { + Key: "ua", + Value: "[\"test\"]", + }, + }, + expectedErrLen: 0, + }, + { + name: "test on req -> []string", + contextToSend: map[string][]string{ + "foobarxx": {"req.Header.Values('Foobar')"}, + }, + match: types.AppsecEvent{ + MatchedRules: types.MatchedRules{ + { + "id": "test", + }, + }, + }, + req: &http.Request{ + Header: map[string][]string{ + "User-Agent": {"test"}, + "Foobar": {"test1", "test2"}, + }, + }, + expectedResult: []*models.MetaItems0{ + { + Key: "foobarxx", + Value: "[\"test1\",\"test2\"]", + }, + }, + expectedErrLen: 0, + }, + { + name: "test on type int", + contextToSend: map[string][]string{ + "foobarxx": {"len(req.Header.Values('Foobar'))"}, + }, + match: types.AppsecEvent{ + MatchedRules: types.MatchedRules{ + { + "id": "test", + }, + }, + }, + req: &http.Request{ + Header: map[string][]string{ + "User-Agent": {"test"}, + "Foobar": {"test1", "test2"}, + }, + }, + expectedResult: []*models.MetaItems0{ + { + Key: "foobarxx", + Value: "[\"2\"]", + }, + }, + expectedErrLen: 0, + }, + } + + for _, test := range tests { + //reset cache + alertContext = Context{} + //compile + if err := NewAlertContext(test.contextToSend, 100); err != nil { + t.Fatalf("failed to compile %s: %s", test.name, err) + } + //run + + metas, errors := AppsecEventToContext(test.match, test.req) + assert.Len(t, errors, test.expectedErrLen) + assert.ElementsMatch(t, test.expectedResult, metas) + } +} diff --git a/pkg/types/appsec_event.go b/pkg/types/appsec_event.go index dc81c63b344..11d70ad368d 100644 --- a/pkg/types/appsec_event.go +++ b/pkg/types/appsec_event.go @@ -18,7 +18,9 @@ len(evt.Waf.ByTagRx("*CVE*").ByConfidence("high").ByAction("block")) > 1 */ -type MatchedRules []map[string]interface{} +type MatchedRules []MatchedRule + +type MatchedRule map[string]interface{} type AppsecEvent struct { HasInBandMatches, HasOutBandMatches bool @@ -45,6 +47,10 @@ const ( Kind Field = "kind" ) +func NewMatchedRule() *MatchedRule { + return &MatchedRule{} +} + func (w AppsecEvent) GetVar(varName string) string { if w.Vars == nil { return "" diff --git a/pkg/types/event.go b/pkg/types/event.go index e016d0294c4..6d275aedf95 100644 --- a/pkg/types/event.go +++ b/pkg/types/event.go @@ -47,6 +47,15 @@ type Event struct { Meta map[string]string `yaml:"Meta,omitempty" json:"Meta,omitempty"` } +func NewEvent() *Event { + return &Event{Type: LOG, + Parsed: make(map[string]string), + Enriched: make(map[string]string), + Meta: make(map[string]string), + Unmarshaled: make(map[string]interface{}), + } +} + func (e *Event) SetMeta(key string, value string) bool { if e.Meta == nil { e.Meta = make(map[string]string)