diff --git a/go.mod b/go.mod index ea3e200..83565f8 100644 --- a/go.mod +++ b/go.mod @@ -52,7 +52,10 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.17.4 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/elastic/elastic-transport-go/v8 v8.3.0 // indirect + github.com/elastic/go-elasticsearch/v8 v8.11.0 // indirect github.com/emicklei/go-restful/v3 v3.10.1 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect diff --git a/go.sum b/go.sum index ea42a6e..b1f5c52 100644 --- a/go.sum +++ b/go.sum @@ -96,6 +96,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.17.4/go.mod h1:bXcN3koeVYiJcdDU89n3k github.com/aws/smithy-go v1.13.4 h1:/RN2z1txIJWeXeOkzX+Hk/4Uuvv7dWtCjbmVJcrskyk= github.com/aws/smithy-go v1.13.4/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -111,6 +113,10 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/elastic/elastic-transport-go/v8 v8.3.0 h1:DJGxovyQLXGr62e9nDMPSxRyWION0Bh6d9eCFBriiHo= +github.com/elastic/elastic-transport-go/v8 v8.3.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI= +github.com/elastic/go-elasticsearch/v8 v8.11.0 h1:gUazf443rdYAEAD7JHX5lSXRgTkG4N4IcsV8dcWQPxM= +github.com/elastic/go-elasticsearch/v8 v8.11.0/go.mod h1:GU1BJHO7WeamP7UhuElYwzzHtvf9SDmeVpSSy9+o6Qg= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= diff --git a/pkg/threatest/matchers/elasticsearch/elasticsearch.go b/pkg/threatest/matchers/elasticsearch/elasticsearch.go index 76897e1..cc3e10d 100644 --- a/pkg/threatest/matchers/elasticsearch/elasticsearch.go +++ b/pkg/threatest/matchers/elasticsearch/elasticsearch.go @@ -1,66 +1,136 @@ package elasticsearch import ( - "bytes" - "context" - "encoding/json" - "errors" "fmt" - "net/http" - "strconv" "strings" - "time" - - es "github.com/elastic/go-elasticsearch/v8" - "github.com/aws/smithy-go/ptr" + log "github.com/sirupsen/logrus" + "encoding/json" + "errors" ) -func (m *ElasticsearchAlertGeneratedAssertion) HasExpectedAlert(detonationUuid string) (bool, error) { +func FilterByUuidPresence(alerts []ElasticsearchQueryHit, uuid string) []ElasticsearchQueryHit { + var filteredAlerts []ElasticsearchQueryHit + var containsUuid bool + for _,alert := range alerts { + containsUuid = false + for _,v := range alert.Source { + if strings.Contains(v.(string), uuid) { + containsUuid = true + break + } + } + if containsUuid { + filteredAlerts = append(filteredAlerts, alert) + } + } + return filteredAlerts +} + +func StripHTTPStatusCode(response string) (string, error) { + index := strings.Index(response, "{") + if index != -1 { + return response[index:], nil + } else { + return "", errors.New("No '{' found in Elasticsearch query response") + } +} + +func RetrieveAlerts(m *ElasticsearchAlertGeneratedAssertion, uuidField, ruleName string) ([]ElasticsearchQueryHit, error) { + // The alias for the Elasticsearch index where alerts are stored + const ALERT_INDEX string = ".siem-signals-default" // Construct the query necessary to find the alert - query := `{ "query": { "match_all": {} } }` - m.AlertAPI.Search( - m.AlertAPI.Search.WithIndex(".siem-signals-*"), + query := ` + { + "_source": [ "%s" ], + "query": { + "bool": { + "filter": [ + { "range": { "@timestamp": { "gte": "now-3d" }}}, + { "term": { "kibana.alert.rule.name": "%s" }}, + { "term": { "kibana.alert.workflow_status": "open" }} + ] + } + } + }` + // Template in the field we expect to find the UUID in, and the rule we hope was triggered. + query = fmt.Sprintf(query, uuidField, ruleName) + // Query the Elasticsearch API + res, err := m.AlertAPI.Search( + m.AlertAPI.Search.WithIndex(ALERT_INDEX), m.AlertAPI.Search.WithBody(strings.NewReader(query)), ) - // Alternative way of doing the same thing? - res, err := m.AlertAPI.Search(). - Index(".siem-signals-*"). - Request(&search.Request{ - Query: &types.Query{ - Match: map[string]types.MatchQuery{ - "name": {Query: detonationUuid}, - }, - }, - }).Do(context.Background()) - - search.Request{ - Query: &types.Query{ - Term: map[string]types.TermQuery{ - "rule": {Value: m.AlertFilter.RuleName}, - "uuid": - }, - }, + if err != nil { + log.Fatal("Error while running Elasticsearch query") + return nil, err + } + // Parse the response + strippedResponse, err := StripHTTPStatusCode(res.String()) + if err != nil { + log.Fatal("Error while stripping prepended HTTP status code") + return nil, err } + var data ElasticsearchQueryResponse + if err := json.Unmarshal([]byte(strippedResponse), &data); err != nil { + log.Fatal("Error unmarshalling JSON string into ElasticsearchQueryResponse struct") + return nil, err + } + + return data.Hits.Hits, nil +} +func (m *ElasticsearchAlertGeneratedAssertion) HasExpectedAlert(detonationUuid string) (bool, error) { + log.Infof("Searching for open alerts for rule: %s with UUID: %s in field: %s", m.AlertFilter.RuleName, detonationUuid, m.AlertFilter.UuidField) + alerts, err := RetrieveAlerts(m, m.AlertFilter.UuidField, m.AlertFilter.RuleName) + if err != nil { + log.Fatal("Failed to retrieve alerts") + return false, err + } + // Filter the alerts, is the one we're looking for here? + alerts = FilterByUuidPresence(alerts, detonationUuid) + if len(alerts) == 1 { + log.Info("One open alert found") + m.AlertId = alerts[0].ID + m.Index = alerts[0].Index + return true, nil + } + if len(alerts) > 1 { + // TODO: It may well be desirable for a suspicious event to trigger multiple alerts + // In future ElasticsearchAlertGeneratedAssertion.AlertFilter should be a list, capable + // of matching and closing multiple alerts associated with a single event. + log.Errorf("More than one alert found") + return false, nil + } + log.Warnf("No alerts found") return false, nil } func (m *ElasticsearchAlertGeneratedAssertion) String() string { - return fmt.Sprintf("Elasticsearch alert '%s'", m.AlertFilter.Rule) + return fmt.Sprintf("Elasticsearch alert '%s'", m.AlertFilter.RuleName) } func (m *ElasticsearchAlertGeneratedAssertion) Cleanup(detonationUuid string) error { - signals, err := m.SignalsAPI.SearchSignals(QueryAllOpenSignals) - if err != nil { - return errors.New("unable to search for Datadog security monitoring signals: " + err.Error()) + log.Infof("Closing alert for detonation: %s, for rule: %s with AlertId: %s in Index: %s", detonationUuid, m.AlertFilter.RuleName, m.AlertId, m.Index) + // If HasExpectedAlert() executed properly then m.AlertId ought to be set with the ID we need + if m.AlertId == "" { + return errors.New("AlertId not set, cannot close alert") } - - for i := range signals { - if m.signalMatchesExecution(signals[i], detonationUuid) { - if err := m.SignalsAPI.CloseSignal(*signals[i].Id); err != nil { - return errors.New("unable to archive signal " + *signals[i].Id + ": " + err.Error()) - } + // We can query via the .siem-signals-default alias, however this isn't the actual index the document is in. + // To write to the index we need the actual index ID. Fortunately that data is in the document and we should + // have written that also when we ran HasExpectedAlert(). + if m.Index == "" { + return errors.New("Index not set, cannot close alert") + } + update_request_body := ` + { + "doc": { + "kibana.alert.workflow_status": "closed" } + }` + resp, err := m.AlertAPI.Update(m.Index, m.AlertId, strings.NewReader(update_request_body)) + log.Info("Logging the update API response:\n",resp, "\n") + if err != nil { + log.Errorf("Error while trying to update document: %s", m.AlertId) + return err } return nil diff --git a/pkg/threatest/matchers/elasticsearch/types.go b/pkg/threatest/matchers/elasticsearch/types.go index 7c2524f..debb50d 100644 --- a/pkg/threatest/matchers/elasticsearch/types.go +++ b/pkg/threatest/matchers/elasticsearch/types.go @@ -1,24 +1,47 @@ package elasticsearch import ( - "context" "os" + "time" + "github.com/cenkalti/backoff/v4" + "strings" + "fmt" - elasticsearch "github.com/elastic/go-elasticsearch/v8" + log "github.com/sirupsen/logrus" + es "github.com/elastic/go-elasticsearch/v8" ) +type ElasticsearchQueryResponse struct { + Hits struct { + Total struct { + Value int `json:"value"` + } `json:"total"` + Hits []ElasticsearchQueryHit `json:"hits"` + } `json:"hits"` +} + +type ElasticsearchQueryHit struct { + Index string `json:"_index"` + ID string `json:"_id"` + Source map[string]interface{} `json:"_source"` +} + type ElasticsearchAlertFilter struct { - RuleName string `yaml:"rule-name"` + RuleName string `yaml:"rule-name"` + UuidField string `yaml:"uuid-field"` } type ElasticsearchAlertGeneratedAssertion struct { - AlertAPI ElasticsearchClient + AlertAPI es.Client AlertFilter *ElasticsearchAlertFilter + AlertId string + Index string } -func GetElasticSearchClient() (ElasticsearchClient, error) { +func ElasticsearchAlert(ruleName, uuidField string) *ElasticsearchAlertGeneratedAssertion { + retryBackoff := backoff.NewExponentialBackOff() // New Elasticsearch client - cfg := elasticsearch.Config{ + esClient, err := es.NewClient(es.Config{ Addresses: []string{os.Getenv("ELASTICSEARCH_URL")}, Username: os.Getenv("ELASTICSEARCH_USERNAME"), Password: os.Getenv("ELASTICSEARCH_PASSWORD"), @@ -33,27 +56,59 @@ func GetElasticSearchClient() (ElasticsearchClient, error) { }, // Retry up to 5 attempts MaxRetries: 5, - } - es, err := elasticsearch.NewTypedClient(cfg) + }) if err != nil { - return nil, fmt.Errorf("failed to create Elasticsearch client: %w", err) + log.Fatalf("failed to create Elasticsearch client: %w", err) } - info, err := es.Info().Do(ctx) + info, err := esClient.Info() if err != nil { - return nil, fmt.Errorf("failed to get Elasticsearch cluster info: %w", err) + log.Fatalf("failed to get Elasticsearch cluster info: %w", err) + } + log.Info("Elasticsearch cluster info:\n", info.String()) + return &ElasticsearchAlertGeneratedAssertion{ + AlertAPI: *esClient, + AlertFilter: &ElasticsearchAlertFilter{RuleName: ruleName, UuidField: uuidField}, } - log.Info("Elasticsearch cluster info", "cluster_name", info.ClusterName, "version", info.Version.Number) - return es, nil } -func ElasticsearchAlert(rule, tellTale string) *ElasticsearchAlertGeneratedAssertion { - es, err := GetElasticSearchClient() + +// Dumping Ground +func CreateIndex(m *ElasticsearchAlertGeneratedAssertion, index string) { + mapping := ` + { + "settings": { + "number_of_shards": 1 + }, + "mappings": { + "properties": { + "field1": { + "type": "text" + }, + "date": { + "type": "date" + } + } + } + }` + res, err := m.AlertAPI.Indices.Create( + index, + m.AlertAPI.Indices.Create.WithBody(strings.NewReader(mapping)), + ) + if err != nil { + log.Fatal(err) + } + log.Println(res) +} + +func WriteToIndex(m *ElasticsearchAlertGeneratedAssertion, index string) { + entry := ` + { + "field1": "helloo there abc1234", + "@timestamp": "%s" + }` + res, err := m.AlertAPI.Index(index, strings.NewReader(fmt.Sprintf(entry, time.Now().Format("2006/01/02 15:04:05")))) if err != nil { - log.Error(err, "failed to create Elasticsearch client") - return nil - } - return &ElasticsearchAlertGeneratedAssertion{ - AlertAPI: es, - AlertFilter: &ElasticsearchAlertFilter{RuleName: rule}, + log.Fatal(err) } + log.Println(res) }