Alert retrieval and alert closing working for Elasticsearch matcher

go.mod

@@ -52,7 +52,10 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.17.4 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/elastic/elastic-transport-go/v8 v8.3.0 // indirect
+	github.com/elastic/go-elasticsearch/v8 v8.11.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.10.1 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect

go.sum

@@ -96,6 +96,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.17.4/go.mod h1:bXcN3koeVYiJcdDU89n3k
 github.com/aws/smithy-go v1.13.4 h1:/RN2z1txIJWeXeOkzX+Hk/4Uuvv7dWtCjbmVJcrskyk=
 github.com/aws/smithy-go v1.13.4/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
 github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
+github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
+github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@@ -111,6 +113,10 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/elastic/elastic-transport-go/v8 v8.3.0 h1:DJGxovyQLXGr62e9nDMPSxRyWION0Bh6d9eCFBriiHo=
+github.com/elastic/elastic-transport-go/v8 v8.3.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI=
+github.com/elastic/go-elasticsearch/v8 v8.11.0 h1:gUazf443rdYAEAD7JHX5lSXRgTkG4N4IcsV8dcWQPxM=
+github.com/elastic/go-elasticsearch/v8 v8.11.0/go.mod h1:GU1BJHO7WeamP7UhuElYwzzHtvf9SDmeVpSSy9+o6Qg=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
 github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ=
 github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=

pkg/threatest/matchers/elasticsearch/elasticsearch.go

@@ -1,66 +1,136 @@
 package elasticsearch
 
 import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"errors"
 	"fmt"
-	"net/http"
-	"strconv"
 	"strings"
-	"time"
-
-	es "github.com/elastic/go-elasticsearch/v8"
-	"github.com/aws/smithy-go/ptr"
+	log "github.com/sirupsen/logrus"
+	"encoding/json"
+	"errors"
 )
 
-func (m *ElasticsearchAlertGeneratedAssertion) HasExpectedAlert(detonationUuid string) (bool, error) {
+func FilterByUuidPresence(alerts []ElasticsearchQueryHit, uuid string) []ElasticsearchQueryHit {
+	var filteredAlerts []ElasticsearchQueryHit
+	var containsUuid bool
+	for _,alert := range alerts {
+		containsUuid = false
+		for _,v := range alert.Source {
+			if strings.Contains(v.(string), uuid) {
+				containsUuid = true
+				break
+			}
+		}
+		if containsUuid {
+			filteredAlerts = append(filteredAlerts, alert)
+		}
+	}
+	return filteredAlerts
+}
+
+func StripHTTPStatusCode(response string) (string, error) {
+    index := strings.Index(response, "{")
+    if index != -1 {
+        return response[index:], nil
+    } else {
+        return "", errors.New("No '{' found in Elasticsearch query response")
+    }
+}
+
+func RetrieveAlerts(m *ElasticsearchAlertGeneratedAssertion, uuidField, ruleName string) ([]ElasticsearchQueryHit, error) {
+	// The alias for the Elasticsearch index where alerts are stored
+	const ALERT_INDEX string = ".siem-signals-default"
 	// Construct the query necessary to find the alert
-	query := `{ "query": { "match_all": {} } }`
-	m.AlertAPI.Search(
-		m.AlertAPI.Search.WithIndex(".siem-signals-*"),
+	query := `
+	{
+		"_source": [ "%s" ],
+		"query": {
+			"bool": { 
+				"filter": [ 
+					{ "range": { "@timestamp": { "gte": "now-3d" }}},
+					{ "term":  { "kibana.alert.rule.name": "%s" }},
+					{ "term":  { "kibana.alert.workflow_status": "open" }}
+				]
+			}
+		}
+	}`
+	// Template in the field we expect to find the UUID in, and the rule we hope was triggered.
+	query = fmt.Sprintf(query, uuidField, ruleName)
+	// Query the Elasticsearch API
+	res, err := m.AlertAPI.Search(
+		m.AlertAPI.Search.WithIndex(ALERT_INDEX),
 		m.AlertAPI.Search.WithBody(strings.NewReader(query)),
 	)
-	// Alternative way of doing the same thing?
-	res, err := m.AlertAPI.Search().
-		Index(".siem-signals-*"). 
-		Request(&search.Request{ 
-			Query: &types.Query{
-				Match: map[string]types.MatchQuery{
-					"name": {Query: detonationUuid}, 
-				},
-			},
-		}).Do(context.Background())
-
-	search.Request{
-		Query: &types.Query{
-			Term: map[string]types.TermQuery{
-				"rule": {Value: m.AlertFilter.RuleName},
-				"uuid": 
-			},
-		},
+	if err != nil {
+		log.Fatal("Error while running Elasticsearch query")
+		return nil, err
+	}
+	// Parse the response
+	strippedResponse, err := StripHTTPStatusCode(res.String())
+	if err != nil {
+		log.Fatal("Error while stripping prepended HTTP status code")
+		return nil, err
 	}
+	var data ElasticsearchQueryResponse
+	if err := json.Unmarshal([]byte(strippedResponse), &data); err != nil {
+		log.Fatal("Error unmarshalling JSON string into ElasticsearchQueryResponse struct")
+		return nil, err
+	}
+
+	return data.Hits.Hits, nil
+}
 
+func (m *ElasticsearchAlertGeneratedAssertion) HasExpectedAlert(detonationUuid string) (bool, error) {
+	log.Infof("Searching for open alerts for rule: %s with UUID: %s in field: %s", m.AlertFilter.RuleName, detonationUuid, m.AlertFilter.UuidField)
+	alerts, err := RetrieveAlerts(m, m.AlertFilter.UuidField, m.AlertFilter.RuleName)
+	if err != nil {
+		log.Fatal("Failed to retrieve alerts")
+		return false, err
+	}
+	// Filter the alerts, is the one we're looking for here?
+	alerts = FilterByUuidPresence(alerts, detonationUuid)
+	if len(alerts) == 1 {
+		log.Info("One open alert found")
+		m.AlertId = alerts[0].ID
+		m.Index = alerts[0].Index
+		return true, nil
+	}
+	if len(alerts) > 1 {
+		// TODO: It may well be desirable for a suspicious event to trigger multiple alerts
+		// In future ElasticsearchAlertGeneratedAssertion.AlertFilter should be a list, capable
+		// of matching and closing multiple alerts associated with a single event.
+		log.Errorf("More than one alert found")
+		return false, nil
+	}
+	log.Warnf("No alerts found")
 	return false, nil
 }
 
 func (m *ElasticsearchAlertGeneratedAssertion) String() string {
-	return fmt.Sprintf("Elasticsearch alert '%s'", m.AlertFilter.Rule)
+	return fmt.Sprintf("Elasticsearch alert '%s'", m.AlertFilter.RuleName)
 }
 
 func (m *ElasticsearchAlertGeneratedAssertion) Cleanup(detonationUuid string) error {
-	signals, err := m.SignalsAPI.SearchSignals(QueryAllOpenSignals)
-	if err != nil {
-		return errors.New("unable to search for Datadog security monitoring signals: " + err.Error())
+	log.Infof("Closing alert for detonation: %s, for rule: %s with AlertId: %s in Index: %s", detonationUuid, m.AlertFilter.RuleName, m.AlertId, m.Index)
+	// If HasExpectedAlert() executed properly then m.AlertId ought to be set with the ID we need
+	if m.AlertId == "" {
+		return errors.New("AlertId not set, cannot close alert")
 	}
-
-	for i := range signals {
-		if m.signalMatchesExecution(signals[i], detonationUuid) {
-			if err := m.SignalsAPI.CloseSignal(*signals[i].Id); err != nil {
-				return errors.New("unable to archive signal " + *signals[i].Id + ": " + err.Error())
-			}
+	// We can query via the .siem-signals-default alias, however this isn't the actual index the document is in.
+	// To write to the index we need the actual index ID. Fortunately that data is in the document and we should
+	// have written that also when we ran HasExpectedAlert().
+	if m.Index == "" {
+		return errors.New("Index not set, cannot close alert")
+	}	
+	update_request_body := `
+	{
+		"doc": {
+			"kibana.alert.workflow_status": "closed"
 		}
+	}`
+	resp, err := m.AlertAPI.Update(m.Index, m.AlertId, strings.NewReader(update_request_body))
+	log.Info("Logging the update API response:\n",resp, "\n")
+	if err != nil {
+		log.Errorf("Error while trying to update document: %s", m.AlertId)
+		return err
 	}
 
 	return nil

pkg/threatest/matchers/elasticsearch/types.go

@@ -1,24 +1,47 @@
 package elasticsearch
 
 import (
-	"context"
 	"os"
+	"time"
+	"github.com/cenkalti/backoff/v4"
+	"strings"
+	"fmt"
 
-	elasticsearch "github.com/elastic/go-elasticsearch/v8"
+	log "github.com/sirupsen/logrus"
+	es "github.com/elastic/go-elasticsearch/v8"
 )
 
+type ElasticsearchQueryResponse struct {
+	Hits struct {
+		Total struct {
+			Value    int    `json:"value"`
+		} `json:"total"`
+		Hits []ElasticsearchQueryHit `json:"hits"`
+	} `json:"hits"`
+}
+
+type ElasticsearchQueryHit struct {
+	Index  string                 `json:"_index"`
+	ID     string                 `json:"_id"`
+	Source map[string]interface{} `json:"_source"`
+}
+
 type ElasticsearchAlertFilter struct {
-	RuleName string `yaml:"rule-name"`
+	RuleName  string `yaml:"rule-name"`
+	UuidField string `yaml:"uuid-field"`
 }
 
 type ElasticsearchAlertGeneratedAssertion struct {
-	AlertAPI    ElasticsearchClient
+	AlertAPI    es.Client
 	AlertFilter *ElasticsearchAlertFilter
+	AlertId     string
+	Index       string
 }
 
-func GetElasticSearchClient() (ElasticsearchClient, error) {
+func ElasticsearchAlert(ruleName, uuidField string) *ElasticsearchAlertGeneratedAssertion {
+	retryBackoff := backoff.NewExponentialBackOff()
 	// New Elasticsearch client
-	cfg := elasticsearch.Config{
+	esClient, err := es.NewClient(es.Config{
 		Addresses: []string{os.Getenv("ELASTICSEARCH_URL")},
 		Username:  os.Getenv("ELASTICSEARCH_USERNAME"),
 		Password:  os.Getenv("ELASTICSEARCH_PASSWORD"),
@@ -33,27 +56,59 @@ func GetElasticSearchClient() (ElasticsearchClient, error) {
 		},
 		// Retry up to 5 attempts
 		MaxRetries: 5,
-	}
-	es, err := elasticsearch.NewTypedClient(cfg)
+	})
 	if err != nil {
-		return nil, fmt.Errorf("failed to create Elasticsearch client: %w", err)
+		log.Fatalf("failed to create Elasticsearch client: %w", err)
 	}
-	info, err := es.Info().Do(ctx)
+	info, err := esClient.Info()
 	if err != nil {
-		return nil, fmt.Errorf("failed to get Elasticsearch cluster info: %w", err)
+		log.Fatalf("failed to get Elasticsearch cluster info: %w", err)
+	}
+	log.Info("Elasticsearch cluster info:\n", info.String())
+	return &ElasticsearchAlertGeneratedAssertion{
+		AlertAPI:    *esClient,
+		AlertFilter: &ElasticsearchAlertFilter{RuleName: ruleName, UuidField: uuidField},
 	}
-	log.Info("Elasticsearch cluster info", "cluster_name", info.ClusterName, "version", info.Version.Number)
-	return es, nil
 }
 
-func ElasticsearchAlert(rule, tellTale string) *ElasticsearchAlertGeneratedAssertion {
-	es, err := GetElasticSearchClient()
+
+// Dumping Ground
+func CreateIndex(m *ElasticsearchAlertGeneratedAssertion, index string) {
+	mapping := `
+    {
+      "settings": {
+        "number_of_shards": 1
+      },
+      "mappings": {
+        "properties": {
+          "field1": {
+            "type": "text"
+          },
+		  "date": {
+			"type": "date" 
+		  }
+        }
+      }
+    }`
+	res, err := m.AlertAPI.Indices.Create(
+        index,
+        m.AlertAPI.Indices.Create.WithBody(strings.NewReader(mapping)),
+    )
+    if err != nil {
+        log.Fatal(err)
+    }
+    log.Println(res)
+}
+
+func WriteToIndex(m *ElasticsearchAlertGeneratedAssertion, index string) {
+	entry := `
+	{
+		"field1": "helloo there abc1234",
+		"@timestamp": "%s"
+	}`
+	res, err := m.AlertAPI.Index(index, strings.NewReader(fmt.Sprintf(entry, time.Now().Format("2006/01/02 15:04:05"))))
 	if err != nil {
-		log.Error(err, "failed to create Elasticsearch client")
-		return nil
-	}
-	return &ElasticsearchAlertGeneratedAssertion{
-		AlertAPI:    es,
-		AlertFilter: &ElasticsearchAlertFilter{RuleName: rule},
+		log.Fatal(err)
 	}
+	log.Println(res)
 }