From 87c444d3578e08baf9c4235df366baa6562640bf Mon Sep 17 00:00:00 2001 From: Aayush garg Date: Tue, 13 Feb 2024 11:17:45 +0530 Subject: [PATCH 01/30] added support for secure cookie event --- security_intercept/intercept.go | 34 ++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/security_intercept/intercept.go b/security_intercept/intercept.go index e502fbc..abce225 100644 --- a/security_intercept/intercept.go +++ b/security_intercept/intercept.go @@ -509,12 +509,25 @@ func DissociateInboundRequest() { removeFuzzFile(tmpFiles) } +func checkSecureCookies(responseHeader http.Header) { + logger.Debugln("Response Header", responseHeader) + if responseHeader != nil { + tmpRes := http.Response{Header: responseHeader} + cookies := tmpRes.Cookies() + for _, cookie := range cookies { + var arg []bool + arg = append(arg, cookie.Secure) + secConfig.Secure.SendEvent("SECURE_COOKIE", arg) + } + } +} + func XssCheck() { if !isAgentInitialized() { return } r := secConfig.Secure.GetRequest() - + checkSecureCookies(r.ResponseHeader) if r != nil { if r.ResponseBody != "" && !IsRXSSDisable() { @@ -724,6 +737,8 @@ func SendEvent(caseType string, data ...interface{}) interface{} { DissociateInboundRequest() case "INBOUND_WRITE": httpresponseHandler(data...) + case "RESPONSE_HEADER": + httpresponseHeader(data...) case "OUTBOUND": return outboundcallHandler(data[0]) case "GRPC": @@ -825,6 +840,7 @@ func httpresponseHandler(data ...interface{}) { responseHeader = hdr } + fmt.Println("responseHeader", data[1]) if contentType != "" && !secUtils.IsContentTypeSupported(contentType) { logger.Debugln("No need to send RXSS event ContentType not supported for rxss event validation", contentType) return @@ -842,6 +858,22 @@ func httpresponseHandler(data ...interface{}) { } } +func httpresponseHeader(data ...interface{}) { + if len(data) < 1 { + return + } + header := data[0] + + contentType := "" + responseHeader := http.Header{} + if hdr, ok := header.(http.Header); ok && hdr != nil { + contentType = hdr.Get("content-type") + responseHeader = hdr + } + logger.Debugln("HTTP response header api called ", responseHeader) + AssociateResponseBody("", contentType, responseHeader) +} + func grpcRequestHandler(data ...interface{}) { if data == nil || !isAgentInitialized() { return From 3df6e957322e6a2efd7071608c0db8ed9008c7cf Mon Sep 17 00:00:00 2001 From: Aayush garg Date: Wed, 5 Jun 2024 12:13:07 +0530 Subject: [PATCH 02/30] Update secure cookie event --- security_intercept/intercept.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/security_intercept/intercept.go b/security_intercept/intercept.go index abce225..db3ace7 100644 --- a/security_intercept/intercept.go +++ b/security_intercept/intercept.go @@ -514,11 +514,17 @@ func checkSecureCookies(responseHeader http.Header) { if responseHeader != nil { tmpRes := http.Response{Header: responseHeader} cookies := tmpRes.Cookies() + var arg []map[string]interface{} for _, cookie := range cookies { - var arg []bool - arg = append(arg, cookie.Secure) - secConfig.Secure.SendEvent("SECURE_COOKIE", arg) + + arg = append(arg, map[string]interface{}{ + "name": cookie.Name, + "isHttpOnly": cookie.HttpOnly, + "isSecure": cookie.Secure, + "value": cookie.Value, + }) } + secConfig.Secure.SendEvent("SECURE_COOKIE", arg) } } From 7c86c1209698d63c1720bf69d7dcdee9258dca33 Mon Sep 17 00:00:00 2001 From: Aayush garg Date: Fri, 12 Jul 2024 11:12:15 +0530 Subject: [PATCH 03/30] Added new key identifiers in all the JSONs --- security_event_generation/event_generation.go | 21 ++++--------------- .../event_generation_utils.go | 21 +++++++++---------- security_handlers/control_command_handler.go | 4 ++-- 3 files changed, 16 insertions(+), 30 deletions(-) diff --git a/security_event_generation/event_generation.go b/security_event_generation/event_generation.go index a71dc9b..821969b 100644 --- a/security_event_generation/event_generation.go +++ b/security_event_generation/event_generation.go @@ -101,6 +101,8 @@ func getApplicationIdentifiers(jsonName string) ApplicationIdentifiers { applicationIdentifier.Pid = secConfig.GlobalInfo.ApplicationInfo.GetPid() agentStartTime := secConfig.GlobalInfo.ApplicationInfo.GetStarttimestr().Unix() * 1000 applicationIdentifier.StartTime = secUtils.Int64ToString(agentStartTime) + applicationIdentifier.AppAccountId = secConfig.GlobalInfo.MetaData.GetAccountID() + applicationIdentifier.AppEntityGuid = secConfig.GlobalInfo.MetaData.GetEntityGuid() applicationIdentifier.LinkingMetadata = secConfig.GlobalInfo.MetaData.GetLinkingMetadata() return applicationIdentifier @@ -400,26 +402,12 @@ func SendExitEvent(eventTracker *secUtils.EventTracker, requestIdentifier string } } -func SendUpdatedPolicy(policy secConfig.Policy) { - logger.Infoln("Sending Updated policy ", policy.Version) - type policy1 struct { - JSONName string `json:"jsonName"` - secConfig.Policy - } - - _, err := sendEvent(policy1{"lc-policy", policy}, "", "") - if err != nil { - logger.Errorln(err) - } -} - func IASTDataRequest(batchSize int, completedRequestIds interface{}, pendingRequestIds []string) { var tmp_event IASTDataRequestBeen tmp_event.CompletedRequests = completedRequestIds tmp_event.PendingRequestIds = pendingRequestIds tmp_event.BatchSize = batchSize - tmp_event.ApplicationUUID = secConfig.GlobalInfo.ApplicationInfo.GetAppUUID() - tmp_event.JSONName = "iast-data-request" + tmp_event.ApplicationIdentifiers = getApplicationIdentifiers("iast-data-request") _, err := sendEvent(tmp_event, "", "") if err != nil { logger.Errorln(err) @@ -500,8 +488,7 @@ func SendLogMessage(log, caller, logLevel string) { var tmp_event LogMessage - tmp_event.ApplicationUUID = secConfig.GlobalInfo.ApplicationInfo.GetAppUUID() - tmp_event.JSONName = "critical-messages" + tmp_event.ApplicationIdentifiers = getApplicationIdentifiers("critical-messages") tmp_event.Timestamp = time.Now().Unix() * 1000 tmp_event.Level = logLevel tmp_event.Message = log diff --git a/security_event_generation/event_generation_utils.go b/security_event_generation/event_generation_utils.go index 395d0aa..1dce77c 100644 --- a/security_event_generation/event_generation_utils.go +++ b/security_event_generation/event_generation_utils.go @@ -162,6 +162,8 @@ type ApplicationIdentifiers struct { JSONName string `json:"jsonName"` Pid string `json:"pid"` StartTime string `json:"startTime"` + AppAccountId string `json:"appAccountId"` + AppEntityGuid string `json:"appEntityGuid"` LinkingMetadata interface{} `json:"linkingMetadata"` } @@ -171,8 +173,7 @@ type FuzzFailBean struct { } type IASTDataRequestBeen struct { - JSONName string `json:"jsonName"` - ApplicationUUID string `json:"applicationUUID"` + ApplicationIdentifiers BatchSize int `json:"batchSize"` CompletedRequests interface{} `json:"completedRequests"` PendingRequestIds []string `json:"pendingRequestIds"` @@ -191,15 +192,13 @@ type Urlmappings struct { } type LogMessage struct { - JSONName string `json:"jsonName"` - ApplicationUUID string `json:"applicationUUID"` - Timestamp int64 `json:"timestamp"` - Level string `json:"level"` - Message string `json:"message"` - Caller string `json:"caller"` - Exception Exception `json:"exception"` - ThreadName string `json:"threadName"` - LinkingMetadata interface{} `json:"linkingMetadata"` + ApplicationIdentifiers + Timestamp int64 `json:"timestamp"` + Level string `json:"level"` + Message string `json:"message"` + Caller string `json:"caller"` + Exception Exception `json:"exception"` + ThreadName string `json:"threadName"` } type Exception struct { diff --git a/security_handlers/control_command_handler.go b/security_handlers/control_command_handler.go index a298fc9..77ab812 100644 --- a/security_handlers/control_command_handler.go +++ b/security_handlers/control_command_handler.go @@ -96,8 +96,8 @@ func parseControlCommand(arg []byte) (error, bool) { logger.Errorln("Unable to unmarshall cc100 ", err) } else { logger.Debugln("defaultPolicy", defaultPolicy.Data) - policy := secConfig.UpdateGlobalConf(defaultPolicy.Data, string(arg)) - eventGeneration.SendUpdatedPolicy(policy) + secConfig.UpdateGlobalConf(defaultPolicy.Data, string(arg)) + } FuzzHandler.InitFuzzScheduler() case POLICY_UPDATE_FAILED_DUE_TO_VALIDATION_ERROR: From 65ebefc64b78b940dc7d81c18c1d1f5e777cbdca Mon Sep 17 00:00:00 2001 From: Aayush garg Date: Fri, 12 Jul 2024 11:18:55 +0530 Subject: [PATCH 04/30] update json version --- internal/security_utils/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/security_utils/config.go b/internal/security_utils/config.go index bccdef1..bc16bac 100644 --- a/internal/security_utils/config.go +++ b/internal/security_utils/config.go @@ -5,7 +5,7 @@ package security_utils const ( CollectorVersion = "1.3.0" - JsonVersion = "1.2.3" + JsonVersion = "1.2.4" CollectorType = "GOLANG" BuildNumber = "160" ) From d9c1b58f4c25010113fc6cf2034ac766740fdf59 Mon Sep 17 00:00:00 2001 From: Aayush garg Date: Fri, 12 Jul 2024 12:28:34 +0530 Subject: [PATCH 05/30] update fuzz header parsing --- internal/security_utils/encryptorUtils.go | 5 ++ internal/security_utils/string_utils.go | 13 ++++ security_intercept/intercept.go | 57 ++------------- security_intercept/intercept_utils.go | 85 ++++++++++++++++++++++- 4 files changed, 106 insertions(+), 54 deletions(-) diff --git a/internal/security_utils/encryptorUtils.go b/internal/security_utils/encryptorUtils.go index 006080b..f23c988 100644 --- a/internal/security_utils/encryptorUtils.go +++ b/internal/security_utils/encryptorUtils.go @@ -24,6 +24,7 @@ const ( ENCRYPTED_DATA_DECRYPTED_DATA = "Encrypted Data: %s, Decrypted data: %s" ERROR_WHILE_GENERATING_REQUIRED_SALT_FROM_S_S = "Error while generating required salt from %s" ERROR_WHILE_VERIFY_HASH_DATA = "Hash Data not macth %s: %s" + EMPTY_DATA = "Empty decrypted data" ) func Decrypt(password, encryptedData, hashVerifier string) (string, error) { @@ -61,6 +62,10 @@ func Decrypt(password, encryptedData, hashVerifier string) (string, error) { decrypted = removePadding(decrypted) decryptedData := string(decrypted[OFFSET:]) + if IsBlank(decryptedData) { + return "", fmt.Errorf(EMPTY_DATA) + } + if verifyHashData(hashVerifier, decryptedData) { return decryptedData, nil } else { diff --git a/internal/security_utils/string_utils.go b/internal/security_utils/string_utils.go index 97cf56f..4a4fd29 100644 --- a/internal/security_utils/string_utils.go +++ b/internal/security_utils/string_utils.go @@ -101,3 +101,16 @@ func StringSHA256(f string) string { hex.Encode(dst, sum[:]) return string(dst) } + +func IsBlank(in string) bool { + return in == "" +} + +func IsAnyBlank(stringSequence ...string) bool { + for in := range stringSequence { + if IsBlank(stringSequence[in]) { + return true + } + } + return false +} diff --git a/security_intercept/intercept.go b/security_intercept/intercept.go index de7b39d..6789412 100644 --- a/security_intercept/intercept.go +++ b/security_intercept/intercept.go @@ -344,7 +344,9 @@ func TraceIncommingRequest(url, host string, hdrMap map[string][]string, method (*infoReq).RequestIdentifier = RequestIdentifier (*infoReq).Request.ServerName = serverName (*infoReq).BodyLimit = secConfig.GlobalInfo.BodyLimit() - (*infoReq).TmpFiles = createFuzzFile(RequestIdentifier) + + requestIdentifier := parseFuzzRequestIdentifierHeader(RequestIdentifier) + (*infoReq).TmpFiles = requestIdentifier.TempFiles (*infoReq).ParentID = parentID if type1 == "gRPC" { (*infoReq).Request.IsGRPC = true @@ -493,7 +495,9 @@ func tracerpcRequestWithHeader(header map[string]string, data []byte) { if (*infoReq).RequestIdentifier != "" { header[NR_CSEC_FUZZ_REQUEST_ID] = (*infoReq).RequestIdentifier } - (*infoReq).TmpFiles = createFuzzFile((*infoReq).RequestIdentifier) + + requestIdentifier := parseFuzzRequestIdentifierHeader((*infoReq).RequestIdentifier) + (*infoReq).TmpFiles = requestIdentifier.TempFiles if (*infoReq).Request.ServerName == "" { (*infoReq).Request.ServerName = host } @@ -566,55 +570,6 @@ func XssCheck() { * Handling for IAST mode */ -// create a remove fuzz file for verfy file acesss attack -func createFuzzFile(fuzzheaders string) (tmpFiles []string) { - DSON := true - if DSON && fuzzheaders != "" { - additionalData := strings.Split(fuzzheaders, IAST_SEP) - logger.Debugln("additionalData:", additionalData) - if len(additionalData) >= 8 { - encryptedData := additionalData[6] - hashVerifier := additionalData[7] - logger.Debugln("Encrypted file name : ", encryptedData) - filesToCreate, err := secUtils.Decrypt(secConfig.GlobalInfo.MetaData.GetEntityGuid(), encryptedData, hashVerifier) - - if err != nil { - logger.Errorln(err) - SendLogMessage(err.Error(), "createFuzzFile", "SEVERE") - return - } - - logger.Debugln("Decrypted file name : ", filesToCreate) - allFiles := strings.Split(filesToCreate, COMMA_DELIMETER) - - for i := range allFiles { - fileName := allFiles[i] - dsFilePath := filepath.Join(secConfig.GlobalInfo.SecurityHomePath(), "nr-security-home", "tmp") - fileName = strings.Replace(fileName, "{{NR_CSEC_VALIDATOR_HOME_TMP}}", dsFilePath, -1) - fileName = strings.Replace(fileName, "%7B%7BNR_CSEC_VALIDATOR_HOME_TMP%7D%7D", dsFilePath, -1) - absfileName, _ := filepath.Abs(fileName) - if absfileName != "" { - fileName = absfileName - } - tmpFiles = append(tmpFiles, fileName) - dir := filepath.Dir(fileName) - if dir != "" { - err := os.MkdirAll(dir, 0770) - if err != nil { - logger.Debugln("Error while creating file : ", err.Error()) - } - } - emptyFile, err := os.Create(fileName) - if err != nil { - logger.Debugln("Error while creating file : ", err.Error(), fileName) - } - emptyFile.Close() - } - } - } - return tmpFiles -} - func removeFuzzFile(tmpFiles []string) { for _, path := range tmpFiles { err := os.Remove(path) diff --git a/security_intercept/intercept_utils.go b/security_intercept/intercept_utils.go index 8459156..4b74b46 100644 --- a/security_intercept/intercept_utils.go +++ b/security_intercept/intercept_utils.go @@ -174,9 +174,7 @@ func getContentType(header map[string]string) string { return "" } -func getIpAndPort(data string) (string, string) { - var port = "" - var ip = "" +func getIpAndPort(data string) (ip string, port string) { if data == "" { return ip, port } @@ -227,3 +225,84 @@ type parameters struct { Payload interface{} `json:"payload"` PayloadType interface{} `json:"payloadType"` } + +type NrRequestIdentifier struct { + Raw string + RefID string + RefValue string + APIRecordID string + NrRequest bool + NextStage string + RecordIndex string + RefKey string + TempFiles []string +} + +func parseFuzzRequestIdentifierHeader(requestHeaderVal string) (nrRequestIdentifier NrRequestIdentifier) { + nrRequestIdentifier.Raw = requestHeaderVal + if !secUtils.IsBlank(requestHeaderVal) { + return + } + data := strings.Split(requestHeaderVal, IAST_SEP) + + if len(data) >= 8 { + nrRequestIdentifier.APIRecordID = data[0] + nrRequestIdentifier.RefID = data[1] + nrRequestIdentifier.RefValue = data[2] + nrRequestIdentifier.NextStage = data[3] + nrRequestIdentifier.RecordIndex = data[4] + nrRequestIdentifier.RefKey = data[5] + nrRequestIdentifier.NrRequest = true + + } + if !secUtils.IsAnyBlank(data[6], data[7]) { + + encryptedData := data[6] + hashVerifier := data[7] + logger.Debugln("Request Identifier, Encrypted Files = ", encryptedData) + + filesToCreate, err := secUtils.Decrypt(secConfig.GlobalInfo.MetaData.GetEntityGuid(), encryptedData, hashVerifier) + + if err != nil { + logger.Errorln("Request Identifier, decryption of files failed ", err) + SendLogMessage("Request Identifier, decryption of files failed "+err.Error(), "parseFuzzRequestIdentifierHeader", "SEVERE") + return + } + logger.Debugln("Request Identifier, Decrypted Files = ", filesToCreate) + nrRequestIdentifier.TempFiles = createFuzzFileTemp(filesToCreate) + } + + return +} + +func createFuzzFileTemp(filesToCreate string) (tmpFiles []string) { + + allFiles := strings.Split(filesToCreate, COMMA_DELIMETER) + + for i := range allFiles { + fileName := allFiles[i] + dsFilePath := filepath.Join(secConfig.GlobalInfo.SecurityHomePath(), "nr-security-home", "tmp") + fileName = strings.Replace(fileName, "{{NR_CSEC_VALIDATOR_HOME_TMP}}", dsFilePath, -1) + fileName = strings.Replace(fileName, "%7B%7BNR_CSEC_VALIDATOR_HOME_TMP%7D%7D", dsFilePath, -1) + + absfileName, _ := filepath.Abs(fileName) + if absfileName != "" { + fileName = absfileName + } + + tmpFiles = append(tmpFiles, fileName) + dir := filepath.Dir(fileName) + if dir != "" { + err := os.MkdirAll(dir, 0770) + if err != nil { + logger.Debugln("Failed to create directory : ", err.Error()) + } + } + emptyFile, err := os.Create(fileName) + if err != nil { + logger.Debugln("Failed to create file : ", err.Error(), fileName) + } + emptyFile.Close() + } + return tmpFiles +} From a74e53c87a3c0e0b3f41a57d90ebf781c0611075 Mon Sep 17 00:00:00 2001 From: Aayush garg Date: Fri, 12 Jul 2024 12:57:48 +0530 Subject: [PATCH 06/30] refactoring to prevent duplicate storage for header --- internal/security_utils/global_utils.go | 16 ++- security_event_generation/event_generation.go | 11 +- security_implementation/implementation.go | 6 +- security_intercept/intercept.go | 136 +++--------------- security_intercept/intercept_utils.go | 37 +++-- 5 files changed, 60 insertions(+), 146 deletions(-) diff --git a/internal/security_utils/global_utils.go b/internal/security_utils/global_utils.go index b64c261..231e730 100644 --- a/internal/security_utils/global_utils.go +++ b/internal/security_utils/global_utils.go @@ -11,13 +11,11 @@ type Info_req struct { ResponseBody string ResponseHeader http.Header ResponseContentType string - GrpcByte [][]byte GrpcBody []interface{} ReqTraceData string - RequestIdentifier string + RequestIdentifier NrRequestIdentifier Request RequestInfo VulnerabilityDetails VulnerabilityDetails - TmpFiles []string ReflectedMetaData ReflectedMetaData ParentID string BodyLimit int @@ -81,6 +79,18 @@ type VulnerabilityDetails struct { Stacktrace []string `json:"stacktrace"` } +type NrRequestIdentifier struct { + Raw string + RefID string + RefValue string + APIRecordID string + NrRequest bool + NextStage string + RecordIndex string + RefKey string + TempFiles []string +} + var CaCert = ` -----BEGIN CERTIFICATE----- MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw diff --git a/security_event_generation/event_generation.go b/security_event_generation/event_generation.go index a71dc9b..6a09ff9 100644 --- a/security_event_generation/event_generation.go +++ b/security_event_generation/event_generation.go @@ -321,7 +321,7 @@ func SendVulnerableEvent(req *secUtils.Info_req, category string, args interface tmp_event.VulnerabilityDetails = vulnerabilityDetails tmp_event.ApplicationIdentifiers = getApplicationIdentifiers("Event") - fuzzHeader := (*req).RequestIdentifier + requestIdentifier := (*req).RequestIdentifier // if (*req).RequestIdentifier != "" { // tmp_event.Stacktrace = []string{} // } @@ -336,10 +336,9 @@ func SendVulnerableEvent(req *secUtils.Info_req, category string, args interface tmp_event.MetaData.AppServerInfo.ApplicationDirectory = secConfig.GlobalInfo.EnvironmentInfo.Wd tmp_event.MetaData.AppServerInfo.ServerBaseDirectory = secConfig.GlobalInfo.EnvironmentInfo.Wd - requestType := "raspEvent" if secConfig.GlobalInfo.GetCurrentPolicy().VulnerabilityScan.Enabled && secConfig.GlobalInfo.GetCurrentPolicy().VulnerabilityScan.IastScan.Enabled { - if fuzzHeader != "" { + if requestIdentifier.NrRequest { requestType = "iastEvent" tmp_event.IsIASTRequest = true } @@ -361,9 +360,9 @@ func SendVulnerableEvent(req *secUtils.Info_req, category string, args interface } } - if req.ParentID != "" && req.RequestIdentifier != "" { + if req.ParentID != "" && requestIdentifier.NrRequest { tmp_event.ParentId = req.ParentID - apiId := strings.Split(req.RequestIdentifier, ":")[0] + apiId := requestIdentifier.APIRecordID if apiId == vulnerabilityDetails.APIID { (secConfig.SecureWS).AddCompletedRequests(req.ParentID, eventId) } @@ -383,7 +382,7 @@ func SendVulnerableEvent(req *secUtils.Info_req, category string, args interface logging.Disableinitlogs() } tracingHeader := tmp_event.HTTPRequest.Headers[NR_CSEC_TRACING_DATA] - return &secUtils.EventTracker{APIID: tmp_event.APIID, ID: tmp_event.ID, CaseType: tmp_event.CaseType, TracingHeader: tracingHeader, RequestIdentifier: fuzzHeader} + return &secUtils.EventTracker{APIID: tmp_event.APIID, ID: tmp_event.ID, CaseType: tmp_event.CaseType, TracingHeader: tracingHeader, RequestIdentifier: requestIdentifier.Raw} } diff --git a/security_implementation/implementation.go b/security_implementation/implementation.go index a998472..2e73a79 100644 --- a/security_implementation/implementation.go +++ b/security_implementation/implementation.go @@ -144,7 +144,7 @@ func (k Secureimpl) AssociateGrpcDataBytes(data []byte) bool { logger.Errorln("(AssociateGrpcDataBytes) GRPC Request Not Found creating new request without headers") return false } - request.GrpcByte = append(request.GrpcByte, data) + // request.GrpcByte = append(request.GrpcByte, data) // deprecated return true } @@ -184,7 +184,7 @@ func (k Secureimpl) GetFuzzHeader() string { if request == nil { return "" } else { - return request.RequestIdentifier + return request.RequestIdentifier.Raw } } @@ -193,7 +193,7 @@ func (k Secureimpl) GetTmpFiles() []string { if request == nil { return make([]string, 0) } else { - return request.TmpFiles + return request.RequestIdentifier.TempFiles } } diff --git a/security_intercept/intercept.go b/security_intercept/intercept.go index 6789412..98f3909 100644 --- a/security_intercept/intercept.go +++ b/security_intercept/intercept.go @@ -289,74 +289,36 @@ func AssociateApplicationPort(data string) { // TraceIncommingRequest - interception of incoming request -func TraceIncommingRequest(url, host string, hdrMap map[string][]string, method string, body string, queryparam map[string][]string, protocol, serverName, type1 string, bodyReader secUtils.SecWriter, csecAttributes map[string]any) { +func TraceIncommingRequest(url, host string, hdrMap map[string][]string, method string, body string, queryparam map[string][]string, protocol, serverName, reqtype string, bodyReader secUtils.SecWriter, csecAttributes map[string]any) { if !isAgentInitialized() { return } - clientIp := "" - clientPort := "" - if host != "" { - clientIp, clientPort = getIpAndPort(host) - } - - // filter request headers - filterHeader := map[string]string{} - RequestIdentifier := "" - traceData := "" - parentID := "" - - for k, v := range hdrMap { - if secUtils.CaseInsensitiveEquals(k, NR_CSEC_TRACING_DATA) { - traceData = strings.Join(v, ",") - } else if secUtils.CaseInsensitiveEquals(k, NR_CSEC_FUZZ_REQUEST_ID) { - RequestIdentifier = strings.Join(v, ",") - } else if secUtils.CaseInsensitiveEquals(k, NR_CSEC_PARENT_ID) { - parentID = strings.Join(v, ",") - } else { - filterHeader[k] = strings.Join(v, ",") - } - } - if traceData != "" { - filterHeader[NR_CSEC_TRACING_DATA] = traceData - } - if RequestIdentifier != "" { - filterHeader[NR_CSEC_FUZZ_REQUEST_ID] = RequestIdentifier - } - if parentID != "" { - filterHeader[NR_CSEC_PARENT_ID] = parentID - } - // record incoming request + infoReq := new(secUtils.Info_req) - (*infoReq).Request.URL = url - (*infoReq).Request.ParameterMap = queryparam - (*infoReq).Request.ClientIP = clientIp - (*infoReq).Request.ClientPort = clientPort - (*infoReq).Request.ServerPort = getServerPort() - (*infoReq).Request.IsGRPC = false - (*infoReq).Request.Headers = filterHeader - (*infoReq).GrpcByte = make([][]byte, 0) - (*infoReq).Request.Method = method - (*infoReq).Request.Body = body - (*infoReq).Request.BodyReader = bodyReader - (*infoReq).Request.Protocol = protocol - (*infoReq).Request.ContentType = getContentType(filterHeader) - (*infoReq).ReqTraceData = traceData - (*infoReq).RequestIdentifier = RequestIdentifier - (*infoReq).Request.ServerName = serverName - (*infoReq).BodyLimit = secConfig.GlobalInfo.BodyLimit() - - requestIdentifier := parseFuzzRequestIdentifierHeader(RequestIdentifier) - (*infoReq).TmpFiles = requestIdentifier.TempFiles - (*infoReq).ParentID = parentID - if type1 == "gRPC" { - (*infoReq).Request.IsGRPC = true + infoReq.Request.URL = url + infoReq.Request.ParameterMap = queryparam + infoReq.Request.ClientIP, infoReq.Request.ClientPort = getIpAndPort(host) + infoReq.Request.ServerPort = getServerPort() + infoReq.Request.Headers = ToOneValueMap(hdrMap) + infoReq.Request.Method = method + infoReq.Request.Body = body + infoReq.Request.BodyReader = bodyReader + infoReq.Request.Protocol = protocol + infoReq.Request.ContentType = getContentType(hdrMap) + infoReq.ReqTraceData = getHeaderValue(hdrMap, NR_CSEC_TRACING_DATA) + infoReq.RequestIdentifier = parseFuzzRequestIdentifierHeader(getHeaderValue(hdrMap, NR_CSEC_FUZZ_REQUEST_ID)) + infoReq.Request.ServerName = serverName + infoReq.BodyLimit = secConfig.GlobalInfo.BodyLimit() + infoReq.ParentID = getHeaderValue(hdrMap, NR_CSEC_PARENT_ID) + + if reqtype == "gRPC" { + infoReq.Request.IsGRPC = true } for k, v := range csecAttributes { if secUtils.CaseInsensitiveEquals(k, AttributeCsecRoute) { - (*infoReq).Request.Route, _ = v.(string) + infoReq.Request.Route, _ = v.(string) } - } secConfig.Secure.AssociateInboundRequest(infoReq) @@ -448,61 +410,7 @@ func DissociateGrpcConnectionData() { } func tracerpcRequestWithHeader(header map[string]string, data []byte) { - if !isAgentInitialized() { - return - } - infoReq := &secUtils.Info_req{} - - (*infoReq).Request.Headers = make(map[string]string) - (*infoReq).Request.ParameterMap = make(map[string][]string, 0) - (*infoReq).Request.Method = "gRPC" - (*infoReq).Request.ServerPort = getServerPort() - (*infoReq).Request.IsGRPC = true - - if data != nil { - (*infoReq).GrpcByte = append((*infoReq).GrpcByte, data) - } - - host := "" - - for k, v := range header { - - if k == ":method" { - (*infoReq).Request.Method = v - } else if k == ":path" { - (*infoReq).Request.URL = v - } else if k == ":scheme" { - (*infoReq).Request.Protocol = v - } else if k == "content-type" { - (*infoReq).Request.ContentType = v - } else if secUtils.CaseInsensitiveEquals(k, NR_CSEC_TRACING_DATA) { - (*infoReq).ReqTraceData = v - delete(header, k) - } else if secUtils.CaseInsensitiveEquals(k, NR_CSEC_FUZZ_REQUEST_ID) { - (*infoReq).RequestIdentifier = v - delete(header, k) - } else if secUtils.CaseInsensitiveEquals(k, ":authority") { - (*infoReq).Request.ServerName = k - } else if secUtils.CaseInsensitiveEquals(k, ":host") { - host = k - } - } - - // reassign all deleted header - if (*infoReq).ReqTraceData != "" { - header[NR_CSEC_TRACING_DATA] = (*infoReq).ReqTraceData - } - if (*infoReq).RequestIdentifier != "" { - header[NR_CSEC_FUZZ_REQUEST_ID] = (*infoReq).RequestIdentifier - } - - requestIdentifier := parseFuzzRequestIdentifierHeader((*infoReq).RequestIdentifier) - (*infoReq).TmpFiles = requestIdentifier.TempFiles - if (*infoReq).Request.ServerName == "" { - (*infoReq).Request.ServerName = host - } - (*infoReq).Request.Headers = header - secConfig.Secure.AssociateInboundRequest(infoReq) + // deprecated } /** diff --git a/security_intercept/intercept_utils.go b/security_intercept/intercept_utils.go index 4b74b46..ca0bc54 100644 --- a/security_intercept/intercept_utils.go +++ b/security_intercept/intercept_utils.go @@ -6,6 +6,7 @@ package security_intercept import ( "fmt" "net/http" + "net/textproto" "net/url" "os" "path/filepath" @@ -164,14 +165,12 @@ func recoverFromPanic(functionName string) { } } -func getContentType(header map[string]string) string { +func getContentType(header map[string][]string) string { + return getHeaderValue(header, "Content-type") +} - for key, v := range header { - if secUtils.CaseInsensitiveEquals(key, "Content-type") { - return v - } - } - return "" +func getHeaderValue(header map[string][]string, key string) string { + return textproto.MIMEHeader(header).Get(key) } func getIpAndPort(data string) (ip string, port string) { @@ -226,19 +225,7 @@ type parameters struct { PayloadType interface{} `json:"payloadType"` } -type NrRequestIdentifier struct { - Raw string - RefID string - RefValue string - APIRecordID string - NrRequest bool - NextStage string - RecordIndex string - RefKey string - TempFiles []string -} - -func parseFuzzRequestIdentifierHeader(requestHeaderVal string) (nrRequestIdentifier NrRequestIdentifier) { +func parseFuzzRequestIdentifierHeader(requestHeaderVal string) (nrRequestIdentifier secUtils.NrRequestIdentifier) { nrRequestIdentifier.Raw = requestHeaderVal if !secUtils.IsBlank(requestHeaderVal) { return @@ -306,3 +293,13 @@ func createFuzzFileTemp(filesToCreate string) (tmpFiles []string) { } return tmpFiles } + +func ToOneValueMap(header map[string][]string) (filterHeader map[string]string) { + if header == nil { + return + } + for k, v := range header { + filterHeader[k] = strings.Join(v, ",") + } + return +} From 6ae94c49e553fd1aeab1054226275169d3d16bed Mon Sep 17 00:00:00 2001 From: Aayush garg Date: Fri, 12 Jul 2024 13:01:22 +0530 Subject: [PATCH 07/30] minor fir for distributed trace header key --- security_intercept/intercept.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/security_intercept/intercept.go b/security_intercept/intercept.go index 98f3909..8ff83cb 100644 --- a/security_intercept/intercept.go +++ b/security_intercept/intercept.go @@ -895,7 +895,7 @@ func DistributedTraceHeaders(hdrs *http.Request, secureAgentevent interface{}) { } value = GetFuzzHeader() if value != "" { - hdrs.Header.Add("NR_CSEC_FUZZ_REQUEST_ID", value) + hdrs.Header.Add(NR_CSEC_FUZZ_REQUEST_ID, value) } } From 10ce7d6cffa5d5f279b2c48b37b88724be48c2e5 Mon Sep 17 00:00:00 2001 From: Aayush garg Date: Fri, 12 Jul 2024 13:32:44 +0530 Subject: [PATCH 08/30] Updated Event send pipeline, added eventCategory parameter and remove swithc case for event category. --- internal/security_utils/security_interface.go | 2 +- security_event_generation/event_generation.go | 14 +--------- security_implementation/implementation.go | 18 ++++++------- security_intercept/intercept.go | 26 +++++++++---------- 4 files changed, 24 insertions(+), 36 deletions(-) diff --git a/internal/security_utils/security_interface.go b/internal/security_utils/security_interface.go index 3bcdac3..dee13c6 100644 --- a/internal/security_utils/security_interface.go +++ b/internal/security_utils/security_interface.go @@ -30,7 +30,7 @@ type Secureiface interface { AssociateFastHttpData(net.Conn) DisassociateFastHttpData() GetFastHttpData() net.Conn - SendEvent(category string, args interface{}) *EventTracker + SendEvent(caseType, eventCategory string, args interface{}) *EventTracker GetFuzzHeader() string GetTmpFiles() []string NewGoroutineLinker(interface{}) diff --git a/security_event_generation/event_generation.go b/security_event_generation/event_generation.go index a71dc9b..2ef0d11 100644 --- a/security_event_generation/event_generation.go +++ b/security_event_generation/event_generation.go @@ -289,21 +289,9 @@ func sendUrlMappingEvent() { } } -func SendVulnerableEvent(req *secUtils.Info_req, category string, args interface{}, vulnerabilityDetails secUtils.VulnerabilityDetails, eventId string) *secUtils.EventTracker { +func SendVulnerableEvent(req *secUtils.Info_req, category, eventCategory string, args interface{}, vulnerabilityDetails secUtils.VulnerabilityDetails, eventId string) *secUtils.EventTracker { var tmp_event eventJson - eventCategory := category - if eventCategory == "SQL_DB_COMMAND" { - eventCategory = "SQLITE" - } else if category == "NOSQL_DB_COMMAND" { - eventCategory = "MONGO" - } else if category == "REDIS_DB_COMMAND" { - eventCategory = "REDIS" - category = "SQL_DB_COMMAND" - } else if category == "DYNAMO_DB_COMMAND" { - eventCategory = "DQL" - } - tmp_event.ID = eventId tmp_event.CaseType = category tmp_event.EventCategory = eventCategory diff --git a/security_implementation/implementation.go b/security_implementation/implementation.go index a998472..7386f3e 100644 --- a/security_implementation/implementation.go +++ b/security_implementation/implementation.go @@ -56,7 +56,7 @@ func (k Secureimpl) SecureExecPrepareStatement(q_address string, qargs interface "parameters": qargs, } arg11 = append(arg11, tmp_map) - return k.SendEvent("SQL_DB_COMMAND", arg11) + return k.SendEvent("SQL_DB_COMMAND", "SQLITE", arg11) } /** @@ -262,13 +262,13 @@ func (k Secureimpl) Send5xxEvent(code int) { eventGeneration.Store5xxError(req, key, code) } -func (k Secureimpl) SendEvent(category string, args interface{}) *secUtils.EventTracker { - secConfig.AddEventDataToListener(secConfig.TestArgs{Parameters: fmt.Sprintf("%v", args), CaseType: category}) +func (k Secureimpl) SendEvent(caseType, eventCategory string, args interface{}) *secUtils.EventTracker { + secConfig.AddEventDataToListener(secConfig.TestArgs{Parameters: fmt.Sprintf("%v", args), CaseType: caseType}) if !isAgentReady() { return nil } eventId := increaseCount() - return sendEvent(eventId, category, args) + return sendEvent(eventId, caseType, eventCategory, args) } func (k Secureimpl) SendExitEvent(event *secUtils.EventTracker) { @@ -287,21 +287,21 @@ func (k Secureimpl) SendExitEvent(event *secUtils.EventTracker) { eventGeneration.SendExitEvent(event, requestIdentifier) } -func sendEvent(eventId, category string, args interface{}) *secUtils.EventTracker { +func sendEvent(eventId, caseType, eventCategory string, args interface{}) *secUtils.EventTracker { id := getID() req := getRequest(id) if !isAgentReady() || (req == nil) { - logger.Debugln(category, "no incoming skipping Event") + logger.Debugln(caseType, "no incoming skipping Event") return nil } var vulnerabilityDetails secUtils.VulnerabilityDetails - if category == "REFLECTED_XSS" && (*req).VulnerabilityDetails.APIID != "" { + if caseType == "REFLECTED_XSS" && (*req).VulnerabilityDetails.APIID != "" { vulnerabilityDetails = (*req).VulnerabilityDetails } else { - vulnerabilityDetails = presentStack((*req).Request.Method, category) + vulnerabilityDetails = presentStack((*req).Request.Method, caseType) } - return eventGeneration.SendVulnerableEvent(req, category, args, vulnerabilityDetails, getEventID(eventId, id)) + return eventGeneration.SendVulnerableEvent(req, caseType, eventCategory, args, vulnerabilityDetails, getEventID(eventId, id)) } diff --git a/security_intercept/intercept.go b/security_intercept/intercept.go index de7b39d..134d28e 100644 --- a/security_intercept/intercept.go +++ b/security_intercept/intercept.go @@ -106,9 +106,9 @@ func TraceFileOperation(fname string, flag int, isFileOpen bool) *secUtils.Event args = append(args, absolutePath) } if isFileOpen && isFileModified(flag) && fileInApplicationPath(fname) && fileExecByExtension(fname) { - return secConfig.Secure.SendEvent("FILE_INTEGRITY", args) + return secConfig.Secure.SendEvent("FILE_INTEGRITY", "FILE_INTEGRITY", args) } else { - return secConfig.Secure.SendEvent("FILE_OPERATION", args) + return secConfig.Secure.SendEvent("FILE_OPERATION", "FILE_INTEGRITY", args) } } @@ -122,7 +122,7 @@ func TraceSystemCommand(command string) *secUtils.EventTracker { } var arg []string arg = append(arg, command) - return secConfig.Secure.SendEvent("SYSTEM_COMMAND", arg) + return secConfig.Secure.SendEvent("SYSTEM_COMMAND", "SYSTEM_COMMAND", arg) } @@ -158,7 +158,7 @@ func TraceMongoOperation(arguments []byte, queryType string) *secUtils.EventTrac } arg12 = append(arg12, tmp_map1) - return secConfig.Secure.SendEvent("NOSQL_DB_COMMAND", arg12) + return secConfig.Secure.SendEvent("NOSQL_DB_COMMAND", "MONGO", arg12) } @@ -191,7 +191,7 @@ func TraceSqlOperation(query string, args ...interface{}) *secUtils.EventTracker } arg11 = append(arg11, tmp_map) - return secConfig.Secure.SendEvent("SQL_DB_COMMAND", arg11) + return secConfig.Secure.SendEvent("SQL_DB_COMMAND", "SQLITE", arg11) } func TracePrepareStatement(q, p string) { @@ -227,7 +227,7 @@ func TraceXpathOperation(a string) *secUtils.EventTracker { } var arg []string arg = append(arg, a) - return secConfig.Secure.SendEvent("XPATH", arg) + return secConfig.Secure.SendEvent("XPATH", "XPATH", arg) } /** @@ -240,7 +240,7 @@ func TraceLdapOperation(a map[string]string) *secUtils.EventTracker { } var arg []interface{} arg = append(arg, a) - return secConfig.Secure.SendEvent("LDAP", arg) + return secConfig.Secure.SendEvent("LDAP", "LDAP", arg) } /** @@ -254,7 +254,7 @@ func TraceJsOperation(a string) *secUtils.EventTracker { var arg []string arg = append(arg, a) - return secConfig.Secure.SendEvent("JAVASCRIPT_INJECTION", arg) + return secConfig.Secure.SendEvent("JAVASCRIPT_INJECTION", "JAVASCRIPT_INJECTION", arg) } /** @@ -555,7 +555,7 @@ func XssCheck() { var arg []string arg = append(arg, out) arg = append(arg, r.ResponseBody) - secConfig.Secure.SendEvent("REFLECTED_XSS", arg) + secConfig.Secure.SendEvent("REFLECTED_XSS", "REFLECTED_XSS", arg) } logger.Debugln("Called check for reflected XSS" + out) } @@ -845,7 +845,7 @@ func outboundcallHandler(req interface{}) *secUtils.EventTracker { } var args []interface{} args = append(args, r.URL.String()) - event := secConfig.Secure.SendEvent("HTTP_REQUEST", args) + event := secConfig.Secure.SendEvent("HTTP_REQUEST", "HTTP_REQUEST", args) return event } @@ -1016,7 +1016,7 @@ func mongoHandler(data ...interface{}) *secUtils.EventTracker { "payload": arg11, } arg12 = append(arg12, tmp_map1) - return secConfig.Secure.SendEvent("NOSQL_DB_COMMAND", arg12) + return secConfig.Secure.SendEvent("NOSQL_DB_COMMAND", "MONGO", arg12) } return nil } @@ -1041,7 +1041,7 @@ func dynamodbHandler(data ...interface{}) { if data == nil || !isAgentInitialized() { return } - secConfig.Secure.SendEvent("DYNAMO_DB_COMMAND", data[0]) + secConfig.Secure.SendEvent("DYNAMO_DB_COMMAND", "DQL", data[0]) } func redisHandler(data ...interface{}) { @@ -1049,7 +1049,7 @@ func redisHandler(data ...interface{}) { return } - secConfig.Secure.SendEvent("REDIS_DB_COMMAND", data) + secConfig.Secure.SendEvent("REDIS_DB_COMMAND", "REDIS", data) } func panicHandler(data ...interface{}) { From 54b40d4e14b9af28296e97fb229b1bd855dca4a7 Mon Sep 17 00:00:00 2001 From: Aayush garg Date: Fri, 12 Jul 2024 13:45:59 +0530 Subject: [PATCH 09/30] update event generation pipline --- security_event_generation/event_generation.go | 77 +++++++++---------- security_intercept/intercept_utils.go | 2 +- 2 files changed, 38 insertions(+), 41 deletions(-) diff --git a/security_event_generation/event_generation.go b/security_event_generation/event_generation.go index 2ef0d11..4281a68 100644 --- a/security_event_generation/event_generation.go +++ b/security_event_generation/event_generation.go @@ -296,24 +296,35 @@ func SendVulnerableEvent(req *secUtils.Info_req, category, eventCategory string, tmp_event.CaseType = category tmp_event.EventCategory = eventCategory tmp_event.Parameters = args - tmp_event.EventGenerationTime = strconv.FormatInt(time.Now().Unix()*1000, 10) tmp_event.BlockingProcessingTime = "1" tmp_event.HTTPRequest = req.Request - tmp_event.HTTPResponse = secUtils.ResponseInfo{ContentType: req.ResponseContentType} - if req.Request.BodyReader.GetBody != nil { - tmp_event.HTTPRequest.Body = string(req.Request.BodyReader.GetBody()) - } - if req.Request.BodyReader.IsDataTruncated != nil { - tmp_event.HTTPRequest.DataTruncated = req.Request.BodyReader.IsDataTruncated() - } tmp_event.VulnerabilityDetails = vulnerabilityDetails tmp_event.ApplicationIdentifiers = getApplicationIdentifiers("Event") + tmp_event.EventGenerationTime = strconv.FormatInt(time.Now().Unix()*1000, 10) + tmp_event.HTTPResponse = secUtils.ResponseInfo{ContentType: req.ResponseContentType} + tmp_event.MetaData.AppServerInfo.ApplicationDirectory = secConfig.GlobalInfo.EnvironmentInfo.Wd + tmp_event.MetaData.AppServerInfo.ServerBaseDirectory = secConfig.GlobalInfo.EnvironmentInfo.Wd + + if !req.Request.IsGRPC { + + if req.Request.BodyReader.GetBody != nil { + tmp_event.HTTPRequest.Body = string(req.Request.BodyReader.GetBody()) + } + if req.Request.BodyReader.IsDataTruncated != nil { + tmp_event.HTTPRequest.DataTruncated = req.Request.BodyReader.IsDataTruncated() + } + + } else { + + body := req.GrpcBody + grpc_body_json, err1 := json.Marshal(body) + if err1 != nil { + logger.Errorln("Error parsing gRPC request body: Invalid JSON format detected" + string(grpc_body_json)) + return nil + } else { + tmp_event.HTTPRequest.Body = string(grpc_body_json) + } - fuzzHeader := (*req).RequestIdentifier - // if (*req).RequestIdentifier != "" { - // tmp_event.Stacktrace = []string{} - // } - if req.Request.IsGRPC { tmp_event.MetaData.ReflectedMetaData = secUtils.ReflectedMetaData{ IsGrpcClientStream: req.ReflectedMetaData.IsGrpcClientStream, IsServerStream: req.ReflectedMetaData.IsServerStream, @@ -322,40 +333,26 @@ func SendVulnerableEvent(req *secUtils.Info_req, category, eventCategory string, } } - tmp_event.MetaData.AppServerInfo.ApplicationDirectory = secConfig.GlobalInfo.EnvironmentInfo.Wd - tmp_event.MetaData.AppServerInfo.ServerBaseDirectory = secConfig.GlobalInfo.EnvironmentInfo.Wd - requestType := "raspEvent" - if secConfig.GlobalInfo.GetCurrentPolicy().VulnerabilityScan.Enabled && secConfig.GlobalInfo.GetCurrentPolicy().VulnerabilityScan.IastScan.Enabled { + + fuzzHeader := req.RequestIdentifier + + if secConfig.GlobalInfo.IsIASTEnable() { + tmp_event.IsIASTEnable = true + if fuzzHeader != "" { requestType = "iastEvent" tmp_event.IsIASTRequest = true } - tmp_event.IsIASTEnable = true - } - - if tmp_event.HTTPRequest.ServerPort == "" { - tmp_event.HTTPRequest.ServerPort = "-1" - } - - if tmp_event.HTTPRequest.IsGRPC { - body := (*req).GrpcBody - grpc_bodyJson, err1 := json.Marshal(body) - if err1 != nil { - logger.Errorln("grpc_body JSON invalid" + string(grpc_bodyJson)) - return nil - } else { - tmp_event.HTTPRequest.Body = string(grpc_bodyJson) - } - } - if req.ParentID != "" && req.RequestIdentifier != "" { - tmp_event.ParentId = req.ParentID - apiId := strings.Split(req.RequestIdentifier, ":")[0] - if apiId == vulnerabilityDetails.APIID { - (secConfig.SecureWS).AddCompletedRequests(req.ParentID, eventId) + if req.ParentID != "" && req.RequestIdentifier != "" { + tmp_event.ParentId = req.ParentID + apiId := strings.Split(req.RequestIdentifier, ":")[0] + if apiId == vulnerabilityDetails.APIID { + (secConfig.SecureWS).AddCompletedRequests(req.ParentID, eventId) + } + tmp_event.HTTPRequest.Route = "" } - tmp_event.HTTPRequest.Route = "" } event_json, err1 := sendEvent(tmp_event, req.ParentID, requestType) diff --git a/security_intercept/intercept_utils.go b/security_intercept/intercept_utils.go index 8459156..a9e8663 100644 --- a/security_intercept/intercept_utils.go +++ b/security_intercept/intercept_utils.go @@ -71,7 +71,7 @@ func getServerPort() string { if serverPort != nil && len(serverPort) > 0 { return strconv.Itoa(serverPort[0]) } - return "" + return "-1" } func IsFileExist(name string) bool { From 211fcdbcfc21de571c7425bc0db77fa90ecf0d72 Mon Sep 17 00:00:00 2001 From: Aayush garg Date: Mon, 15 Jul 2024 13:01:28 +0530 Subject: [PATCH 10/30] code cleanup for rxss --- security_intercept/intercept.go | 139 +++++++++++++------------- security_intercept/intercept_utils.go | 4 +- 2 files changed, 70 insertions(+), 73 deletions(-) diff --git a/security_intercept/intercept.go b/security_intercept/intercept.go index 58d6839..5eebfa9 100644 --- a/security_intercept/intercept.go +++ b/security_intercept/intercept.go @@ -38,6 +38,7 @@ const ( NR_CSEC_PARENT_ID = "NR-CSEC-PARENT-ID" COMMA_DELIMETER = "," AttributeCsecRoute = "ROUTE" + SKIP_RXSS_EVENT = "Skipping RXSS event transmission: Content type not supported for RXSS event validation" ) /** @@ -360,19 +361,22 @@ func TraceIncommingRequest(url, host string, hdrMap map[string][]string, method secConfig.Secure.AssociateInboundRequest(infoReq) } -func AssociateResponseBody(body, contentType string, header http.Header) { - if !isAgentInitialized() { - return - } +func associateResponseBody(body string) { r := secConfig.Secure.GetRequest() if r != nil { r.ResponseBody = r.ResponseBody + body - r.ResponseContentType = contentType - r.ResponseHeader = header secConfig.Secure.CalculateOutboundApiId() } } +func associateResponseHeader(header http.Header) { + r := secConfig.Secure.GetRequest() + if r != nil { + r.ResponseHeader = header + r.ResponseContentType = header.Get("content-type") + } +} + /** * Handling for FastHttp framework */ @@ -519,14 +523,27 @@ func DissociateInboundRequest() { removeFuzzFile(tmpFiles) } +func traceResponseOperations() { + if !isAgentInitialized() { + return + } + + r := secConfig.Secure.GetRequest() + if r != nil { + checkSecureCookies(r.ResponseHeader) + xssCheck(r) + } +} + func checkSecureCookies(responseHeader http.Header) { - logger.Debugln("Response Header", responseHeader) if responseHeader != nil { + logger.Debugln("Verifying SecureCookies, response header", responseHeader) tmpRes := http.Response{Header: responseHeader} cookies := tmpRes.Cookies() var arg []map[string]interface{} + check := false for _, cookie := range cookies { - + check = true arg = append(arg, map[string]interface{}{ "name": cookie.Name, "isHttpOnly": cookie.HttpOnly, @@ -534,51 +551,46 @@ func checkSecureCookies(responseHeader http.Header) { "value": cookie.Value, }) } - secConfig.Secure.SendEvent("SECURE_COOKIE", "SECURE_COOKIE", arg) + if check { + secConfig.Secure.SendEvent("SECURE_COOKIE", "SECURE_COOKIE", arg) + } } } -func XssCheck() { - if !isAgentInitialized() { - return - } - r := secConfig.Secure.GetRequest() - checkSecureCookies(r.ResponseHeader) - if r != nil { - if r.ResponseBody != "" && !IsRXSSDisable() { +func xssCheck(r *secUtils.Info_req) { + if IsRxssEnabled() && r.ResponseBody != "" { - if r.ResponseContentType != "" && !secUtils.IsContentTypeSupported(r.ResponseContentType) { - SendLogMessage("No need to send RXSS event ContentType not supported for rxss event validation "+r.ResponseContentType, "XssCheck", "SEVERE") - logger.Debugln("No need to send RXSS event ContentType not supported for rxss event validation", r.ResponseContentType) - return - } + contentType := r.ResponseContentType + if !secUtils.IsContentTypeSupported(contentType) { + SendLogMessage(SKIP_RXSS_EVENT+contentType, "XssCheck", "SEVERE") + logger.Debugln(SKIP_RXSS_EVENT, contentType) + return + } - // Double check befor rxss event validation becouse in some case we don't have contentType in response header. - cType := http.DetectContentType([]byte(r.ResponseBody)) - if !secUtils.IsContentTypeSupported(cType) { - SendLogMessage("No need to send RXSS event ContentType not supported for rxss event validation "+cType, "XssCheck", "SEVERE") - logger.Debugln("No need to send RXSS event ContentType not supported for rxss event validation", cType) - return - } - if r.ResponseContentType == "" { - r.ResponseContentType = cType - } + // Double check befor rxss event validation becouse in some case we don't have contentType in response header. + cType := http.DetectContentType([]byte(r.ResponseBody)) + if !secUtils.IsContentTypeSupported(cType) { + SendLogMessage(SKIP_RXSS_EVENT+cType, "XssCheck", "SEVERE") + logger.Debugln(SKIP_RXSS_EVENT, cType) + return + } + if r.ResponseContentType == "" { + r.ResponseContentType = cType + } - out := secUtils.CheckForReflectedXSS(r) - logger.Debugln("CheckForReflectedXSS out value is : ", out) - - if len(out) == 0 && !secConfig.GlobalInfo.IsIASTEnable() { - logger.Debugln("No need to send xss event as not attack and dynamic scanning is false") - } else { - logger.Debugln("return value of reflected xss string : ", out) - var arg []string - arg = append(arg, out) - arg = append(arg, r.ResponseBody) - secConfig.Secure.SendEvent("REFLECTED_XSS", "REFLECTED_XSS", arg) - } - logger.Debugln("Called check for reflected XSS" + out) + out := secUtils.CheckForReflectedXSS(r) + logger.Debugln("RXSS check result: Out value set to ", out) + + if len(out) == 0 && !secConfig.GlobalInfo.IsIASTEnable() { + logger.Debugln("No need to send xss event as not attack and dynamic scanning is false") + } else { + var arg []string + arg = append(arg, out) + arg = append(arg, r.ResponseBody) + secConfig.Secure.SendEvent("REFLECTED_XSS", "REFLECTED_XSS", arg) } } + } /** @@ -763,7 +775,7 @@ func SendEvent(caseType string, data ...interface{}) interface{} { case "INBOUND": inboundcallHandler(data...) case "INBOUND_END": - XssCheck() + traceResponseOperations() DissociateInboundRequest() case "INBOUND_WRITE": httpresponseHandler(data...) @@ -884,34 +896,26 @@ func httpresponseHandler(data ...interface{}) { if len(data) < 2 { return } - res := data[0] - header := data[1] - if res == nil || !isAgentInitialized() { - return - } contentType := "" - responseHeader := http.Header{} - if hdr, ok := header.(http.Header); ok && hdr != nil { + if hdr, ok := data[1].(http.Header); ok && hdr != nil { contentType = hdr.Get("content-type") - responseHeader = hdr + associateResponseHeader(hdr) } - fmt.Println("responseHeader", data[1]) if contentType != "" && !secUtils.IsContentTypeSupported(contentType) { - logger.Debugln("No need to send RXSS event ContentType not supported for rxss event validation", contentType) + logger.Debugln("Skipping RXSS event transmission: Content type not supported for RXSS event validation", contentType) return } - if responseBody, ok := res.(string); ok { + if responseBody, ok := data[0].(string); ok { // Double check befor rxss event validation becouse in some case we don't have contentType in response header. - cType := http.DetectContentType([]byte(responseBody)) - if !secUtils.IsContentTypeSupported(cType) { - logger.Debugln("No need to send RXSS event ContentType not supported for rxss event validation", cType) + contentType = http.DetectContentType([]byte(responseBody)) + if !secUtils.IsContentTypeSupported(contentType) { + logger.Debugln("Skipping RXSS event transmission: Content type not supported for RXSS event validation", contentType) return } - - AssociateResponseBody(responseBody, contentType, responseHeader) + associateResponseBody(responseBody) } } @@ -919,16 +923,9 @@ func httpresponseHeader(data ...interface{}) { if len(data) < 1 { return } - header := data[0] - - contentType := "" - responseHeader := http.Header{} - if hdr, ok := header.(http.Header); ok && hdr != nil { - contentType = hdr.Get("content-type") - responseHeader = hdr + if hdr, ok := data[0].(http.Header); ok && hdr != nil { + associateResponseHeader(hdr) } - logger.Debugln("HTTP response header api called ", responseHeader) - AssociateResponseBody("", contentType, responseHeader) } func grpcRequestHandler(data ...interface{}) { diff --git a/security_intercept/intercept_utils.go b/security_intercept/intercept_utils.go index a9e8663..98c7211 100644 --- a/security_intercept/intercept_utils.go +++ b/security_intercept/intercept_utils.go @@ -26,8 +26,8 @@ func IsForceDisable() bool { func IsDisable() bool { return !secConfig.GlobalInfo.IsSecurityEnabled() } -func IsRXSSDisable() bool { - return !secConfig.GlobalInfo.IsRxssEnabled() +func IsRxssEnabled() bool { + return secConfig.GlobalInfo.IsRxssEnabled() } func RequestBodyReadLimit() int { From 19472ba370d53ce646dc770eef7f494d74e3e040 Mon Sep 17 00:00:00 2001 From: Aayush garg Date: Thu, 18 Jul 2024 10:55:52 +0530 Subject: [PATCH 11/30] Added websocket and fuzz connection stats --- security_config/global_config.go | 9 +- security_config/limits.go | 3 +- security_config/secure_metric.go | 123 +++++++++++++++++ security_event_generation/event_generation.go | 5 +- .../event_generation_utils.go | 26 ++-- security_handlers/fuzz_request_handler.go | 10 +- security_handlers/web_socket.go | 129 ++++++++---------- security_instrumentation/sec_httpfuzz.go | 5 + 8 files changed, 223 insertions(+), 87 deletions(-) create mode 100644 security_config/secure_metric.go diff --git a/security_config/global_config.go b/security_config/global_config.go index d41e373..050d155 100644 --- a/security_config/global_config.go +++ b/security_config/global_config.go @@ -34,6 +34,9 @@ type Info_struct struct { isForceDisable bool MetaData metaData + + WebSocketConnectionStats WebSocketConnectionStats + IastReplayRequest IastReplayRequest } func (info *Info_struct) GetCurrentPolicy() Policy { @@ -113,7 +116,10 @@ func (info *Info_struct) SetSecurityHomePath(path string) { } func (info *Info_struct) ValidatorServiceUrl() string { - return info.security.Validator_service_url + if info.security.Validator_service_url != "" { + return info.security.Validator_service_url + } + return ValidatorDefaultEndpoint } func (info *Info_struct) SetValidatorServiceUrl(path string) { info.security.Validator_service_url = path @@ -593,6 +599,7 @@ func InitDefaultConfig() { GlobalInfo.ApplicationInfo = runningApplicationInfo{} GlobalInfo.MetaData = metaData{} GlobalInfo.MetaData.linkingMetadata = map[string]string{} + GlobalInfo.WebSocketConnectionStats = WebSocketConnectionStats{} } diff --git a/security_config/limits.go b/security_config/limits.go index c0f85e8..80c83a4 100644 --- a/security_config/limits.go +++ b/security_config/limits.go @@ -1,5 +1,6 @@ package security_config const ( - MaxStackTraceFrames = 100 + MaxStackTraceFrames = 100 + ValidatorDefaultEndpoint = "wss://csec.nr-data.net" ) diff --git a/security_config/secure_metric.go b/security_config/secure_metric.go new file mode 100644 index 0000000..0d8e813 --- /dev/null +++ b/security_config/secure_metric.go @@ -0,0 +1,123 @@ +// Copyright 2024 New Relic Corporation. All rights reserved. +// SPDX-License-Identifier: New Relic Software License v1.0 + +package security_config + +import ( + "sync" +) + +type WebSocketConnectionStats struct { + MessagesSent uint64 `json:"messagesSent"` + MessagesReceived uint64 `json:"messagesReceived"` + ConnectionReconnected uint64 `json:"connectionReconnected"` + ConnectionFailure uint64 `json:"connectionFailure"` + ReceivedReconnectAtWill uint64 `json:"receivedReconnectAtWill"` + SendFailure uint64 `json:"sendFailure"` + sync.Mutex +} + +func (wcs *WebSocketConnectionStats) IncreaseMessagesSent() { + wcs.Lock() + defer wcs.Unlock() + wcs.MessagesSent++ +} + +func (wcs *WebSocketConnectionStats) IncreaseMessagesReceived() { + wcs.Lock() + defer wcs.Unlock() + wcs.MessagesReceived++ +} + +func (wcs *WebSocketConnectionStats) IncreaseConnectionReconnected() { + wcs.Lock() + defer wcs.Unlock() + wcs.ConnectionReconnected++ +} + +func (wcs *WebSocketConnectionStats) IncreaseConnectionFailure() { + wcs.Lock() + defer wcs.Unlock() + wcs.ConnectionFailure++ +} + +func (wcs *WebSocketConnectionStats) IncreaseReceivedReconnectAtWill() { + wcs.Lock() + defer wcs.Unlock() + wcs.ReceivedReconnectAtWill++ +} + +func (wcs *WebSocketConnectionStats) IncreaseSendFailure() { + wcs.Lock() + defer wcs.Unlock() + wcs.SendFailure++ +} +func (wcs *WebSocketConnectionStats) Reset() { + wcs.Lock() + defer wcs.Unlock() + wcs.SendFailure = 0 + wcs.ReceivedReconnectAtWill = 0 + wcs.ConnectionFailure = 0 + wcs.ConnectionReconnected = 0 + wcs.MessagesReceived = 0 + wcs.MessagesSent = 0 +} + +type IastReplayRequest struct { + ReceivedControlCommands uint64 `json:"receivedControlCommands"` + PendingControlCommands uint64 `json:"pendingControlCommands"` + ReplayRequestGenerated uint64 `json:"replayRequestGenerated"` + ReplayRequestExecuted uint64 `json:"replayRequestExecuted"` + ReplayRequestSucceeded uint64 `json:"replayRequestSucceeded"` + ReplayRequestFailed uint64 `json:"replayRequestFailed"` + ReplayRequestRejected uint64 `json:"replayRequestRejected"` + sync.Mutex +} + +func (iast *IastReplayRequest) IncreaseReceivedControlCommands() { + iast.Lock() + defer iast.Unlock() + iast.ReceivedControlCommands++ +} +func (iast *IastReplayRequest) IncreasePendingControlCommands() { + iast.Lock() + defer iast.Unlock() + iast.PendingControlCommands++ +} +func (iast *IastReplayRequest) IncreaseReplayRequestGenerated() { + iast.Lock() + defer iast.Unlock() + iast.ReplayRequestGenerated++ +} +func (iast *IastReplayRequest) IncreaseReplayRequestExecuted() { + iast.Lock() + defer iast.Unlock() + iast.ReplayRequestExecuted++ +} +func (iast *IastReplayRequest) IncreaseReplayRequestSucceeded() { + iast.Lock() + defer iast.Unlock() + iast.ReplayRequestSucceeded++ +} +func (iast *IastReplayRequest) IncreaseReplayRequestFailed() { + iast.Lock() + defer iast.Unlock() + iast.ReplayRequestFailed++ +} +func (iast *IastReplayRequest) IncreaseReplayRequestRejected() { + iast.Lock() + defer iast.Unlock() + iast.ReplayRequestRejected++ +} + +func (iast *IastReplayRequest) Reset() { + iast.Lock() + defer iast.Unlock() + iast.ReceivedControlCommands = 0 + iast.PendingControlCommands = 0 + iast.ReplayRequestGenerated = 0 + iast.ReplayRequestExecuted = 0 + iast.ReplayRequestSucceeded = 0 + iast.ReplayRequestFailed = 0 + iast.ReplayRequestRejected = 0 +} diff --git a/security_event_generation/event_generation.go b/security_event_generation/event_generation.go index 4281a68..1632aab 100644 --- a/security_event_generation/event_generation.go +++ b/security_event_generation/event_generation.go @@ -117,7 +117,8 @@ func SendSecHealthCheck() { hc.IastEventStats = *secConfig.GlobalInfo.EventData.GetIastEventStats() hc.RaspEventStats = *secConfig.GlobalInfo.EventData.GetRaspEventStats() hc.ExitEventStats = *secConfig.GlobalInfo.EventData.GetExitEventStats() - + hc.WebSocketConnectionStats = secConfig.GlobalInfo.WebSocketConnectionStats + hc.IastReplayRequest = secConfig.GlobalInfo.IastReplayRequest //set pending var threadPoolStats ThreadPoolStats if secConfig.SecureWS != nil { threadPoolStats.FuzzRequestQueueSize = secConfig.SecureWS.PendingFuzzTask() @@ -143,6 +144,8 @@ func SendSecHealthCheck() { secConfig.GlobalInfo.EventData.SetHttpRequestCount(0) secConfig.GlobalInfo.EventData.SetFuzzRequestCount(0) secConfig.GlobalInfo.EventData.ResetEventStats() + secConfig.GlobalInfo.WebSocketConnectionStats.Reset() + secConfig.GlobalInfo.IastReplayRequest.Reset() } func SendApplicationInfo() { diff --git a/security_event_generation/event_generation_utils.go b/security_event_generation/event_generation_utils.go index 395d0aa..b5f4af3 100644 --- a/security_event_generation/event_generation_utils.go +++ b/security_event_generation/event_generation_utils.go @@ -110,18 +110,20 @@ type userProvidedApplicationInfo struct { type healthcheck struct { ApplicationIdentifiers - EventType string `json:"eventType"` - ProtectedServer string `json:"protectedServer"` - EventDropCount uint64 `json:"eventDropCount"` - EventProcessed uint64 `json:"eventProcessed"` - EventSentCount uint64 `json:"eventSentCount"` - HTTPRequestCount uint64 `json:"httpRequestCount"` - Stats interface{} `json:"stats"` - ServiceStatus interface{} `json:"serviceStatus"` - IastEventStats secConfig.EventStats `json:"iastEventStats"` - RaspEventStats secConfig.EventStats `json:"raspEventStats"` - ExitEventStats secConfig.EventStats `json:"exitEventStats"` - ThreadPoolStats ThreadPoolStats `json:"threadPoolStats"` + EventType string `json:"eventType"` + ProtectedServer string `json:"protectedServer"` + EventDropCount uint64 `json:"eventDropCount"` + EventProcessed uint64 `json:"eventProcessed"` + EventSentCount uint64 `json:"eventSentCount"` + HTTPRequestCount uint64 `json:"httpRequestCount"` + Stats interface{} `json:"stats"` + ServiceStatus interface{} `json:"serviceStatus"` + IastEventStats secConfig.EventStats `json:"iastEventStats"` + RaspEventStats secConfig.EventStats `json:"raspEventStats"` + ExitEventStats secConfig.EventStats `json:"exitEventStats"` + ThreadPoolStats ThreadPoolStats `json:"threadPoolStats"` + WebSocketConnectionStats secConfig.WebSocketConnectionStats `json:"webSocketConnectionStats"` + IastReplayRequest secConfig.IastReplayRequest `json:"iastReplayRequest"` } type ThreadPoolStats struct { diff --git a/security_handlers/fuzz_request_handler.go b/security_handlers/fuzz_request_handler.go index 60ad808..c10991b 100644 --- a/security_handlers/fuzz_request_handler.go +++ b/security_handlers/fuzz_request_handler.go @@ -38,7 +38,9 @@ func (fTask *FuzzTask) Run() { eventGeneration.SendLogMessage("gRPC rest client not initialised", "security_handlers", "SEVERE") logger.Errorln("gRPC rest client not initialised") FuzzHandler.AppendCompletedRequestIds(fTask.requestID, "") + secConfig.GlobalInfo.IastReplayRequest.IncreaseReplayRequestRejected() } else { + secConfig.GlobalInfo.IastReplayRequest.IncreaseReplayRequestGenerated() FuzzHandler.grpsFuzzRestClient.ExecuteFuzzRequest(fTask.fuzzRequrestHandler, fTask.caseType, fTask.requestID) FuzzHandler.RemovePendingRequestIds(fTask.requestID) } @@ -47,7 +49,9 @@ func (fTask *FuzzTask) Run() { eventGeneration.SendLogMessage("http rest client not initialised", "security_handlers", "SEVERE") logger.Errorln("http rest client not initialised") FuzzHandler.AppendCompletedRequestIds(fTask.requestID, "") + secConfig.GlobalInfo.IastReplayRequest.IncreaseReplayRequestRejected() } else { + secConfig.GlobalInfo.IastReplayRequest.IncreaseReplayRequestGenerated() FuzzHandler.httpFuzzRestClient.ExecuteFuzzRequest(fTask.fuzzRequrestHandler, fTask.caseType, fTask.requestID) FuzzHandler.RemovePendingRequestIds(fTask.requestID) } @@ -55,6 +59,7 @@ func (fTask *FuzzTask) Run() { } func registerFuzzTask(kcc11 *FuzzRequrestHandler, caseType, requestID string) { + secConfig.GlobalInfo.IastReplayRequest.IncreaseReceivedControlCommands() task := &FuzzTask{kcc11, caseType, requestID} if FuzzHandler.threadPool == nil { initRestRequestThreadPool() @@ -67,7 +72,10 @@ func registerFuzzTask(kcc11 *FuzzRequrestHandler, caseType, requestID string) { } secConfig.GlobalInfo.EventData.IncreaseFuzzRequestCount() FuzzHandler.AppendPendingRequestIds(requestID) - FuzzHandler.threadPool.RegisterTask(task) + err := FuzzHandler.threadPool.RegisterTask(task) + if err != nil { + secConfig.GlobalInfo.IastReplayRequest.IncreaseReplayRequestRejected() + } FuzzHandler.SetLastFuzzRequestTime() } diff --git a/security_handlers/web_socket.go b/security_handlers/web_socket.go index 91d4185..a32978f 100644 --- a/security_handlers/web_socket.go +++ b/security_handlers/web_socket.go @@ -8,7 +8,6 @@ import ( "crypto/x509" "errors" "io" - "io/ioutil" "math/rand" "net/http" "os" @@ -25,8 +24,6 @@ import ( var logger = logging.GetLogger("wsclient") -const validatorDefaultEndpoint = "wss://csec.nr-data.net" - type eventStruct struct { event []byte eventType string @@ -57,29 +54,22 @@ func (ws *websocket) clean() { } } -func (ws *websocket) write(s eventStruct) bool { +func (ws *websocket) write(s eventStruct) error { ws.Lock() defer ws.Unlock() if !ws.isWsConnected() { - return false - } - err := ws.conn.WriteMessage(gorillaWS.TextMessage, s.event) - if err != nil { - logger.Errorln("Failed to send event over websocket : ", err.Error()) - increaseEventErrorCount(s.eventType) - return false + return errors.New("event discarded due to inactive WebSocket connection") } - increaseEventEventSentCount(s.eventType) - logger.Debugln("Event sent event over websocket done") - return true + return ws.conn.WriteMessage(gorillaWS.TextMessage, s.event) + } func (ws *websocket) read() ([]byte, error) { - if !ws.isWsConnected() { - return nil, errors.New("Failed to read CC over websocket ,ws is not connected") + return nil, errors.New("error reading control command: Inactive WebSocket connection") } _, message, err := ws.conn.ReadMessage() + if err != nil && err != io.EOF { return message, err } @@ -88,64 +78,40 @@ func (ws *websocket) read() ([]byte, error) { func (ws *websocket) makeConnection() (bool, bool) { if ws.isWsConnected() { - logger.Debugln("Websocket connection already initialized : Skip") - eventGeneration.SendApplicationInfo() // sending updated appinfo + logger.Debugln("webSocket connection already established, skipping new connection initialization") return true, false } ws.Lock() - validatorEndpoint := "" - if validatorEndpoint = secConfig.GlobalInfo.ValidatorServiceUrl(); validatorEndpoint == "" { - validatorEndpoint = validatorDefaultEndpoint - } - connectionHeader := getConnectionHeader() - printConnectionHeader(connectionHeader) var wsDialer = &gorillaWS.Dialer{ Proxy: http.ProxyFromEnvironment, HandshakeTimeout: 30 * time.Second, + TLSClientConfig: createCaCert(), } - caCertPool, err := x509.SystemCertPool() - if err != nil { - logger.Errorln(err) - ws.Unlock() - return false, false - } - caCert := []byte(secUtils.CaCert) - envCert := os.Getenv("NEW_RELIC_SECURITY_CA_BUNDLE_PATH") - if envCert != "" { - caCert, err = ioutil.ReadFile(envCert) - if err != nil { - logger.Errorln(err, "unsing default caCert") - caCert = []byte(secUtils.CaCert) - } - } - caCertPool.AppendCertsFromPEM(caCert) - wsDialer.TLSClientConfig = &tls.Config{RootCAs: caCertPool} + validatorEndpoint := secConfig.GlobalInfo.ValidatorServiceUrl() + conn, res, err := wsDialer.Dial(validatorEndpoint, getConnectionHeader()) - conn, res, err := wsDialer.Dial(validatorEndpoint, connectionHeader) if err != nil || conn == nil { - logging.PrintInitErrolog("Failed to connect Validator " + validatorEndpoint) - logger.Errorln("Failed to connect Validator : ", err, validatorEndpoint) + logging.PrintInitErrolog("Error connecting to Validator service: Connection failed " + err.Error() + validatorEndpoint) + logger.Errorln("Error connecting to Validator service: Connection failed ", err.Error(), validatorEndpoint) ws.conn = nil ws.Unlock() + secConfig.GlobalInfo.WebSocketConnectionStats.IncreaseConnectionFailure() if res != nil { - if body, _ := ioutil.ReadAll(res.Body); secUtils.CaseInsensitiveContains(string(body), "Invalid License Key!") { + if body, _ := io.ReadAll(res.Body); secUtils.CaseInsensitiveContains(string(body), "Invalid License Key!") { logger.Debugln("Invalid License Key No need to reconnect") return false, false } } return false, true } else { - logging.PrintInitlog("Connected to Prevent-Web service at : " + validatorEndpoint) - logger.Infoln("K.Reconnect init k.Conn successful", validatorEndpoint) + logger.Infoln("Security Agent is now ACTIVE for ", secConfig.GlobalInfo.ApplicationInfo.GetAppUUID()) + logging.EndStage("4", "Web socket connection to SaaS validator established successfully at "+validatorEndpoint) ws.conn = conn ws.Unlock() - logger.Infoln("Security Agent is now ACTIVE for ", secConfig.GlobalInfo.ApplicationInfo.GetAppUUID()) - logger.Infoln("!!! Websocket worker goroutine starting...") - logger.Infoln("Web socket connection to SaaS validator established successfully at", validatorEndpoint) - logging.EndStage("4", "Web socket connection to SaaS validator established successfully at "+validatorEndpoint) + logger.Infoln("Initializing WebSocket worker goroutine...") ws.flushWsController() go writeThread(ws) go readThread(ws) @@ -159,11 +125,10 @@ func (ws *websocket) reconnect() { if ws.isWsConnected() { break } - sleeptimeForReconnect := time.Duration(rand.Intn(10)+5) * time.Second - logger.Infoln("sleeping before reconnecting", sleeptimeForReconnect) + logger.Infoln("Sleeping for", sleeptimeForReconnect, " before attempting to reconnect") time.Sleep(sleeptimeForReconnect) - logger.Infoln("sleep end, retrying to connect with validator") + logger.Infoln("Sleep end, retrying connection with Validator service") if ws.isWsConnected() { break @@ -172,7 +137,6 @@ func (ws *websocket) reconnect() { if ok || !reconnect { return } - } } @@ -189,9 +153,9 @@ func (ws *websocket) connect() bool { return false } sleeptimeForReconnect := time.Duration(rand.Intn(10)+5) * time.Second - logger.Infoln("sleeping before reconnecting", sleeptimeForReconnect) + logger.Infoln("Sleeping for", sleeptimeForReconnect, " before attempting to reconnect") time.Sleep(sleeptimeForReconnect) - logger.Infoln("sleep end, retrying to connect with validator") + logger.Infoln("Sleep end, retrying connection with Validator service") } return true } @@ -277,14 +241,9 @@ func (ws *websocket) ReconnectAtWill() { } } secConfig.GlobalInfo.SetSecurityEnabled(false) - - // for ws.pendingEvent() > 0 { - // logger.Debugln("wait for event threadPool empty") - // time.Sleep(100 * time.Millisecond) - // } + secConfig.GlobalInfo.WebSocketConnectionStats.IncreaseReceivedReconnectAtWill() //reset ws connection - ws.closeWs() ws.reconnect() @@ -298,6 +257,7 @@ func (ws *websocket) ReconnectAtAgentRefresh() { return } secConfig.GlobalInfo.SetSecurityEnabled(false) + secConfig.GlobalInfo.WebSocketConnectionStats.IncreaseConnectionReconnected() //reset ws connection ws.closeWs() ws.reconnect() @@ -333,29 +293,35 @@ func InitializeWsConnecton() { // Read,Write Threads func writeThread(ws *websocket) { - logger.Info("Start ws write Thread") + logger.Info("WebSocket write thread started") ws.isWriteThreadRunning = true defer func() { ws.isWriteThreadRunning = false - logger.Info("Close ws write Thread") + logger.Info("WebSocket write thread stopped") }() for { select { case <-ws.writecontroller: return case event := <-ws.eventBuffer: - logger.Debugln("send event", len(ws.eventBuffer), " ", cap(ws.eventBuffer)) - ws.write(event) + err := ws.write(event) + if err != nil { + logger.Errorln("Failed to send event over websocket : ", err.Error()) + secConfig.GlobalInfo.WebSocketConnectionStats.IncreaseConnectionFailure() + } else { + logger.Debugln("Event sent event over websocket done") + secConfig.GlobalInfo.WebSocketConnectionStats.IncreaseMessagesSent() + } } } } func readThread(ws *websocket) { - logger.Info("Start ws read Thread") + logger.Info("WebSocket read thread started") ws.isReadThreadRunning = true defer func() { ws.isReadThreadRunning = false - logger.Info("Close ws read Thread") + logger.Info("WebSocket read thread stopped") }() for { buf, err := ws.read() @@ -371,6 +337,7 @@ func readThread(ws *websocket) { } } + secConfig.GlobalInfo.WebSocketConnectionStats.IncreaseMessagesReceived() err, _ = parseControlCommand(buf) if err != nil { eventGeneration.SendLogMessage("Unable to unmarshall control command"+err.Error(), "security_handlers", "SEVERE") @@ -381,7 +348,7 @@ func readThread(ws *websocket) { // Utils func getConnectionHeader() http.Header { - return http.Header{ + connectionHeader := http.Header{ "NR-CSEC-CONNECTION-TYPE": []string{"LANGUAGE_COLLECTOR"}, "NR-LICENSE-KEY": []string{secConfig.GlobalInfo.ApplicationInfo.GetApiAccessorToken()}, "NR-AGENT-RUN-TOKEN": []string{secConfig.GlobalInfo.MetaData.GetAgentRunId()}, @@ -396,7 +363,8 @@ func getConnectionHeader() http.Header { "NR-CSEC-ENTITY-GUID": []string{secConfig.GlobalInfo.MetaData.GetEntityGuid()}, "NR-CSEC-ENTITY-NAME": []string{secConfig.GlobalInfo.MetaData.GetEntityName()}, } - + printConnectionHeader(connectionHeader) + return connectionHeader } func printConnectionHeader(header http.Header) { @@ -449,6 +417,25 @@ func increaseEventErrorCount(eventType string) { } } +func createCaCert() *tls.Config { + caCertPool, err := x509.SystemCertPool() + if err != nil { + logger.Errorln("erro while createing CA certificate:", err) + return nil + } + caCert := []byte(secUtils.CaCert) + envCert := os.Getenv("NEW_RELIC_SECURITY_CA_BUNDLE_PATH") + if envCert != "" { + caCert, err = os.ReadFile(envCert) + if err != nil { + logger.Errorln(err, "using default CA certificate ") + caCert = []byte(secUtils.CaCert) + } + } + caCertPool.AppendCertsFromPEM(caCert) + return &tls.Config{RootCAs: caCertPool} +} + func (ws *websocket) flushWsController() { logger.Debugln("Flush flush Ws Controller Buffer", len(ws.readcontroller)) for ws.readcontroller != nil && len(ws.readcontroller) > 0 { diff --git a/security_instrumentation/sec_httpfuzz.go b/security_instrumentation/sec_httpfuzz.go index 1a234d3..6e868a2 100644 --- a/security_instrumentation/sec_httpfuzz.go +++ b/security_instrumentation/sec_httpfuzz.go @@ -55,6 +55,8 @@ func getHttpsClient() *http.Client { } func (httpFuzz SecHttpFuzz) ExecuteFuzzRequest(fuzzRequest *sechandler.FuzzRequrestHandler, caseType string, fuzzId string) { + secConfig.GlobalInfo.IastReplayRequest.IncreaseReplayRequestExecuted() + fuzzRequestID := fmt.Sprintf("%v", fuzzRequest.Headers[secIntercept.NR_CSEC_FUZZ_REQUEST_ID]) applicationPort := ":" + strconv.Itoa(fuzzRequest.ServerPort) fuzzRequestURL := secConfig.GlobalInfo.ApplicationInfo.GetServerIp() + applicationPort + fuzzRequest.Url @@ -76,6 +78,7 @@ func (httpFuzz SecHttpFuzz) ExecuteFuzzRequest(fuzzRequest *sechandler.FuzzRequr if req == nil || err != nil { logger.Infoln("ERROR: request type", fuzzRequest.Method, err) secIntercept.SendLogMessage(err.Error(), "security_instrumentation", "SEVERE") + secConfig.GlobalInfo.IastReplayRequest.IncreaseReplayRequestFailed() return } @@ -97,9 +100,11 @@ func (httpFuzz SecHttpFuzz) ExecuteFuzzRequest(fuzzRequest *sechandler.FuzzRequr response, err := fuzzRequestClient.Do(req) if err != nil { logger.Debugln("ERROR: fuzz request fail : ", fuzzRequestID, err.Error()) + secConfig.GlobalInfo.IastReplayRequest.IncreaseReplayRequestFailed() //secIntercept.SendLogMessage("fuzz request fail :"+err.Error(), "security_instrumentation", "SEVERE") } else { defer response.Body.Close() + secConfig.GlobalInfo.IastReplayRequest.IncreaseReplayRequestSucceeded() logger.Debugln("fuzz request successful : ", fuzzRequestID) } } From 4ec61609d9a00ebc0eb7ab46c5ba015688501d0b Mon Sep 17 00:00:00 2001 From: Aayush garg Date: Fri, 19 Jul 2024 17:23:07 +0530 Subject: [PATCH 12/30] Added event stats --- security_config/global_config.go | 125 +----------------- security_config/secure_metric.go | 91 +++++++++++++ security_event_generation/event_generation.go | 18 --- .../event_generation_utils.go | 8 -- security_handlers/fuzz_request_handler.go | 1 - security_handlers/web_socket.go | 57 ++------ security_implementation/implementation.go | 1 - 7 files changed, 109 insertions(+), 192 deletions(-) diff --git a/security_config/global_config.go b/security_config/global_config.go index 050d155..eeb6d4b 100644 --- a/security_config/global_config.go +++ b/security_config/global_config.go @@ -4,7 +4,6 @@ package security_config import ( - "math" "os" "strconv" "sync" @@ -19,7 +18,6 @@ var Secure secUtils.Secureiface var SecureWS secUtils.SecureWSiface type Info_struct struct { - EventData eventData ApiData *sync.Map EnvironmentInfo EnvironmentInfo ApplicationInfo runningApplicationInfo @@ -37,6 +35,7 @@ type Info_struct struct { WebSocketConnectionStats WebSocketConnectionStats IastReplayRequest IastReplayRequest + EventStats EventStats } func (info *Info_struct) GetCurrentPolicy() Policy { @@ -244,11 +243,9 @@ func (m *metaData) SetLinkingMetadata(value interface{}) { // EventData used to track number of request type eventData struct { - httpRequestCount uint64 - fuzzRequestCount uint64 - iastEventStats EventStats - raspEventStats EventStats - exitEventStats EventStats + iastEventStats EventStats + raspEventStats EventStats + exitEventStats EventStats sync.Mutex } @@ -277,70 +274,6 @@ func (e *eventData) GetExitEventStats() *EventStats { return &e.exitEventStats } -func (e *eventData) GetHttpRequestCount() uint64 { - var out uint64 - if e == nil { - return out - } - e.Lock() - defer e.Unlock() - return e.httpRequestCount -} - -func (e *eventData) SetHttpRequestCount(value uint64) { - if e == nil { - return - } - e.Lock() - defer e.Unlock() - e.httpRequestCount = value -} - -func (e *eventData) IncreaseHttpRequestCount() { - if e == nil { - return - } - e.Lock() - defer e.Unlock() - - e.httpRequestCount++ - if e.httpRequestCount == 0 { - e.httpRequestCount = math.MaxUint64 - } -} - -func (e *eventData) GetFuzzRequestCount() uint64 { - var out uint64 - if e == nil { - return out - } - e.Lock() - defer e.Unlock() - return e.fuzzRequestCount -} - -func (e *eventData) SetFuzzRequestCount(value uint64) { - if e == nil { - return - } - e.Lock() - defer e.Unlock() - e.fuzzRequestCount = value -} - -func (e *eventData) IncreaseFuzzRequestCount() { - if e == nil { - return - } - e.Lock() - defer e.Unlock() - - e.fuzzRequestCount++ - if e.fuzzRequestCount == 0 { - e.fuzzRequestCount = math.MaxUint64 - } -} - func (e *eventData) ResetEventStats() { if e == nil { return @@ -354,54 +287,6 @@ func (e *eventData) ResetEventStats() { } -type EventStats struct { - Processed uint64 `json:"processed"` - Sent uint64 `json:"sent"` - Rejected uint64 `json:"rejected"` - ErrorCount uint64 `json:"errorCount"` -} - -func (e *EventStats) IncreaseEventProcessedCount() { - if e == nil { - return - } - e.Processed++ - if e.Processed == 0 { - e.Processed = math.MaxUint64 - } -} - -func (e *EventStats) IncreaseEventSentCount() { - if e == nil { - return - } - e.Sent++ - if e.Sent == 0 { - e.Sent = math.MaxUint64 - } -} - -func (e *EventStats) IncreaseEventRejectedCount() { - if e == nil { - return - } - - e.Rejected++ - if e.Rejected == 0 { - e.Rejected = math.MaxUint64 - } -} - -func (e *EventStats) IncreaseEventErrorCount() { - if e == nil { - return - } - e.ErrorCount++ - if e.ErrorCount == 0 { - e.ErrorCount = math.MaxUint64 - } -} - type Urlmappings struct { Method string `json:"method"` Path string `json:"path"` @@ -593,7 +478,7 @@ type traceHooksApplied struct { func InitDefaultConfig() { //init default info - GlobalInfo.EventData = eventData{} + GlobalInfo.EventStats = EventStats{} GlobalInfo.InstrumentationData = Instrumentation{} GlobalInfo.EnvironmentInfo = EnvironmentInfo{} GlobalInfo.ApplicationInfo = runningApplicationInfo{} diff --git a/security_config/secure_metric.go b/security_config/secure_metric.go index 0d8e813..2c320e7 100644 --- a/security_config/secure_metric.go +++ b/security_config/secure_metric.go @@ -121,3 +121,94 @@ func (iast *IastReplayRequest) Reset() { iast.ReplayRequestFailed = 0 iast.ReplayRequestRejected = 0 } + +type stats struct { + Submitted uint64 `json:"submitted"` + Completed uint64 `json:"completed"` + Rejected uint64 `json:"rejected"` + ErrorCount uint64 `json:"error"` + sync.Mutex +} + +func (e *stats) IncreaseEventSubmittedCount() { + e.Lock() + defer e.Unlock() + e.Submitted++ +} + +func (e *stats) IncreaseCompletedCount() { + e.Lock() + defer e.Unlock() + e.Completed++ +} + +func (e *stats) IncreaseEventRejectedCount() { + e.Lock() + defer e.Unlock() + e.Rejected++ +} + +func (e *stats) IncreaseEventErrorCount() { + e.Lock() + defer e.Unlock() + e.ErrorCount++ +} + +func (e *stats) Reset() { + e.Lock() + defer e.Unlock() + e.ErrorCount = 0 + e.Rejected = 0 + e.Completed = 0 + e.Submitted = 0 + +} + +type EventStats struct { + EventSender stats `json:"eventSender"` // all events, urlMappings, ExitEvent, fuzzfail, app-info etc + IastEvents stats `json:"iastEvents"` // only iast event + Dispatcher stats `json:"dispatcher"` // RASP+IAST + LowSeverityEvents stats `json:"lowSeverityEvents"` //LowSeverityEvents only N/A + ExitEvents stats `json:"exitEvents"` // event event + +} + +func (e *EventStats) Reset() { + e.EventSender.Reset() + e.IastEvents.Reset() + e.Dispatcher.Reset() + e.LowSeverityEvents.Reset() + e.ExitEvents.Reset() +} + +func (e *EventStats) IncreaseEventSubmittedCount(eventType string) { + incrementStat(eventType, e.EventSender.IncreaseEventSubmittedCount, e.IastEvents.IncreaseEventSubmittedCount, e.Dispatcher.IncreaseEventSubmittedCount, e.LowSeverityEvents.IncreaseEventSubmittedCount) +} + +func (e *EventStats) IncreaseCompletedCount(eventType string) { + incrementStat(eventType, e.EventSender.IncreaseCompletedCount, e.IastEvents.IncreaseCompletedCount, e.Dispatcher.IncreaseCompletedCount, e.LowSeverityEvents.IncreaseCompletedCount) + +} + +func (e *EventStats) IncreaseEventRejectedCount(eventType string) { + incrementStat(eventType, e.EventSender.IncreaseEventRejectedCount, e.IastEvents.IncreaseEventRejectedCount, e.Dispatcher.IncreaseEventRejectedCount, e.LowSeverityEvents.IncreaseEventRejectedCount) + +} + +func (e *EventStats) IncreaseEventErrorCount(eventType string) { + incrementStat(eventType, e.EventSender.IncreaseEventErrorCount, e.IastEvents.IncreaseEventErrorCount, e.Dispatcher.IncreaseEventErrorCount, e.LowSeverityEvents.IncreaseEventErrorCount) + +} + +func incrementStat(eventType string, f1, f2, f3, f4 func()) { + switch eventType { + case "iastEvent": + f1() + case "raspEvent": + f2() + case "exitEvent": + f3() + case "LowSeverityEvents": + f4() + } +} diff --git a/security_event_generation/event_generation.go b/security_event_generation/event_generation.go index 1632aab..ff5ff6a 100644 --- a/security_event_generation/event_generation.go +++ b/security_event_generation/event_generation.go @@ -114,24 +114,9 @@ func SendSecHealthCheck() { hc.EventType = "sec_health_check_lc" hc.ApplicationIdentifiers = getApplicationIdentifiers("LAhealthcheck") hc.ProtectedServer = secConfig.GlobalInfo.ApplicationInfo.GetProtectedServer() - hc.IastEventStats = *secConfig.GlobalInfo.EventData.GetIastEventStats() - hc.RaspEventStats = *secConfig.GlobalInfo.EventData.GetRaspEventStats() - hc.ExitEventStats = *secConfig.GlobalInfo.EventData.GetExitEventStats() hc.WebSocketConnectionStats = secConfig.GlobalInfo.WebSocketConnectionStats hc.IastReplayRequest = secConfig.GlobalInfo.IastReplayRequest //set pending - var threadPoolStats ThreadPoolStats - if secConfig.SecureWS != nil { - threadPoolStats.FuzzRequestQueueSize = secConfig.SecureWS.PendingFuzzTask() - threadPoolStats.FuzzRequestCount = secConfig.GlobalInfo.EventData.GetFuzzRequestCount() - threadPoolStats.EventSendQueueSize = secConfig.SecureWS.PendingEvent() - } - hc.ThreadPoolStats = threadPoolStats - - hc.EventDropCount = hc.IastEventStats.Rejected + hc.RaspEventStats.Rejected + hc.ExitEventStats.Rejected - hc.EventProcessed = hc.IastEventStats.Processed + hc.RaspEventStats.Processed + hc.ExitEventStats.Processed - hc.EventSentCount = hc.IastEventStats.Sent + hc.RaspEventStats.Sent + hc.ExitEventStats.Sent - hc.HTTPRequestCount = secConfig.GlobalInfo.EventData.GetHttpRequestCount() stats := sysInfo.GetStats(secConfig.GlobalInfo.ApplicationInfo.GetPid(), secConfig.GlobalInfo.ApplicationInfo.GetBinaryPath()) hc.Stats = stats serviceStatus := getServiceStatus() @@ -141,9 +126,6 @@ func SendSecHealthCheck() { HcBuffer.ForceInsert(healthCheck) populateStatusLogs(serviceStatus, stats) - secConfig.GlobalInfo.EventData.SetHttpRequestCount(0) - secConfig.GlobalInfo.EventData.SetFuzzRequestCount(0) - secConfig.GlobalInfo.EventData.ResetEventStats() secConfig.GlobalInfo.WebSocketConnectionStats.Reset() secConfig.GlobalInfo.IastReplayRequest.Reset() } diff --git a/security_event_generation/event_generation_utils.go b/security_event_generation/event_generation_utils.go index b5f4af3..097bbe9 100644 --- a/security_event_generation/event_generation_utils.go +++ b/security_event_generation/event_generation_utils.go @@ -112,16 +112,8 @@ type healthcheck struct { ApplicationIdentifiers EventType string `json:"eventType"` ProtectedServer string `json:"protectedServer"` - EventDropCount uint64 `json:"eventDropCount"` - EventProcessed uint64 `json:"eventProcessed"` - EventSentCount uint64 `json:"eventSentCount"` - HTTPRequestCount uint64 `json:"httpRequestCount"` Stats interface{} `json:"stats"` ServiceStatus interface{} `json:"serviceStatus"` - IastEventStats secConfig.EventStats `json:"iastEventStats"` - RaspEventStats secConfig.EventStats `json:"raspEventStats"` - ExitEventStats secConfig.EventStats `json:"exitEventStats"` - ThreadPoolStats ThreadPoolStats `json:"threadPoolStats"` WebSocketConnectionStats secConfig.WebSocketConnectionStats `json:"webSocketConnectionStats"` IastReplayRequest secConfig.IastReplayRequest `json:"iastReplayRequest"` } diff --git a/security_handlers/fuzz_request_handler.go b/security_handlers/fuzz_request_handler.go index c10991b..ff6630e 100644 --- a/security_handlers/fuzz_request_handler.go +++ b/security_handlers/fuzz_request_handler.go @@ -70,7 +70,6 @@ func registerFuzzTask(kcc11 *FuzzRequrestHandler, caseType, requestID string) { printlogs := fmt.Sprintf("IAST Scan for API %s with ID : %s started.", kcc11.RequestURI, ids[0]) logger.Infoln(printlogs) } - secConfig.GlobalInfo.EventData.IncreaseFuzzRequestCount() FuzzHandler.AppendPendingRequestIds(requestID) err := FuzzHandler.threadPool.RegisterTask(task) if err != nil { diff --git a/security_handlers/web_socket.go b/security_handlers/web_socket.go index a32978f..b91d152 100644 --- a/security_handlers/web_socket.go +++ b/security_handlers/web_socket.go @@ -182,12 +182,14 @@ func (ws *websocket) closeWs() { } func (ws *websocket) RegisterEvent(s []byte, eventID string, eventType string) { - increaseEventProcessed(eventType) + + secConfig.GlobalInfo.EventStats.IncreaseEventSubmittedCount(eventType) if !ws.isWsConnected() && eventType != "LogMessage" { - increaseEventDropCount(eventType) + secConfig.GlobalInfo.EventStats.IncreaseEventRejectedCount(eventType) logger.Debugln("Drop event WS not connected or Reconnecting", len(ws.eventBuffer), cap(ws.eventBuffer)) return } + select { case ws.eventBuffer <- eventStruct{event: s, eventType: eventType}: logger.Debugln("Added EVENT", len(ws.eventBuffer), " ", cap(ws.eventBuffer)) @@ -195,20 +197,26 @@ func (ws *websocket) RegisterEvent(s []byte, eventID string, eventType string) { if eventID != "" { FuzzHandler.RemoveCompletedRequestIds(eventID) } - increaseEventDropCount(eventType) + secConfig.GlobalInfo.EventStats.IncreaseEventRejectedCount(eventType) logger.Errorln("cring.Full : Unable to add event to cring ", len(ws.eventBuffer), cap(ws.eventBuffer)) } } func (ws *websocket) SendPriorityEvent(s []byte) { + secConfig.GlobalInfo.EventStats.IncreaseEventSubmittedCount("PriorityEvent") if !ws.isWsConnected() { + secConfig.GlobalInfo.EventStats.IncreaseEventRejectedCount("PriorityEvent") logger.Debugln("Drop priority event WS not connected or Reconnecting", len(ws.eventBuffer), cap(ws.eventBuffer)) return } if logger.IsDebug() { logger.Debugln("priority event send", string(s)) } - ws.write(eventStruct{event: s, eventType: "PriorityEvent"}) + err := ws.write(eventStruct{event: s, eventType: "PriorityEvent"}) + if err != nil { + secConfig.GlobalInfo.EventStats.IncreaseEventErrorCount("PriorityEvent") + logger.Errorln("Failed to send event over websocket : ", err.Error()) + } } func (ws *websocket) GetStatus() bool { @@ -307,6 +315,7 @@ func writeThread(ws *websocket) { err := ws.write(event) if err != nil { logger.Errorln("Failed to send event over websocket : ", err.Error()) + secConfig.GlobalInfo.EventStats.IncreaseEventErrorCount(event.eventType) secConfig.GlobalInfo.WebSocketConnectionStats.IncreaseConnectionFailure() } else { logger.Debugln("Event sent event over websocket done") @@ -377,46 +386,6 @@ func printConnectionHeader(header http.Header) { } } -func increaseEventProcessed(eventType string) { - if eventType == "iastEvent" { - secConfig.GlobalInfo.EventData.GetIastEventStats().IncreaseEventProcessedCount() - } else if eventType == "raspEvent" { - secConfig.GlobalInfo.EventData.GetRaspEventStats().IncreaseEventProcessedCount() - } else if eventType == "exitEvent" { - secConfig.GlobalInfo.EventData.GetExitEventStats().IncreaseEventProcessedCount() - } -} - -func increaseEventDropCount(eventType string) { - if eventType == "iastEvent" { - secConfig.GlobalInfo.EventData.GetIastEventStats().IncreaseEventRejectedCount() - } else if eventType == "raspEvent" { - secConfig.GlobalInfo.EventData.GetRaspEventStats().IncreaseEventRejectedCount() - } else if eventType == "exitEvent" { - secConfig.GlobalInfo.EventData.GetExitEventStats().IncreaseEventRejectedCount() - } -} - -func increaseEventEventSentCount(eventType string) { - if eventType == "iastEvent" { - secConfig.GlobalInfo.EventData.GetIastEventStats().IncreaseEventSentCount() - } else if eventType == "raspEvent" { - secConfig.GlobalInfo.EventData.GetRaspEventStats().IncreaseEventSentCount() - } else if eventType == "exitEvent" { - secConfig.GlobalInfo.EventData.GetExitEventStats().IncreaseEventSentCount() - } -} - -func increaseEventErrorCount(eventType string) { - if eventType == "iastEvent" { - secConfig.GlobalInfo.EventData.GetIastEventStats().IncreaseEventErrorCount() - } else if eventType == "raspEvent" { - secConfig.GlobalInfo.EventData.GetRaspEventStats().IncreaseEventErrorCount() - } else if eventType == "exitEvent" { - secConfig.GlobalInfo.EventData.GetExitEventStats().IncreaseEventErrorCount() - } -} - func createCaCert() *tls.Config { caCertPool, err := x509.SystemCertPool() if err != nil { diff --git a/security_implementation/implementation.go b/security_implementation/implementation.go index 7386f3e..d0b4c38 100644 --- a/security_implementation/implementation.go +++ b/security_implementation/implementation.go @@ -67,7 +67,6 @@ func (k Secureimpl) AssociateInboundRequest(r *secUtils.Info_req) { if !isAgentReady() { return } - secConfig.GlobalInfo.EventData.IncreaseHttpRequestCount() goroutineID := getID() if r.Request.IsGRPC { data, err := grpcMap.Load(goroutineID) From b260af3a7b0ba2420782614dc1225dd895a4c423 Mon Sep 17 00:00:00 2001 From: Aayush garg Date: Wed, 24 Jul 2024 12:38:44 +0530 Subject: [PATCH 13/30] Added EventStats in HC --- internal/security_utils/security_interface.go | 2 +- security_config/global_config.go | 1 + security_config/secure_metric.go | 45 +++++++++++++++---- security_event_generation/event_generation.go | 8 +++- .../event_generation_utils.go | 1 + security_handlers/web_socket.go | 7 ++- 6 files changed, 51 insertions(+), 13 deletions(-) diff --git a/internal/security_utils/security_interface.go b/internal/security_utils/security_interface.go index dee13c6..9df0bd6 100644 --- a/internal/security_utils/security_interface.go +++ b/internal/security_utils/security_interface.go @@ -51,5 +51,5 @@ type SecureWSiface interface { SendPriorityEvent([]byte) AddCompletedRequests(string, string) PendingEvent() int - PendingFuzzTask() int + PendingFuzzTask() uint64 } diff --git a/security_config/global_config.go b/security_config/global_config.go index eeb6d4b..c1c9787 100644 --- a/security_config/global_config.go +++ b/security_config/global_config.go @@ -36,6 +36,7 @@ type Info_struct struct { WebSocketConnectionStats WebSocketConnectionStats IastReplayRequest IastReplayRequest EventStats EventStats + DroppedEvent DroppedEvent } func (info *Info_struct) GetCurrentPolicy() Policy { diff --git a/security_config/secure_metric.go b/security_config/secure_metric.go index 2c320e7..2937064 100644 --- a/security_config/secure_metric.go +++ b/security_config/secure_metric.go @@ -182,33 +182,60 @@ func (e *EventStats) Reset() { } func (e *EventStats) IncreaseEventSubmittedCount(eventType string) { - incrementStat(eventType, e.EventSender.IncreaseEventSubmittedCount, e.IastEvents.IncreaseEventSubmittedCount, e.Dispatcher.IncreaseEventSubmittedCount, e.LowSeverityEvents.IncreaseEventSubmittedCount) + incrementStat(eventType, e.EventSender.IncreaseEventSubmittedCount, e.IastEvents.IncreaseEventSubmittedCount, e.Dispatcher.IncreaseEventSubmittedCount, e.LowSeverityEvents.IncreaseEventSubmittedCount, e.ExitEvents.IncreaseEventSubmittedCount) } -func (e *EventStats) IncreaseCompletedCount(eventType string) { - incrementStat(eventType, e.EventSender.IncreaseCompletedCount, e.IastEvents.IncreaseCompletedCount, e.Dispatcher.IncreaseCompletedCount, e.LowSeverityEvents.IncreaseCompletedCount) +func (e *EventStats) IncreaseEventCompletedCount(eventType string) { + incrementStat(eventType, e.EventSender.IncreaseCompletedCount, e.IastEvents.IncreaseCompletedCount, e.Dispatcher.IncreaseCompletedCount, e.LowSeverityEvents.IncreaseCompletedCount, e.ExitEvents.IncreaseCompletedCount) } func (e *EventStats) IncreaseEventRejectedCount(eventType string) { - incrementStat(eventType, e.EventSender.IncreaseEventRejectedCount, e.IastEvents.IncreaseEventRejectedCount, e.Dispatcher.IncreaseEventRejectedCount, e.LowSeverityEvents.IncreaseEventRejectedCount) + incrementStat(eventType, e.EventSender.IncreaseEventRejectedCount, e.IastEvents.IncreaseEventRejectedCount, e.Dispatcher.IncreaseEventRejectedCount, e.LowSeverityEvents.IncreaseEventRejectedCount, e.ExitEvents.IncreaseEventRejectedCount) } func (e *EventStats) IncreaseEventErrorCount(eventType string) { - incrementStat(eventType, e.EventSender.IncreaseEventErrorCount, e.IastEvents.IncreaseEventErrorCount, e.Dispatcher.IncreaseEventErrorCount, e.LowSeverityEvents.IncreaseEventErrorCount) + incrementStat(eventType, e.EventSender.IncreaseEventErrorCount, e.IastEvents.IncreaseEventErrorCount, e.Dispatcher.IncreaseEventErrorCount, e.LowSeverityEvents.IncreaseEventErrorCount, e.ExitEvents.IncreaseEventErrorCount) } -func incrementStat(eventType string, f1, f2, f3, f4 func()) { +func incrementStat(eventType string, f1, f2, f3, f4, f5 func()) { + f1() switch eventType { case "iastEvent": - f1() - case "raspEvent": f2() - case "exitEvent": + f3() + case "raspEvent": f3() case "LowSeverityEvents": f4() + case "exitEvent": + f5() } } + +type DroppedEvent struct { + UnsupportedContentType uint64 `json:"unsupportedContentType"` + RxssDetectionDeactivated uint64 `json:"rxssDetectionDeactivated"` + sync.Mutex +} + +func (d *DroppedEvent) IncreaseUnsupportedContentTypeCount() { + d.Lock() + defer d.Unlock() + d.UnsupportedContentType++ +} + +func (d *DroppedEvent) IncreaseRxssDetectionDeactivatedCount() { + d.Lock() + defer d.Unlock() + d.RxssDetectionDeactivated++ +} + +func (d *DroppedEvent) Reset() { + d.Lock() + defer d.Unlock() + d.UnsupportedContentType = 0 + d.RxssDetectionDeactivated = 0 +} diff --git a/security_event_generation/event_generation.go b/security_event_generation/event_generation.go index ff5ff6a..82e8b46 100644 --- a/security_event_generation/event_generation.go +++ b/security_event_generation/event_generation.go @@ -115,7 +115,12 @@ func SendSecHealthCheck() { hc.ApplicationIdentifiers = getApplicationIdentifiers("LAhealthcheck") hc.ProtectedServer = secConfig.GlobalInfo.ApplicationInfo.GetProtectedServer() hc.WebSocketConnectionStats = secConfig.GlobalInfo.WebSocketConnectionStats - hc.IastReplayRequest = secConfig.GlobalInfo.IastReplayRequest //set pending + hc.IastReplayRequest = secConfig.GlobalInfo.IastReplayRequest + hc.EventStats = secConfig.GlobalInfo.EventStats + + if secConfig.SecureWS != nil { + hc.IastReplayRequest.PendingControlCommands = secConfig.SecureWS.PendingFuzzTask() + } stats := sysInfo.GetStats(secConfig.GlobalInfo.ApplicationInfo.GetPid(), secConfig.GlobalInfo.ApplicationInfo.GetBinaryPath()) hc.Stats = stats @@ -128,6 +133,7 @@ func SendSecHealthCheck() { secConfig.GlobalInfo.WebSocketConnectionStats.Reset() secConfig.GlobalInfo.IastReplayRequest.Reset() + secConfig.GlobalInfo.EventStats.Reset() } func SendApplicationInfo() { diff --git a/security_event_generation/event_generation_utils.go b/security_event_generation/event_generation_utils.go index 097bbe9..b420cd5 100644 --- a/security_event_generation/event_generation_utils.go +++ b/security_event_generation/event_generation_utils.go @@ -116,6 +116,7 @@ type healthcheck struct { ServiceStatus interface{} `json:"serviceStatus"` WebSocketConnectionStats secConfig.WebSocketConnectionStats `json:"webSocketConnectionStats"` IastReplayRequest secConfig.IastReplayRequest `json:"iastReplayRequest"` + EventStats secConfig.EventStats `json:"eventStats"` } type ThreadPoolStats struct { diff --git a/security_handlers/web_socket.go b/security_handlers/web_socket.go index b91d152..1b2aa24 100644 --- a/security_handlers/web_socket.go +++ b/security_handlers/web_socket.go @@ -216,6 +216,8 @@ func (ws *websocket) SendPriorityEvent(s []byte) { if err != nil { secConfig.GlobalInfo.EventStats.IncreaseEventErrorCount("PriorityEvent") logger.Errorln("Failed to send event over websocket : ", err.Error()) + } else { + secConfig.GlobalInfo.EventStats.IncreaseEventCompletedCount("PriorityEvent") } } @@ -280,11 +282,11 @@ func (ws *websocket) AddCompletedRequests(parentId, apiID string) { func (ws *websocket) PendingEvent() int { return ws.pendingEvent() } -func (ws *websocket) PendingFuzzTask() int { +func (ws *websocket) PendingFuzzTask() uint64 { if FuzzHandler.threadPool == nil { return 0 } - return FuzzHandler.threadPool.PendingTask() + return uint64(FuzzHandler.threadPool.PendingTask()) } func InitializeWsConnecton() { @@ -320,6 +322,7 @@ func writeThread(ws *websocket) { } else { logger.Debugln("Event sent event over websocket done") secConfig.GlobalInfo.WebSocketConnectionStats.IncreaseMessagesSent() + secConfig.GlobalInfo.EventStats.IncreaseEventCompletedCount(event.eventType) } } } From 545036ec8a407975e067bd41c720a1cc928047e7 Mon Sep 17 00:00:00 2001 From: Aayush garg Date: Mon, 29 Jul 2024 11:39:23 +0530 Subject: [PATCH 14/30] minor fix for headers --- security_intercept/intercept_utils.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/security_intercept/intercept_utils.go b/security_intercept/intercept_utils.go index 033ab85..bb336f5 100644 --- a/security_intercept/intercept_utils.go +++ b/security_intercept/intercept_utils.go @@ -227,7 +227,7 @@ type parameters struct { func parseFuzzRequestIdentifierHeader(requestHeaderVal string) (nrRequestIdentifier secUtils.NrRequestIdentifier) { nrRequestIdentifier.Raw = requestHeaderVal - if !secUtils.IsBlank(requestHeaderVal) { + if secUtils.IsBlank(requestHeaderVal) { return } data := strings.Split(requestHeaderVal, IAST_SEP) @@ -298,6 +298,7 @@ func ToOneValueMap(header map[string][]string) (filterHeader map[string]string) if header == nil { return } + filterHeader = map[string]string{} for k, v := range header { filterHeader[k] = strings.Join(v, ",") } From 4685ba87427497db17043094f5f49aee9fe63cd9 Mon Sep 17 00:00:00 2001 From: Aayush garg Date: Wed, 31 Jul 2024 10:22:32 +0530 Subject: [PATCH 15/30] Minor fix for critical messages --- security_event_generation/event_generation.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/security_event_generation/event_generation.go b/security_event_generation/event_generation.go index 821969b..dcb265d 100644 --- a/security_event_generation/event_generation.go +++ b/security_event_generation/event_generation.go @@ -475,7 +475,7 @@ func sendBufferLogMessage() { if secConfig.SecureWS != nil { tmp_event, ok := i.(LogMessage) if ok { - tmp_event.ApplicationUUID = secConfig.GlobalInfo.ApplicationInfo.GetAppUUID() + tmp_event.ApplicationIdentifiers = getApplicationIdentifiers("critical-messages") tmp_event.LinkingMetadata = secConfig.GlobalInfo.MetaData.GetLinkingMetadata() sendEvent(tmp_event, "", "LogMessage") } From 206821aa94e3ede834042455b75521f9f2989574 Mon Sep 17 00:00:00 2001 From: Aayush garg Date: Wed, 31 Jul 2024 10:25:13 +0530 Subject: [PATCH 16/30] fix for IsIASTEnable flag --- security_event_generation/event_generation.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/security_event_generation/event_generation.go b/security_event_generation/event_generation.go index 88f01ae..9b0b537 100644 --- a/security_event_generation/event_generation.go +++ b/security_event_generation/event_generation.go @@ -338,15 +338,17 @@ func SendVulnerableEvent(req *secUtils.Info_req, category, eventCategory string, requestIdentifier := req.RequestIdentifier - if secConfig.GlobalInfo.IsIASTEnable() && requestIdentifier.NrRequest { + if secConfig.GlobalInfo.IsIASTEnable() { tmp_event.IsIASTEnable = true - tmp_event.HTTPRequest.Route = "" - requestType = "iastEvent" - - if !secUtils.IsBlank(req.ParentID) { - apiId := requestIdentifier.APIRecordID - if apiId == vulnerabilityDetails.APIID { - (secConfig.SecureWS).AddCompletedRequests(req.ParentID, eventId) + if requestIdentifier.NrRequest { + tmp_event.HTTPRequest.Route = "" + requestType = "iastEvent" + + if !secUtils.IsBlank(req.ParentID) { + apiId := requestIdentifier.APIRecordID + if apiId == vulnerabilityDetails.APIID { + (secConfig.SecureWS).AddCompletedRequests(req.ParentID, eventId) + } } } } From 5aa5fdd271936be665bb889bd7fe073735d28707 Mon Sep 17 00:00:00 2001 From: Aayush garg Date: Wed, 31 Jul 2024 10:34:29 +0530 Subject: [PATCH 17/30] update json version --- internal/security_utils/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/security_utils/config.go b/internal/security_utils/config.go index bccdef1..c92a3c6 100644 --- a/internal/security_utils/config.go +++ b/internal/security_utils/config.go @@ -5,7 +5,7 @@ package security_utils const ( CollectorVersion = "1.3.0" - JsonVersion = "1.2.3" + JsonVersion = "1.2.5" CollectorType = "GOLANG" BuildNumber = "160" ) From c2c5f02bc409ffc122746cce2d4d82cf9c6486ad Mon Sep 17 00:00:00 2001 From: Aayush garg Date: Thu, 1 Aug 2024 13:15:01 +0530 Subject: [PATCH 18/30] fix for index out of range issue --- security_intercept/intercept_utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/security_intercept/intercept_utils.go b/security_intercept/intercept_utils.go index bb336f5..a95590d 100644 --- a/security_intercept/intercept_utils.go +++ b/security_intercept/intercept_utils.go @@ -232,7 +232,7 @@ func parseFuzzRequestIdentifierHeader(requestHeaderVal string) (nrRequestIdentif } data := strings.Split(requestHeaderVal, IAST_SEP) - if len(data) >= 8 { + if len(data) >= 6 { nrRequestIdentifier.APIRecordID = data[0] nrRequestIdentifier.RefID = data[1] nrRequestIdentifier.RefValue = data[2] @@ -242,7 +242,7 @@ func parseFuzzRequestIdentifierHeader(requestHeaderVal string) (nrRequestIdentif nrRequestIdentifier.NrRequest = true } - if !secUtils.IsAnyBlank(data[6], data[7]) { + if len(data) >= 8 && !secUtils.IsAnyBlank(data[6], data[7]) { encryptedData := data[6] hashVerifier := data[7] From 4b85df8516a906820fa5cc890af12a396f53a69a Mon Sep 17 00:00:00 2001 From: Aayush garg Date: Thu, 1 Aug 2024 16:15:31 +0530 Subject: [PATCH 19/30] fix for IsIASTRequest flag --- security_event_generation/event_generation.go | 1 + 1 file changed, 1 insertion(+) diff --git a/security_event_generation/event_generation.go b/security_event_generation/event_generation.go index 9b0b537..0ac1aab 100644 --- a/security_event_generation/event_generation.go +++ b/security_event_generation/event_generation.go @@ -343,6 +343,7 @@ func SendVulnerableEvent(req *secUtils.Info_req, category, eventCategory string, if requestIdentifier.NrRequest { tmp_event.HTTPRequest.Route = "" requestType = "iastEvent" + tmp_event.IsIASTRequest = true if !secUtils.IsBlank(req.ParentID) { apiId := requestIdentifier.APIRecordID From f54ef22487998fa0e708733728b84620162737b6 Mon Sep 17 00:00:00 2001 From: Aayush garg Date: Wed, 7 Aug 2024 11:00:35 +0530 Subject: [PATCH 20/30] update agent version for fuzz header changes --- internal/security_utils/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/security_utils/config.go b/internal/security_utils/config.go index bccdef1..bc16bac 100644 --- a/internal/security_utils/config.go +++ b/internal/security_utils/config.go @@ -5,7 +5,7 @@ package security_utils const ( CollectorVersion = "1.3.0" - JsonVersion = "1.2.3" + JsonVersion = "1.2.4" CollectorType = "GOLANG" BuildNumber = "160" ) From ae98fe2fbdc28f4f7ce7498634ab86765e29e076 Mon Sep 17 00:00:00 2001 From: Aayush garg Date: Thu, 8 Aug 2024 15:47:47 +0530 Subject: [PATCH 21/30] Added rxss fix for applicationxml content type --- internal/security_utils/xss_validation.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/security_utils/xss_validation.go b/internal/security_utils/xss_validation.go index 065322e..a090b02 100644 --- a/internal/security_utils/xss_validation.go +++ b/internal/security_utils/xss_validation.go @@ -324,7 +324,7 @@ func decodeRequestData(rq *Info_req) []string { processedData = append(processedData, string(value)) } case "application/xml": - processedData = append(processedData, body) + processedData = append(processedData, html.UnescapeString(body)) case "application/x-www-form-urlencoded": unescapedString, err := url.QueryUnescape(body) if err != nil { From 35a31259c15bf9b088ea445f1b9981e36db09a48 Mon Sep 17 00:00:00 2001 From: Aayush garg Date: Fri, 9 Aug 2024 10:32:07 +0530 Subject: [PATCH 22/30] remove empty content-type header --- security_instrumentation/sec_httpfuzz.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/security_instrumentation/sec_httpfuzz.go b/security_instrumentation/sec_httpfuzz.go index 1a234d3..a043a1b 100644 --- a/security_instrumentation/sec_httpfuzz.go +++ b/security_instrumentation/sec_httpfuzz.go @@ -88,7 +88,10 @@ func (httpFuzz SecHttpFuzz) ExecuteFuzzRequest(fuzzRequest *sechandler.FuzzRequr value := fmt.Sprintf("%v", headerValue) req.Header.Set(headerKey, value) } - req.Header.Set("Content-Type", fuzzRequest.ContentType) + + if !secUtils.IsBlank(fuzzRequest.ContentType) { + req.Header.Set("Content-Type", fuzzRequest.ContentType) + } req.Header.Set("nr-csec-parent-id", fuzzId) if fuzzRequestClient == nil { From 3888deaf19ecf865d9e2e8ca547084b35aca9dd7 Mon Sep 17 00:00:00 2001 From: Aayush garg Date: Mon, 12 Aug 2024 11:43:58 +0530 Subject: [PATCH 23/30] added unescape string for text/xml --- internal/security_utils/xss_validation.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/security_utils/xss_validation.go b/internal/security_utils/xss_validation.go index a090b02..401e7ad 100644 --- a/internal/security_utils/xss_validation.go +++ b/internal/security_utils/xss_validation.go @@ -325,6 +325,8 @@ func decodeRequestData(rq *Info_req) []string { } case "application/xml": processedData = append(processedData, html.UnescapeString(body)) + case "text/xml": + processedData = append(processedData, html.UnescapeString(body)) case "application/x-www-form-urlencoded": unescapedString, err := url.QueryUnescape(body) if err != nil { From 0a51fb06cf29be9ca60b8652518832804df2c4a3 Mon Sep 17 00:00:00 2001 From: Aayush garg Date: Tue, 13 Aug 2024 10:15:41 +0530 Subject: [PATCH 24/30] remove snapshot files. --- security_event_generation/event_generation.go | 2 +- security_initialization.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/security_event_generation/event_generation.go b/security_event_generation/event_generation.go index 077a352..6d955a7 100644 --- a/security_event_generation/event_generation.go +++ b/security_event_generation/event_generation.go @@ -140,7 +140,7 @@ func SendSecHealthCheck() { healthCheck, _ := sendPriorityEvent(hc) HcBuffer.ForceInsert(healthCheck) - populateStatusLogs(serviceStatus, stats) + // populateStatusLogs(serviceStatus, stats) secConfig.GlobalInfo.EventData.SetHttpRequestCount(0) secConfig.GlobalInfo.EventData.SetFuzzRequestCount(0) diff --git a/security_initialization.go b/security_initialization.go index a4806e4..088d40b 100644 --- a/security_initialization.go +++ b/security_initialization.go @@ -122,4 +122,8 @@ func initSecurityAgent(applicationName, licenseKey string, isDebugLog bool, secu initEnvironmentInfo() initApplicationInfo(applicationName) + logger.Infoln("Security HOME:", secConfig.GlobalInfo.SecurityHomePath()) + logger.Infoln("Agent location ", secConfig.GlobalInfo.EnvironmentInfo.Gopath) + logger.Infoln("Current working directory: ", filepath.Dir(secConfig.GlobalInfo.ApplicationInfo.GetBinaryPath())) + } From 5bc681a5d9b27359dbc0d18eda86a8a0931b7757 Mon Sep 17 00:00:00 2001 From: Aayush garg Date: Fri, 16 Aug 2024 11:26:23 +0530 Subject: [PATCH 25/30] minor fix for user metrics --- security_config/secure_metric.go | 22 +++++++++++++------- security_handlers/control_command_handler.go | 4 ++++ security_handlers/fuzz_request_handler.go | 1 - security_handlers/web_socket.go | 2 +- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/security_config/secure_metric.go b/security_config/secure_metric.go index 2937064..cc2f797 100644 --- a/security_config/secure_metric.go +++ b/security_config/secure_metric.go @@ -64,13 +64,14 @@ func (wcs *WebSocketConnectionStats) Reset() { } type IastReplayRequest struct { - ReceivedControlCommands uint64 `json:"receivedControlCommands"` - PendingControlCommands uint64 `json:"pendingControlCommands"` - ReplayRequestGenerated uint64 `json:"replayRequestGenerated"` - ReplayRequestExecuted uint64 `json:"replayRequestExecuted"` - ReplayRequestSucceeded uint64 `json:"replayRequestSucceeded"` - ReplayRequestFailed uint64 `json:"replayRequestFailed"` - ReplayRequestRejected uint64 `json:"replayRequestRejected"` + ReceivedControlCommands uint64 `json:"receivedControlCommands"` + PendingControlCommands uint64 `json:"pendingControlCommands"` + ReplayRequestGenerated uint64 `json:"replayRequestGenerated"` + ReplayRequestExecuted uint64 `json:"replayRequestExecuted"` + ReplayRequestSucceeded uint64 `json:"replayRequestSucceeded"` + ReplayRequestFailed uint64 `json:"replayRequestFailed"` + ReplayRequestRejected uint64 `json:"replayRequestRejected"` + ProcessedControlCommands uint64 `json:"processedControlCommands"` sync.Mutex } @@ -109,6 +110,11 @@ func (iast *IastReplayRequest) IncreaseReplayRequestRejected() { defer iast.Unlock() iast.ReplayRequestRejected++ } +func (iast *IastReplayRequest) IncreaseProcessedControlCommands() { + iast.Lock() + defer iast.Unlock() + iast.ProcessedControlCommands++ +} func (iast *IastReplayRequest) Reset() { iast.Lock() @@ -120,6 +126,7 @@ func (iast *IastReplayRequest) Reset() { iast.ReplayRequestSucceeded = 0 iast.ReplayRequestFailed = 0 iast.ReplayRequestRejected = 0 + iast.ProcessedControlCommands = 0 } type stats struct { @@ -212,6 +219,7 @@ func incrementStat(eventType string, f1, f2, f3, f4, f5 func()) { f4() case "exitEvent": f5() + f3() } } diff --git a/security_handlers/control_command_handler.go b/security_handlers/control_command_handler.go index a298fc9..72ff98f 100644 --- a/security_handlers/control_command_handler.go +++ b/security_handlers/control_command_handler.go @@ -57,10 +57,12 @@ func parseControlCommand(arg []byte) (error, bool) { case STARTUP_WELCOME_MSG: logger.Infoln("CC10", string(arg)) case FUZZ_REQUEST: + secConfig.GlobalInfo.IastReplayRequest.IncreaseReceivedControlCommands() if FuzzHandler.threadPool == nil { initRestRequestThreadPool() } if len(cc.Arguments) <= 1 { + secConfig.GlobalInfo.IastReplayRequest.IncreaseReplayRequestRejected() return errors.New("unable to process cc11, need minimum 2 arguments "), false } dsFilePath := filepath.Join(secConfig.GlobalInfo.SecurityHomePath(), "nr-security-home", "tmp") @@ -71,11 +73,13 @@ func parseControlCommand(arg []byte) (error, bool) { var cc11 FuzzRequrestHandler err = json.Unmarshal(arg, &cc11) if err != nil { + secConfig.GlobalInfo.IastReplayRequest.IncreaseReplayRequestRejected() eventGeneration.SendLogMessage("Unable to unmarshall cc11 : "+err.Error(), "parseControlCommand", "SEVERE") return errors.New("Unable to unmarshall cc11 : " + err.Error()), false } else { logger.Debugln("Fuzz request received :", cc.Id, cc.Arguments[1], string(arg)) logger.Debugln("will fuzz, parsedOK ..") + secConfig.GlobalInfo.IastReplayRequest.IncreaseProcessedControlCommands() cc11.MetaData = cc.ReflectedMetaData registerFuzzTask(&cc11, cc.Arguments[1], cc.Id) break diff --git a/security_handlers/fuzz_request_handler.go b/security_handlers/fuzz_request_handler.go index ff6630e..844c687 100644 --- a/security_handlers/fuzz_request_handler.go +++ b/security_handlers/fuzz_request_handler.go @@ -59,7 +59,6 @@ func (fTask *FuzzTask) Run() { } func registerFuzzTask(kcc11 *FuzzRequrestHandler, caseType, requestID string) { - secConfig.GlobalInfo.IastReplayRequest.IncreaseReceivedControlCommands() task := &FuzzTask{kcc11, caseType, requestID} if FuzzHandler.threadPool == nil { initRestRequestThreadPool() diff --git a/security_handlers/web_socket.go b/security_handlers/web_socket.go index 1b2aa24..c65e20c 100644 --- a/security_handlers/web_socket.go +++ b/security_handlers/web_socket.go @@ -121,6 +121,7 @@ func (ws *websocket) makeConnection() (bool, bool) { } func (ws *websocket) reconnect() { + secConfig.GlobalInfo.WebSocketConnectionStats.IncreaseConnectionReconnected() for { if ws.isWsConnected() { break @@ -267,7 +268,6 @@ func (ws *websocket) ReconnectAtAgentRefresh() { return } secConfig.GlobalInfo.SetSecurityEnabled(false) - secConfig.GlobalInfo.WebSocketConnectionStats.IncreaseConnectionReconnected() //reset ws connection ws.closeWs() ws.reconnect() From 2b312f1b55b1f37d4606eb7df8b6360cc0cdf37f Mon Sep 17 00:00:00 2001 From: Aayush garg Date: Fri, 16 Aug 2024 15:52:30 +0530 Subject: [PATCH 26/30] minor fix for connextion connect --- security_handlers/web_socket.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/security_handlers/web_socket.go b/security_handlers/web_socket.go index c65e20c..c7141c3 100644 --- a/security_handlers/web_socket.go +++ b/security_handlers/web_socket.go @@ -121,7 +121,6 @@ func (ws *websocket) makeConnection() (bool, bool) { } func (ws *websocket) reconnect() { - secConfig.GlobalInfo.WebSocketConnectionStats.IncreaseConnectionReconnected() for { if ws.isWsConnected() { break @@ -136,6 +135,7 @@ func (ws *websocket) reconnect() { } ok, reconnect := ws.makeConnection() if ok || !reconnect { + secConfig.GlobalInfo.WebSocketConnectionStats.IncreaseConnectionReconnected() return } } From 3120d3d279b66aa9191825921f65936b4a329940 Mon Sep 17 00:00:00 2001 From: Aayush garg Date: Fri, 23 Aug 2024 10:01:49 +0530 Subject: [PATCH 27/30] Added implementation for low severity unique event sending pipline --- internal/security_utils/security_interface.go | 2 ++ security_implementation/implementation.go | 25 +++++++++++++++++-- security_initialization.go | 2 +- security_intercept/intercept.go | 2 +- security_intercept/intercept_utils.go | 11 ++++++++ 5 files changed, 38 insertions(+), 4 deletions(-) diff --git a/internal/security_utils/security_interface.go b/internal/security_utils/security_interface.go index 9df0bd6..1a042d7 100644 --- a/internal/security_utils/security_interface.go +++ b/internal/security_utils/security_interface.go @@ -31,12 +31,14 @@ type Secureiface interface { DisassociateFastHttpData() GetFastHttpData() net.Conn SendEvent(caseType, eventCategory string, args interface{}) *EventTracker + SendLowSeverityEvent(caseType, eventCategory string, args interface{}) *EventTracker GetFuzzHeader() string GetTmpFiles() []string NewGoroutineLinker(interface{}) NewGoroutine() interface{} SendPanicEvent(string) Send5xxEvent(int) + CleanLowSeverityEvent() } // --------------------------------------------------- diff --git a/security_implementation/implementation.go b/security_implementation/implementation.go index 1aee19d..3b0020d 100644 --- a/security_implementation/implementation.go +++ b/security_implementation/implementation.go @@ -25,6 +25,7 @@ var logger = logging.GetLogger("impl") var grpcMap = sync.Map{} var fastHttpMap = sync.Map{} var requestMap = sync.Map{} +var lowSeverityEventMap = sync.Map{} const ( identity = "github.com/newrelic/csec-go-agent" @@ -267,7 +268,16 @@ func (k Secureimpl) SendEvent(caseType, eventCategory string, args interface{}) return nil } eventId := increaseCount() - return sendEvent(eventId, caseType, eventCategory, args) + return sendEvent(eventId, caseType, eventCategory, args, false) +} + +func (k Secureimpl) SendLowSeverityEvent(caseType, eventCategory string, args interface{}) *secUtils.EventTracker { + secConfig.AddEventDataToListener(secConfig.TestArgs{Parameters: fmt.Sprintf("%v", args), CaseType: caseType}) + if !isAgentReady() { + return nil + } + eventId := increaseCount() + return sendEvent(eventId, caseType, eventCategory, args, true) } func (k Secureimpl) SendExitEvent(event *secUtils.EventTracker) { @@ -286,7 +296,11 @@ func (k Secureimpl) SendExitEvent(event *secUtils.EventTracker) { eventGeneration.SendExitEvent(event, requestIdentifier) } -func sendEvent(eventId, caseType, eventCategory string, args interface{}) *secUtils.EventTracker { +func (k Secureimpl) CleanLowSeverityEvent() { + lowSeverityEventMap = sync.Map{} +} + +func sendEvent(eventId, caseType, eventCategory string, args interface{}, isLowSeverityEvent bool) *secUtils.EventTracker { id := getID() req := getRequest(id) if !isAgentReady() || (req == nil) { @@ -299,6 +313,13 @@ func sendEvent(eventId, caseType, eventCategory string, args interface{}) *secUt } else { vulnerabilityDetails = presentStack((*req).Request.Method, caseType) } + if isLowSeverityEvent { + if _, ok := lowSeverityEventMap.Load(vulnerabilityDetails.APIID); ok { + return nil + } else { + lowSeverityEventMap.Store(vulnerabilityDetails.APIID, 1) + } + } return eventGeneration.SendVulnerableEvent(req, caseType, eventCategory, args, vulnerabilityDetails, getEventID(eventId, id)) diff --git a/security_initialization.go b/security_initialization.go index 088d40b..bf14871 100644 --- a/security_initialization.go +++ b/security_initialization.go @@ -121,7 +121,7 @@ func initSecurityAgent(applicationName, licenseKey string, isDebugLog bool, secu logging.EndStage("1", "Security agent is starting") initEnvironmentInfo() initApplicationInfo(applicationName) - + go secIntercept.InitLowSeverityEventScheduler() logger.Infoln("Security HOME:", secConfig.GlobalInfo.SecurityHomePath()) logger.Infoln("Agent location ", secConfig.GlobalInfo.EnvironmentInfo.Gopath) logger.Infoln("Current working directory: ", filepath.Dir(secConfig.GlobalInfo.ApplicationInfo.GetBinaryPath())) diff --git a/security_intercept/intercept.go b/security_intercept/intercept.go index 581c862..561247d 100644 --- a/security_intercept/intercept.go +++ b/security_intercept/intercept.go @@ -464,7 +464,7 @@ func checkSecureCookies(responseHeader http.Header) { }) } if check { - secConfig.Secure.SendEvent("SECURE_COOKIE", "SECURE_COOKIE", arg) + secConfig.Secure.SendLowSeverityEvent("SECURE_COOKIE", "SECURE_COOKIE", arg) } } } diff --git a/security_intercept/intercept_utils.go b/security_intercept/intercept_utils.go index f9eee3d..4736d75 100644 --- a/security_intercept/intercept_utils.go +++ b/security_intercept/intercept_utils.go @@ -14,6 +14,7 @@ import ( "strconv" "strings" "syscall" + "time" logging "github.com/newrelic/csec-go-agent/internal/security_logs" secUtils "github.com/newrelic/csec-go-agent/internal/security_utils" @@ -304,3 +305,13 @@ func ToOneValueMap(header map[string][]string) (filterHeader map[string]string) } return } + +func InitLowSeverityEventScheduler() { + t := time.NewTicker(30 * time.Second) + for { + select { + case <-t.C: + secConfig.Secure.CleanLowSeverityEvent() + } + } +} From 1a45e6a7fa2457d9f1986f788e9e1f26097988b2 Mon Sep 17 00:00:00 2001 From: Aayush garg Date: Fri, 23 Aug 2024 11:24:16 +0530 Subject: [PATCH 28/30] Added route in apiid generation --- security_implementation/implementation.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/security_implementation/implementation.go b/security_implementation/implementation.go index 3b0020d..6de8731 100644 --- a/security_implementation/implementation.go +++ b/security_implementation/implementation.go @@ -311,7 +311,7 @@ func sendEvent(eventId, caseType, eventCategory string, args interface{}, isLowS if caseType == "REFLECTED_XSS" && (*req).VulnerabilityDetails.APIID != "" { vulnerabilityDetails = (*req).VulnerabilityDetails } else { - vulnerabilityDetails = presentStack((*req).Request.Method, caseType) + vulnerabilityDetails = presentStack((*req).Request.Method+"||"+(*req).Request.Route, caseType) } if isLowSeverityEvent { if _, ok := lowSeverityEventMap.Load(vulnerabilityDetails.APIID); ok { From 367f20cc8e73520583e0b36a643d8cde9c7b3a12 Mon Sep 17 00:00:00 2001 From: Aayush garg Date: Fri, 23 Aug 2024 12:12:39 +0530 Subject: [PATCH 29/30] Update low severity event scheduler time --- security_intercept/intercept_utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/security_intercept/intercept_utils.go b/security_intercept/intercept_utils.go index 4736d75..dadd246 100644 --- a/security_intercept/intercept_utils.go +++ b/security_intercept/intercept_utils.go @@ -307,7 +307,7 @@ func ToOneValueMap(header map[string][]string) (filterHeader map[string]string) } func InitLowSeverityEventScheduler() { - t := time.NewTicker(30 * time.Second) + t := time.NewTicker(30 * time.Minute) for { select { case <-t.C: From 068dafb92ec2b106388017a061e59dcb35483812 Mon Sep 17 00:00:00 2001 From: Aayush garg Date: Tue, 27 Aug 2024 09:23:13 +0530 Subject: [PATCH 30/30] added changelog for version v1.4.0 --- Changelog.md | 17 +++++++++++++++++ internal/security_utils/config.go | 4 ++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/Changelog.md b/Changelog.md index 1fde977..df92c35 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,22 @@ # Changelog +## [v1.4.0] - 2024-08-27 +### Features: +* Added new key identifiers to all event JSONs. +* Introduced detailed IAST scan metric reporting via HealthCheck for better insights. +* Added support for Secure Cookie event reporting to provide detailed vulnerability information. +* Added support for application/xml and text/xml content-types for RXSS vulnerability detection. +* Implemented a new mechanism to uniquely generate low severity events based on API ID, with a 30-minute time interval + +### Changes: +* Update IAST Header Parsing Minimum Expected Length Set to 8. +* Updated API ID generation to utilize both stacktrace and route information. +* Performed comprehensive code refactoring and cleanup for improved system efficiency and maintainability. +* Json Version bump to 1.2.5 + +### Deprecations: +* Status File Used for Debugging: This feature has been deprecated. All debugging capabilities have been moved to either Init Logging or Error Inbox and will be removed in a future agent release + ## [v1.3.0] - 2024-06-24 ### Features * Added functionality to report panics in user code. diff --git a/internal/security_utils/config.go b/internal/security_utils/config.go index d6541f2..eb2b803 100644 --- a/internal/security_utils/config.go +++ b/internal/security_utils/config.go @@ -4,8 +4,8 @@ package security_utils const ( - CollectorVersion = "1.3.0" - JsonVersion = "1.2.5" + CollectorVersion = "1.4.0" + JsonVersion = "1.2.5" CollectorType = "GOLANG" BuildNumber = "160" )