Merge pull request #67 from newrelic/dev

Release v1.4.0

Changelog.md

@@ -1,5 +1,22 @@
 # Changelog
 
+## [v1.4.0] - 2024-08-27
+### Features:
+* Added new key identifiers to all event JSONs.
+* Introduced detailed IAST scan metric reporting via HealthCheck for better insights.
+* Added support for Secure Cookie event reporting to provide detailed vulnerability information.
+* Added support for application/xml and text/xml content-types for RXSS vulnerability detection.
+* Implemented a new mechanism to uniquely generate low severity events based on API ID, with a 30-minute time interval
+
+### Changes:
+* Update IAST Header Parsing Minimum Expected Length Set to 8.
+* Updated API ID generation to utilize both stacktrace and route information.
+* Performed comprehensive code refactoring and cleanup for improved system efficiency and maintainability.
+* Json Version bump to 1.2.5
+
+### Deprecations:
+* Status File Used for Debugging: This feature has been deprecated. All debugging capabilities have been moved to either Init Logging or Error Inbox and will be removed in a future agent release
+
 ## [v1.3.0] - 2024-06-24
 ### Features
 * Added functionality to report panics in user code.

internal/security_utils/config.go

@@ -4,8 +4,8 @@
 package security_utils
 
 const (
-	CollectorVersion = "1.3.0"
-	JsonVersion      = "1.2.3"
+	CollectorVersion = "1.4.0"
+	JsonVersion      = "1.2.5"
 	CollectorType    = "GOLANG"
 	BuildNumber      = "160"
 )

internal/security_utils/encryptorUtils.go

@@ -24,6 +24,7 @@ const (
 	ENCRYPTED_DATA_DECRYPTED_DATA                 = "Encrypted Data: %s, Decrypted data: %s"
 	ERROR_WHILE_GENERATING_REQUIRED_SALT_FROM_S_S = "Error while generating required salt from %s"
 	ERROR_WHILE_VERIFY_HASH_DATA                  = "Hash Data not macth %s: %s"
+	EMPTY_DATA                                    = "Empty decrypted data"
 )
 
 func Decrypt(password, encryptedData, hashVerifier string) (string, error) {
@@ -61,6 +62,10 @@ func Decrypt(password, encryptedData, hashVerifier string) (string, error) {
 	decrypted = removePadding(decrypted)
 	decryptedData := string(decrypted[OFFSET:])
 
+	if IsBlank(decryptedData) {
+		return "", fmt.Errorf(EMPTY_DATA)
+	}
+
 	if verifyHashData(hashVerifier, decryptedData) {
 		return decryptedData, nil
 	} else {

internal/security_utils/global_utils.go

@@ -11,13 +11,11 @@ type Info_req struct {
 	ResponseBody         string
 	ResponseHeader       http.Header
 	ResponseContentType  string
-	GrpcByte             [][]byte
 	GrpcBody             []interface{}
 	ReqTraceData         string
-	RequestIdentifier    string
+	RequestIdentifier    NrRequestIdentifier
 	Request              RequestInfo
 	VulnerabilityDetails VulnerabilityDetails
-	TmpFiles             []string
 	ReflectedMetaData    ReflectedMetaData
 	ParentID             string
 	BodyLimit            int
@@ -81,6 +79,18 @@ type VulnerabilityDetails struct {
 	Stacktrace     []string `json:"stacktrace"`
 }
 
+type NrRequestIdentifier struct {
+	Raw         string
+	RefID       string
+	RefValue    string
+	APIRecordID string
+	NrRequest   bool
+	NextStage   string
+	RecordIndex string
+	RefKey      string
+	TempFiles   []string
+}
+
 var CaCert = `
 -----BEGIN CERTIFICATE-----
 MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw

internal/security_utils/security_interface.go

@@ -30,13 +30,15 @@ type Secureiface interface {
 	AssociateFastHttpData(net.Conn)
 	DisassociateFastHttpData()
 	GetFastHttpData() net.Conn
-	SendEvent(category string, args interface{}) *EventTracker
+	SendEvent(caseType, eventCategory string, args interface{}) *EventTracker
+	SendLowSeverityEvent(caseType, eventCategory string, args interface{}) *EventTracker
 	GetFuzzHeader() string
 	GetTmpFiles() []string
 	NewGoroutineLinker(interface{})
 	NewGoroutine() interface{}
 	SendPanicEvent(string)
 	Send5xxEvent(int)
+	CleanLowSeverityEvent()
 }
 
 // ---------------------------------------------------
@@ -51,5 +53,5 @@ type SecureWSiface interface {
 	SendPriorityEvent([]byte)
 	AddCompletedRequests(string, string)
 	PendingEvent() int
-	PendingFuzzTask() int
+	PendingFuzzTask() uint64
 }

internal/security_utils/string_utils.go

@@ -101,3 +101,16 @@ func StringSHA256(f string) string {
 	hex.Encode(dst, sum[:])
 	return string(dst)
 }
+
+func IsBlank(in string) bool {
+	return in == ""
+}
+
+func IsAnyBlank(stringSequence ...string) bool {
+	for in := range stringSequence {
+		if IsBlank(stringSequence[in]) {
+			return true
+		}
+	}
+	return false
+}

internal/security_utils/xss_validation.go

@@ -324,7 +324,9 @@ func decodeRequestData(rq *Info_req) []string {
 				processedData = append(processedData, string(value))
 			}
 		case "application/xml":
-			processedData = append(processedData, body)
+			processedData = append(processedData, html.UnescapeString(body))
+		case "text/xml":
+			processedData = append(processedData, html.UnescapeString(body))
 		case "application/x-www-form-urlencoded":
 			unescapedString, err := url.QueryUnescape(body)
 			if err != nil {

security_config/global_config.go

@@ -4,7 +4,6 @@
 package security_config
 
 import (
-	"math"
 	"os"
 	"strconv"
 	"sync"
@@ -19,7 +18,6 @@ var Secure secUtils.Secureiface
 var SecureWS secUtils.SecureWSiface
 
 type Info_struct struct {
-	EventData           eventData
 	ApiData             *sync.Map
 	EnvironmentInfo     EnvironmentInfo
 	ApplicationInfo     runningApplicationInfo
@@ -34,6 +32,11 @@ type Info_struct struct {
 	isForceDisable bool
 
 	MetaData metaData
+
+	WebSocketConnectionStats WebSocketConnectionStats
+	IastReplayRequest        IastReplayRequest
+	EventStats               EventStats
+	DroppedEvent             DroppedEvent
 }
 
 func (info *Info_struct) GetCurrentPolicy() Policy {
@@ -113,7 +116,10 @@ func (info *Info_struct) SetSecurityHomePath(path string) {
 }
 
 func (info *Info_struct) ValidatorServiceUrl() string {
-	return info.security.Validator_service_url
+	if info.security.Validator_service_url != "" {
+		return info.security.Validator_service_url
+	}
+	return ValidatorDefaultEndpoint
 }
 func (info *Info_struct) SetValidatorServiceUrl(path string) {
 	info.security.Validator_service_url = path
@@ -238,11 +244,9 @@ func (m *metaData) SetLinkingMetadata(value interface{}) {
 
 // EventData used to track number of request
 type eventData struct {
-	httpRequestCount uint64
-	fuzzRequestCount uint64
-	iastEventStats   EventStats
-	raspEventStats   EventStats
-	exitEventStats   EventStats
+	iastEventStats EventStats
+	raspEventStats EventStats
+	exitEventStats EventStats
 	sync.Mutex
 }
 
@@ -271,70 +275,6 @@ func (e *eventData) GetExitEventStats() *EventStats {
 	return &e.exitEventStats
 }
 
-func (e *eventData) GetHttpRequestCount() uint64 {
-	var out uint64
-	if e == nil {
-		return out
-	}
-	e.Lock()
-	defer e.Unlock()
-	return e.httpRequestCount
-}
-
-func (e *eventData) SetHttpRequestCount(value uint64) {
-	if e == nil {
-		return
-	}
-	e.Lock()
-	defer e.Unlock()
-	e.httpRequestCount = value
-}
-
-func (e *eventData) IncreaseHttpRequestCount() {
-	if e == nil {
-		return
-	}
-	e.Lock()
-	defer e.Unlock()
-
-	e.httpRequestCount++
-	if e.httpRequestCount == 0 {
-		e.httpRequestCount = math.MaxUint64
-	}
-}
-
-func (e *eventData) GetFuzzRequestCount() uint64 {
-	var out uint64
-	if e == nil {
-		return out
-	}
-	e.Lock()
-	defer e.Unlock()
-	return e.fuzzRequestCount
-}
-
-func (e *eventData) SetFuzzRequestCount(value uint64) {
-	if e == nil {
-		return
-	}
-	e.Lock()
-	defer e.Unlock()
-	e.fuzzRequestCount = value
-}
-
-func (e *eventData) IncreaseFuzzRequestCount() {
-	if e == nil {
-		return
-	}
-	e.Lock()
-	defer e.Unlock()
-
-	e.fuzzRequestCount++
-	if e.fuzzRequestCount == 0 {
-		e.fuzzRequestCount = math.MaxUint64
-	}
-}
-
 func (e *eventData) ResetEventStats() {
 	if e == nil {
 		return
@@ -348,54 +288,6 @@ func (e *eventData) ResetEventStats() {
 
 }
 
-type EventStats struct {
-	Processed  uint64 `json:"processed"`
-	Sent       uint64 `json:"sent"`
-	Rejected   uint64 `json:"rejected"`
-	ErrorCount uint64 `json:"errorCount"`
-}
-
-func (e *EventStats) IncreaseEventProcessedCount() {
-	if e == nil {
-		return
-	}
-	e.Processed++
-	if e.Processed == 0 {
-		e.Processed = math.MaxUint64
-	}
-}
-
-func (e *EventStats) IncreaseEventSentCount() {
-	if e == nil {
-		return
-	}
-	e.Sent++
-	if e.Sent == 0 {
-		e.Sent = math.MaxUint64
-	}
-}
-
-func (e *EventStats) IncreaseEventRejectedCount() {
-	if e == nil {
-		return
-	}
-
-	e.Rejected++
-	if e.Rejected == 0 {
-		e.Rejected = math.MaxUint64
-	}
-}
-
-func (e *EventStats) IncreaseEventErrorCount() {
-	if e == nil {
-		return
-	}
-	e.ErrorCount++
-	if e.ErrorCount == 0 {
-		e.ErrorCount = math.MaxUint64
-	}
-}
-
 type Urlmappings struct {
 	Method  string `json:"method"`
 	Path    string `json:"path"`
@@ -587,12 +479,13 @@ type traceHooksApplied struct {
 
 func InitDefaultConfig() {
 	//init default info
-	GlobalInfo.EventData = eventData{}
+	GlobalInfo.EventStats = EventStats{}
 	GlobalInfo.InstrumentationData = Instrumentation{}
 	GlobalInfo.EnvironmentInfo = EnvironmentInfo{}
 	GlobalInfo.ApplicationInfo = runningApplicationInfo{}
 	GlobalInfo.MetaData = metaData{}
 	GlobalInfo.MetaData.linkingMetadata = map[string]string{}
+	GlobalInfo.WebSocketConnectionStats = WebSocketConnectionStats{}
 
 }
 

security_config/limits.go

@@ -1,5 +1,6 @@
 package security_config
 
 const (
-	MaxStackTraceFrames = 100
+	MaxStackTraceFrames      = 100
+	ValidatorDefaultEndpoint = "wss://csec.nr-data.net"
 )