feat: global matchers (#5701)

* feat: global matchers Signed-off-by: Dwi Siswanto <[email protected]> Co-authored-by: Ice3man543 <[email protected]> * feat(globalmatchers): make `Callback` as type Signed-off-by: Dwi Siswanto <[email protected]> * feat: update `passive` term to `(matchers-)static` Signed-off-by: Dwi Siswanto <[email protected]> * feat(globalmatchers): add `origin-template-*` event also use `Set` method instead of `maps.Clone` Signed-off-by: Dwi Siswanto <[email protected]> * feat: update `matchers-static` term to `global-matchers` Signed-off-by: Dwi Siswanto <[email protected]> * feat(globalmatchers): clone event before `operator.Execute` Signed-off-by: Dwi Siswanto <[email protected]> * fix(tmplexec): don't store `matched` on `global-matchers` templ This will end up generating 2 events from the same `scan.ScanContext` if one of the templates has `global-matchers` enabled. This way, non- `global-matchers` templates can enter the `writeFailureCallback` func to log failure output. Signed-off-by: Dwi Siswanto <[email protected]> * feat(globalmatchers): initializes `requests` on `New` Signed-off-by: Dwi Siswanto <[email protected]> * feat(globalmatchers): add `hasStorage` method Signed-off-by: Dwi Siswanto <[email protected]> * refactor(templates): rename global matchers checks method Signed-off-by: Dwi Siswanto <[email protected]> * fix(loader): handle nil `templates.Template` pointer Signed-off-by: Dwi Siswanto <[email protected]> --------- Signed-off-by: Dwi Siswanto <[email protected]> Co-authored-by: Ice3man543 <[email protected]>
projectdiscovery · Oct 14, 2024 · cc5c550 · cc5c550
1 parent aab2cad
commit cc5c550
Show file tree

Hide file tree

Showing 11 changed files with 170 additions and 2 deletions.
diff --git a/internal/runner/runner.go b/internal/runner/runner.go
@@ -48,6 +48,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/automaticscan"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/globalmatchers"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
@@ -475,6 +476,7 @@ func (r *Runner) RunEnumeration() error {
 		TemporaryDirectory:  r.tmpDir,
 		Parser:              r.parser,
 		FuzzParamsFrequency: fuzzFreqCache,
+		GlobalMatchers:      globalmatchers.New(),
 	}
 
 	if config.DefaultConfig.IsDebugArgEnabled(config.DebugExportURLPattern) {

diff --git a/pkg/catalog/loader/loader.go b/pkg/catalog/loader/loader.go
@@ -353,6 +353,20 @@ func (store *Store) areWorkflowOrTemplatesValid(filteredTemplatePaths map[string
 			if isParsingError("Error occurred parsing template %s: %s\n", templatePath, err) {
 				areTemplatesValid = false
 			}
+		} else if template == nil {
+			// NOTE(dwisiswant0): possibly global matchers template.
+			// This could definitely be handled better, for example by returning an
+			// `ErrGlobalMatchersTemplate` during `templates.Parse` and checking it
+			// with `errors.Is`.
+			//
+			// However, I’m not sure if every reference to it should be handled
+			// that way. Returning a `templates.Template` pointer would mean it’s
+			// an active template (sending requests), and adding a specific field
+			// like `isGlobalMatchers` in `templates.Template` (then checking it
+			// with a `*templates.Template.IsGlobalMatchersEnabled` method) would
+			// just introduce more unknown issues - like during template
+			// clustering, AFAIK.
+			continue
 		} else {
 			if existingTemplatePath, found := templateIDPathMap[template.ID]; !found {
 				templateIDPathMap[template.ID] = templatePath

diff --git a/pkg/output/format_screen.go b/pkg/output/format_screen.go
@@ -39,6 +39,11 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte {
 			}
 		}
 
+		if output.GlobalMatchers {
+			builder.WriteString("] [")
+			builder.WriteString(w.aurora.BrightMagenta("global").String())
+		}
+
 		builder.WriteString("] [")
 		builder.WriteString(w.aurora.BrightBlue(output.Type).String())
 		builder.WriteString("] ")

diff --git a/pkg/output/output.go b/pkg/output/output.go
@@ -184,6 +184,9 @@ type ResultEvent struct {
 	MatcherStatus bool `json:"matcher-status"`
 	// Lines is the line count for the specified match
 	Lines []int `json:"matched-line,omitempty"`
+	// GlobalMatchers identifies whether the matches was detected in the response
+	// of another template's result event
+	GlobalMatchers bool `json:"global-matchers,omitempty"`
 
 	// IssueTrackers is the metadata for issue trackers
 	IssueTrackers map[string]IssueTrackerMetadata `json:"issue_trackers,omitempty"`

diff --git a/pkg/protocols/common/globalmatchers/globalmatchers.go b/pkg/protocols/common/globalmatchers/globalmatchers.go
@@ -0,0 +1,84 @@
+package globalmatchers
+
+import (
+	"maps"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/model"
+	"github.com/projectdiscovery/nuclei/v3/pkg/operators"
+	"github.com/projectdiscovery/nuclei/v3/pkg/output"
+)
+
+// Storage is a struct that holds the global matchers
+type Storage struct {
+	requests []*Item
+}
+
+// Callback is called when a global matcher is matched.
+// It receives internal event & result of the operator execution.
+type Callback func(event output.InternalEvent, result *operators.Result)
+
+// Item is a struct that holds the global matchers
+// details for a template
+type Item struct {
+	TemplateID   string
+	TemplatePath string
+	TemplateInfo model.Info
+	Operators    []*operators.Operators
+}
+
+// New creates a new storage for global matchers
+func New() *Storage {
+	return &Storage{requests: make([]*Item, 0)}
+}
+
+// hasStorage checks if the Storage is initialized
+func (s *Storage) hasStorage() bool {
+	return s != nil
+}
+
+// AddOperator adds a new operator to the global matchers
+func (s *Storage) AddOperator(item *Item) {
+	if !s.hasStorage() {
+		return
+	}
+
+	s.requests = append(s.requests, item)
+}
+
+// HasMatchers returns true if we have global matchers
+func (s *Storage) HasMatchers() bool {
+	if !s.hasStorage() {
+		return false
+	}
+
+	return len(s.requests) > 0
+}
+
+// Match matches the global matchers against the response
+func (s *Storage) Match(
+	event output.InternalEvent,
+	matchFunc operators.MatchFunc,
+	extractFunc operators.ExtractFunc,
+	isDebug bool,
+	callback Callback,
+) {
+	for _, item := range s.requests {
+		for _, operator := range item.Operators {
+			newEvent := maps.Clone(event)
+			newEvent.Set("origin-template-id", event["template-id"])
+			newEvent.Set("origin-template-info", event["template-info"])
+			newEvent.Set("origin-template-path", event["template-path"])
+			newEvent.Set("template-id", item.TemplateID)
+			newEvent.Set("template-info", item.TemplateInfo)
+			newEvent.Set("template-path", item.TemplatePath)
+			newEvent.Set("global-matchers", true)
+
+			result, matched := operator.Execute(newEvent, matchFunc, extractFunc, isDebug)
+			if !matched {
+				continue
+			}
+
+			callback(newEvent, result)
+		}
+	}
+}
diff --git a/pkg/protocols/http/http.go b/pkg/protocols/http/http.go
@@ -223,6 +223,9 @@ type Request struct {
 	//  FuzzPreConditionOperator is the operator between multiple PreConditions for fuzzing Default is OR
 	FuzzPreConditionOperator string                 `yaml:"pre-condition-operator,omitempty" json:"pre-condition-operator,omitempty" jsonschema:"title=condition between the filters,description=Operator to use between multiple per-conditions,enum=and,enum=or"`
 	fuzzPreConditionOperator matchers.ConditionType `yaml:"-" json:"-"`
+	// description: |
+	//   GlobalMatchers marks matchers as static and applies globally to all result events from other templates
+	GlobalMatchers bool `yaml:"global-matchers,omitempty" json:"global-matchers,omitempty" jsonschema:"title=global matchers,description=marks matchers as static and applies globally to all result events from other templates"`
 }
 
 func (e Request) JSONSchemaExtend(schema *jsonschema.Schema) {

diff --git a/pkg/protocols/http/operators.go b/pkg/protocols/http/operators.go
@@ -166,6 +166,10 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
 	if types.ToString(wrapped.InternalEvent["path"]) != "" {
 		fields.Path = types.ToString(wrapped.InternalEvent["path"])
 	}
+	var isGlobalMatchers bool
+	if value, ok := wrapped.InternalEvent["global-matchers"]; ok {
+		isGlobalMatchers = value.(bool)
+	}
 	data := &output.ResultEvent{
 		TemplateID:       types.ToString(wrapped.InternalEvent["template-id"]),
 		TemplatePath:     types.ToString(wrapped.InternalEvent["template-path"]),
@@ -183,6 +187,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
 		Timestamp:        time.Now(),
 		MatcherStatus:    true,
 		IP:               fields.Ip,
+		GlobalMatchers:   isGlobalMatchers,
 		Request:          types.ToString(wrapped.InternalEvent["request"]),
 		Response:         request.truncateResponse(wrapped.InternalEvent["response"]),
 		CURLCommand:      types.ToString(wrapped.InternalEvent["curl-command"]),

diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go
@@ -973,13 +973,22 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 		// prune signature internal values if any
 		request.pruneSignatureInternalValues(generatedRequest.meta)
 
-		event := eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(generatedRequest.dynamicValues, finalEvent), request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) {
+		interimEvent := generators.MergeMaps(generatedRequest.dynamicValues, finalEvent)
+		isDebug := request.options.Options.Debug || request.options.Options.DebugResponse
+		event := eventcreator.CreateEventWithAdditionalOptions(request, interimEvent, isDebug, func(internalWrappedEvent *output.InternalWrappedEvent) {
 			internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta
 		})
+
 		if hasInteractMatchers {
 			event.UsesInteractsh = true
 		}
 
+		if request.options.GlobalMatchers.HasMatchers() {
+			request.options.GlobalMatchers.Match(interimEvent, request.Match, request.Extract, isDebug, func(event output.InternalEvent, result *operators.Result) {
+				callback(eventcreator.CreateEventWithOperatorResults(request, event, result))
+			})
+		}
+
 		// if requrlpattern is enabled, only then it is reflected in result event else it is empty string
 		// consult @Ice3man543 before changing this logic (context: vuln_hash)
 		if request.options.ExportReqURLPattern {

diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go
@@ -25,6 +25,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/progress"
 	"github.com/projectdiscovery/nuclei/v3/pkg/projectfile"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/globalmatchers"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/excludematchers"
@@ -126,6 +127,8 @@ type ExecutorOptions struct {
 	// ExportReqURLPattern exports the request URL pattern
 	// in ResultEvent it contains the exact url pattern (ex: {{BaseURL}}/{{randstr}}/xyz) used in the request
 	ExportReqURLPattern bool
+	// GlobalMatchers is the storage for global matchers with http passive templates
+	GlobalMatchers *globalmatchers.Storage
 }
 
 // todo: centralizing components is not feasible with current clogged architecture

diff --git a/pkg/templates/compile.go b/pkg/templates/compile.go
@@ -20,6 +20,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/globalmatchers"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/offlinehttp"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates/signer"
 	"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec"
@@ -81,6 +82,18 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Executo
 	if err != nil {
 		return nil, err
 	}
+	if template.isGlobalMatchersEnabled() {
+		item := &globalmatchers.Item{
+			TemplateID:   template.ID,
+			TemplatePath: filePath,
+			TemplateInfo: template.Info,
+		}
+		for _, request := range template.RequestsHTTP {
+			item.Operators = append(item.Operators, request.CompiledOperators)
+		}
+		options.GlobalMatchers.AddOperator(item)
+		return nil, nil
+	}
 	// Compile the workflow request
 	if len(template.Workflows) > 0 {
 		compiled := &template.Workflow
@@ -96,6 +109,25 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Executo
 	return template, nil
 }
 
+// isGlobalMatchersEnabled checks if any of requests in the template
+// have global matchers enabled. It iterates through all requests and
+// returns true if at least one request has global matchers enabled;
+// otherwise, it returns false.
+//
+// Note: This method only checks the `RequestsHTTP`
+// field of the template, which is specific to http-protocol-based
+// templates.
+//
+// TODO: support all protocols.
+func (template *Template) isGlobalMatchersEnabled() bool {
+	for _, request := range template.RequestsHTTP {
+		if request.GlobalMatchers {
+			return true
+		}
+	}
+	return false
+}
+
 // parseSelfContainedRequests parses the self contained template requests.
 func (template *Template) parseSelfContainedRequests() {
 	if template.Signature.Value.String() != "" {

diff --git a/pkg/tmplexec/exec.go b/pkg/tmplexec/exec.go
@@ -174,7 +174,15 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
 		if !event.HasOperatorResult() && event.InternalEvent != nil {
 			lastMatcherEvent = event
 		} else {
-			if writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient) {
+			var isGlobalMatchers bool
+			isGlobalMatchers, _ = event.InternalEvent["global-matchers"].(bool)
+			// NOTE(dwisiswant0): Don't store `matched` on a `global-matchers` template.
+			// This will end up generating 2 events from the same `scan.ScanContext` if
+			// one of the templates has `global-matchers` enabled. This way,
+			// non-`global-matchers` templates can enter the `writeFailureCallback`
+			// func to log failure output.
+			wr := writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient)
+			if wr && !isGlobalMatchers {
 				matched.Store(true)
 			} else {
 				lastMatcherEvent = event