filter uagents

cmd/update-useragent/main.go

@@ -9,9 +9,11 @@ import (
 	"log"
 	"net/http"
 	"os"
+	"regexp"
 
 	"github.com/projectdiscovery/useragent"
 	sliceutil "github.com/projectdiscovery/utils/slice"
+	stringsutil "github.com/projectdiscovery/utils/strings"
 )
 
 var (
@@ -70,6 +72,17 @@ func getUserAgents() []*useragent.UserAgent {
 			continue
 		}
 		for _, userAgent := range whatismybrowserResponse.SearchResults.UserAgents {
+			if stringsutil.ContainsAnyI(userAgent.UserAgent,
+				"sleep", "timeout", "get-help", "start", "system",
+				"Functionize", "Edg/", "assetnote", "gzip", "norton",
+				"avast", "ccleaner", "avg", "xtpt", "promptmanager", "SznProhlizec",
+				"(0-0)", "unknown", "ddg", "opx", "RDDocuments", "Reeder",
+				"Topee", "TulipSAT", "Opendium", "DingTalk", "Gear",
+				"GoodAccess", "uacq") ||
+				hasUidSuffix(userAgent.UserAgent) {
+				continue
+			}
+
 			tags := buildTags(userAgent)
 			userAgent := &useragent.UserAgent{
 				Raw:  userAgent.UserAgent,
@@ -81,6 +94,11 @@ func getUserAgents() []*useragent.UserAgent {
 	return userAgents
 }
 
+func hasUidSuffix(s string) bool {
+	var guidPattern = regexp.MustCompile(`\(?[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}\)?$`)
+	return guidPattern.MatchString(s)
+}
+
 func buildTags(userAgent UserAgents) []string {
 	var tags []string
 	tags = append(tags, userAgent.Parse.SimpleSoftwareString)