Line-based File Scanning

v2/pkg/runner/options.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"strconv"
 	"strings"
 
 	"github.com/projectdiscovery/chaos-client/pkg/chaos"
@@ -48,6 +49,9 @@ type Options struct {
 	MaxEnumerationTime int                 // MaxEnumerationTime is the maximum amount of time in minutes to wait for enumeration
 	Domain             goflags.StringSlice // Domain is the domain to find subdomains for
 	DomainsFile        string              // DomainsFile is the file containing list of domains to find subdomains for
+	Lines              string              // Lines is the range of lines to be read from the file in the "start,end" format.
+	StartLine          int                 // StartLine is the number of the first line to be read from the file.
+	EndLine            int                 // EndLine is the number of the last line to be read from the file.
 	Output             io.Writer
 	OutputFile         string               // Output is the file to write found subdomains to.
 	OutputDirectory    string               // OutputDirectory is the directory to write results to in case list of domains is given
@@ -85,6 +89,7 @@ func ParseOptions() *Options {
 	flagSet.CreateGroup("input", "Input",
 		flagSet.StringSliceVarP(&options.Domain, "domain", "d", nil, "domains to find subdomains for", goflags.NormalizedStringSliceOptions),
 		flagSet.StringVarP(&options.DomainsFile, "list", "dL", "", "file containing list of domains for subdomain discovery"),
+		flagSet.StringVar(&options.Lines, "lines", "", "range of lines that will be read from the file [start,end]"),
 	)
 
 	flagSet.CreateGroup("source", "Source",
@@ -235,6 +240,32 @@ func listSources(options *Options) {
 	}
 }
 
+/*
+parseLinesArgument processes the "lines" argument in the format [start, end].
+If no error occurs, it sets options.StartLine and options.EndLine.
+*/
+func (options *Options) parseLinesArgument() error {
+	parts := strings.Split(options.Lines, ",")
+	if len(parts) != 2 {
+		return errors.New("invalid lines argument format, expected 'start,end'")
+	}
+
+	start, err := strconv.Atoi(strings.TrimSpace(parts[0]))
+	if err != nil {
+		return errors.New("invalid start line value, expected a number")
+	}
+	options.StartLine = start
+
+	end, err := strconv.Atoi(strings.TrimSpace(parts[1]))
+	if err != nil {
+		options.StartLine = 0
+		return errors.New("invalid end line value, expected a number")
+	}
+	options.EndLine = end
+
+	return nil
+}
+
 func (options *Options) preProcessDomains() {
 	for i, domain := range options.Domain {
 		options.Domain[i] = preprocessDomain(domain)

v2/pkg/runner/runner.go

@@ -93,8 +93,19 @@ func (r *Runner) RunEnumerationWithCtx(ctx context.Context) error {
 		if err != nil {
 			return err
 		}
-		err = r.EnumerateMultipleDomainsWithCtx(ctx, f, outputs)
-		f.Close()
+		defer f.Close()
+
+		var reader io.Reader = f
+
+		if r.options.StartLine != 0 && r.options.EndLine != 0 {
+			filteredReader, err := filterLinesByRange(f, r.options.StartLine, r.options.EndLine)
+			if err != nil {
+				return err
+			}
+			reader = filteredReader
+		}
+
+		err = r.EnumerateMultipleDomainsWithCtx(ctx, reader, outputs)
 		return err
 	}
 

v2/pkg/runner/util.go

@@ -1,8 +1,13 @@
 package runner
 
 import (
+	"bufio"
+	"bytes"
+	"errors"
+	"github.com/projectdiscovery/gologger"
 	fileutil "github.com/projectdiscovery/utils/file"
 	stringsutil "github.com/projectdiscovery/utils/strings"
+	"io"
 )
 
 func loadFromFile(file string) ([]string, error) {
@@ -30,3 +35,33 @@ func preprocessDomain(s string) string {
 		},
 	)
 }
+
+func filterLinesByRange(r io.Reader, start, end int) (io.Reader, error) {
+	scanner := bufio.NewScanner(r)
+	var buffer bytes.Buffer
+
+	lineNumber := 0
+	for scanner.Scan() {
+		lineNumber++
+		if lineNumber >= start && lineNumber <= end {
+			buffer.WriteString(scanner.Text() + "\n")
+		}
+		if lineNumber > end {
+			break
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+
+	if buffer.Len() == 0 {
+		return nil, errors.New("incorrect file read boundaries")
+	}
+
+	if lineNumber < end {
+		gologger.Info().Msgf("The provided upper bound (%d) is greater than the file size (%d). Scanning will continue to the end of the file.", end, lineNumber)
+	}
+
+	return &buffer, nil
+}

v2/pkg/runner/validate.go

@@ -65,6 +65,24 @@ func (options *Options) validateOptions() error {
 			return fmt.Errorf("invalid source %s specified in -rls flag", source)
 		}
 	}
+
+	if options.Lines != "" {
+		err := options.parseLinesArgument()
+		if err != nil {
+			return err
+		}
+	}
+
+	if options.DomainsFile == "" && options.Lines != "" {
+		return errors.New("cannot specify lines range without a file")
+	}
+	if options.StartLine > options.EndLine {
+		return errors.New("start line cannot be greater than end line")
+	}
+	if options.StartLine <= 0 || options.EndLine <= 0 {
+		return errors.New("start and end lines starts cannot be equal or less than zero")
+	}
+
 	return nil
 }
 func stripRegexString(val string) string {