diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 25901b1..b878080 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -19,25 +19,15 @@ jobs: with: go-version: '1.22.4' - - name: Check formatting - run: go fmt ./... - - - name: Run vet - run: go vet ./... - - - name: Verify go.mod - run: go mod tidy && git diff --exit-code - - name: Build run: go build -v ./... - - name: Test - run: go test -v -cover ./... - release: needs: build runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/v') + permissions: + contents: write steps: - uses: actions/checkout@v4 with: diff --git a/README.md b/README.md index 1317e0d..a8d0ed5 100644 --- a/README.md +++ b/README.md @@ -13,10 +13,8 @@ -A concurrent, non-blank line counter for source code directories, written in GO. - +A concurrent non-blank line counter for source code directories, written in Go. It recursively walks a directory, concurrently analyzes files, and reports the number of non-blank lines of code, grouped by file extension. - The tool is designed for performance, utilizing goroutines to process files in parallel. ## Installation @@ -24,33 +22,57 @@ The tool is designed for performance, utilizing goroutines to process files in p To install the `lines` command-line tool, ensure you have [Go](https://go.dev/doc/install) installed and configured, then run: ```shell -go install github.com/moderrek/lines/cmd/lines@latest +go install github.com/Moderrek/lines/cmd/lines@v1.3.0 ``` -This will download the source, compile it, and place the `lines` binary in your Go bin directory (`$GOPATH/bin` or `$HOME/go/in`). +This will download the source, compile it, and place the `lines` binary in your Go bin directory (`$GOPATH/bin` or `$HOME/go/bin`). The binary works on Linux and Windows, and the release assets are published for both platforms. + +If you want to use the library in another Go module, add it with: + +```shell +go get github.com/Moderrek/lines@v1.3.0 +``` + +Then import the package from `pkg/lines`: + +```go +import "github.com/Moderrek/lines/pkg/lines" +``` ## Usage -The `lines` command accepts the following flags: +The `lines` command accepts the following flags and targets: +```text +Usage: lines [options] [--dir PATH ...] [PATH ...] ``` -Usage: lines [options] +You can analyze multiple directories or files in one run by repeating `--dir` or passing positional arguments. + +``` Options: - -dir string - The directory to analyze (default ".") - -hidden - Include hidden files and directories in the analysis - -top uint - Show only the top N extensions by line count + -color + Force color output (e.g. when piping) -no-color - Disable colorized output + Disable color output + -help + Print the help message + -hidden + Allows to analyze hidden files + -jobs uint + Specifies the number of jobs -json Output results in JSON format + -top uint + Print the top N extensions + -verbose + Verbose output -version - Print version information and exit - -help - Show this help message and exit + Print the version +``` + +```shell +lines --dir ./cmd --dir ./pkg ./README.md ``` ### Example @@ -58,23 +80,23 @@ Options: To analyze the directory `~/projects/my-app` and display the top 5 extensions: ```shell -lines --dir ~/projects/my-app --top 5 +lines --top 5 ~/projects/my-app ``` To get the output in JSON format, which can be piped to other tools like `jq`: ```shell -lines --dir ~/projects/my-app --json +lines --json ~/projects/my-app ``` Example output (`--json`): ```json { - ".css": 1122, - ".go": 15230, - ".html": 4357, - ".js": 8828, - ".mod": 4980 + ".css": 1122, + ".go": 15230, + ".html": 4357, + ".js": 8828, + ".mod": 4980 } ``` @@ -83,10 +105,6 @@ Example output (`--json`): The core counting logic is available as a library. It can be imported into other Go projects. -```go -import "github.com/moderrek/lines/pkg/lines" -``` - ### Example ```go @@ -96,7 +114,7 @@ import ( "fmt" "log" - "github.com/moderrek/lines/pkg/lines" + "github.com/Moderrek/lines/pkg/lines" ) func main() { @@ -126,14 +144,14 @@ You can customize which directories and file extensions to ignore: ```go config := lines.Config{ IncludeHidden: false, - IgnoredDirs: []string{"node_modules", "vendor", ".git", "target", "dist"}, - IgnoredExtensions: []string{".exe", ".dll", ".jpg", ".png"}, + IgnoredDirs: lines.IgnoredDirSet("node_modules", ".git"), + IgnoredExtensions: lines.IgnoredExtensionSet("exe", ".env"), } counter := lines.NewCounter(config) result, err := counter.Run("./src") ``` -If `IgnoredDirs` or `IgnoredExtensions` are not provided, the library uses sensible defaults. +If `IgnoredDirs` or `IgnoredExtensions` are not provided, the library uses sensible defaults. The helper functions above are optional, but they make the config easier to read. ## Building from Source @@ -151,7 +169,23 @@ If `IgnoredDirs` or `IgnoredExtensions` are not provided, the library uses sensi ``` This will create a `lines` executable in the current directory. +### Releasing + +For v1.3.0, the intended distribution flow is: + +```shell +go install github.com/Moderrek/lines/cmd/lines@v1.3.0 +``` + +Or, if you are integrating the library into another module: + +```shell +go get github.com/Moderrek/lines@v1.3.0 +``` + +The GitHub release pipeline builds binaries for Linux and Windows so users without Go installed can download a ready-made executable. + ## License This project is licensed under the MIT License. -See the [LICENSE](LICENSE) file for details. +See the [LICENSE](./LICENSE) file for details. diff --git a/cmd/lines/cli.go b/cmd/lines/cli.go index c992c3e..5fd5481 100644 --- a/cmd/lines/cli.go +++ b/cmd/lines/cli.go @@ -3,10 +3,11 @@ package main import ( "flag" "io" + "strings" ) type cliOptions struct { - dir string + dirs multiStringFlag version bool help bool hidden bool @@ -14,21 +15,37 @@ type cliOptions struct { noColor bool color bool json bool + jobs uint + verbose bool +} + +type multiStringFlag []string + +func (m *multiStringFlag) String() string { + return strings.Join(*m, ",") +} + +func (m *multiStringFlag) Set(value string) error { + *m = append(*m, value) + return nil } func parseFlags(stderr io.Writer, args []string) (*cliOptions, *flag.FlagSet, error) { opts := &cliOptions{} - fs := flag.NewFlagSet("lines", flag.ContinueOnError) + fs := flag.NewFlagSet(ProgramName, flag.ContinueOnError) fs.SetOutput(stderr) - fs.StringVar(&opts.dir, "dir", ".", "The directory to analyze") + fs.Var(&opts.dirs, "dir", "The directories or files to analyze. Can be repeated.") fs.BoolVar(&opts.version, "version", false, "Print the version and exit") + fs.BoolVar(&opts.version, "v", false, "Print the version and exit") fs.BoolVar(&opts.help, "help", false, "Print the help message and exit") - fs.BoolVar(&opts.hidden, "hidden", false, "Allows to analize hidden files") + fs.BoolVar(&opts.hidden, "hidden", false, "Allows to analyze hidden files") fs.UintVar(&opts.top, "top", 0, "Print the top N extensions") + fs.UintVar(&opts.jobs, "jobs", 0, "Specifies the number of jobs") fs.BoolVar(&opts.noColor, "no-color", false, "Disable color output") fs.BoolVar(&opts.color, "color", false, "Force color output (e.g. when piping)") fs.BoolVar(&opts.json, "json", false, "Output results in JSON format") + fs.BoolVar(&opts.verbose, "verbose", false, "Verbose output") err := fs.Parse(args[1:]) if err != nil { diff --git a/cmd/lines/config.go b/cmd/lines/config.go new file mode 100644 index 0000000..cec46d0 --- /dev/null +++ b/cmd/lines/config.go @@ -0,0 +1,5 @@ +package main + +const ProgramName = "lines" +const Version = "v1.3.0" +const Author = "Tymon Wozniak @Moderrek" diff --git a/cmd/lines/main.go b/cmd/lines/main.go index 2742951..b28acba 100644 --- a/cmd/lines/main.go +++ b/cmd/lines/main.go @@ -7,7 +7,7 @@ import ( func main() { if err := run(os.Stdout, os.Stderr, os.Args); err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) + _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) os.Exit(1) } } diff --git a/cmd/lines/output.go b/cmd/lines/output.go index 42a6a41..c7d1272 100644 --- a/cmd/lines/output.go +++ b/cmd/lines/output.go @@ -6,8 +6,8 @@ import ( "io" "sort" + "github.com/Moderrek/lines/pkg/lines" "github.com/fatih/color" - "github.com/moderrek/lines/pkg/lines" ) func printJSONOutput(w io.Writer, result *lines.Result) error { @@ -15,16 +15,18 @@ func printJSONOutput(w io.Writer, result *lines.Result) error { if err != nil { return fmt.Errorf("error generating JSON: %w", err) } - fmt.Fprintln(w, string(jsonOutput)) - return nil + _, err = fmt.Fprintln(w, string(jsonOutput)) + return err } func printHumanOutput(w io.Writer, result *lines.Result, opts *cliOptions) { lineMap := result.LinesByExtension + sortedKeys := make([]string, 0, len(lineMap)) for key := range lineMap { sortedKeys = append(sortedKeys, key) } + sort.Slice(sortedKeys, func(i, j int) bool { return lineMap[sortedKeys[i]] > lineMap[sortedKeys[j]] }) @@ -36,13 +38,14 @@ func printHumanOutput(w io.Writer, result *lines.Result, opts *cliOptions) { if opts.top > 0 && uint(i) >= opts.top { break } + linesCount := lineMap[key] if linesCount == 0 { continue } extColor.Fprintf(w, "%s", key) - fmt.Fprint(w, " ") // Separator + fmt.Fprint(w, "\t") linesColor.Fprintf(w, "%d\n", linesCount) } } diff --git a/cmd/lines/run.go b/cmd/lines/run.go index bbd9318..4ec1ab1 100644 --- a/cmd/lines/run.go +++ b/cmd/lines/run.go @@ -6,9 +6,9 @@ import ( "os" "time" + "github.com/Moderrek/lines/pkg/lines" "github.com/fatih/color" "github.com/mattn/go-isatty" - "github.com/moderrek/lines/pkg/lines" ) func run(stdout, stderr io.Writer, args []string) error { @@ -17,44 +17,94 @@ func run(stdout, stderr io.Writer, args []string) error { return err } - isTerminal := isatty.IsTerminal(os.Stdout.Fd()) - useColor := (isTerminal || opts.color) && !opts.noColor + targets := make([]string, 0, len(opts.dirs)+len(fs.Args())) + targets = append(targets, opts.dirs...) + targets = append(targets, fs.Args()...) + if len(targets) == 0 { + targets = []string{"."} + } + + isStdoutTerminal := isatty.IsTerminal(os.Stdout.Fd()) + isStderrTerminal := isatty.IsTerminal(os.Stderr.Fd()) + + useColor := (isStdoutTerminal || opts.color) && !opts.noColor color.NoColor = !useColor if opts.version { - fmt.Fprintln(stdout, "Lines version 1.2.0 created by @Moderrek") + fmt.Printf("%s version %s created by %s\n", ProgramName, Version, Author) return nil } if opts.help { - fmt.Fprintln(stderr, "Usage: lines [options]") + fmt.Fprintf(stdout, "Usage: %s [options] [--dir PATH ...] [PATH ...]\n", ProgramName) + fmt.Fprintln(stdout, "") + fmt.Fprintln(stdout, "You can pass multiple directories or files using repeated --dir flags or positional arguments.") + fmt.Fprintln(stdout, "") + fs.SetOutput(stdout) fs.PrintDefaults() return nil } - startTime := time.Now() - if isTerminal && !opts.json { - fmt.Fprintf(stderr, "Analyzing.. %s\n\n", opts.dir) - } - config := lines.Config{ + Verbose: opts.verbose, IncludeHidden: opts.hidden, + NumWorkers: int(opts.jobs), } counter := lines.NewCounter(config) - result, err := counter.Run(opts.dir) + + stopProgress := make(chan struct{}) + + startTime := time.Now() + + showProgress := isStderrTerminal && !opts.json + if showProgress { + go startProgressReporter(stderr, stopProgress, startTime, counter) + } + + result, err := counter.Run(targets) if err != nil { return err } + if showProgress { + close(stopProgress) + reportProgress(stderr, startTime, counter) + fmt.Fprintf(stderr, "\n") + } + if opts.json { return printJSONOutput(stdout, result) } printHumanOutput(stdout, result, opts) - if isTerminal { - color.New(color.FgGreen).Fprintf(stderr, "\nTime taken: %v to analyze files\n", time.Since(startTime)) + return nil +} + +func startProgressReporter(w io.Writer, stop chan struct{}, startTime time.Time, counter *lines.Counter) { + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-stop: + return + case <-ticker.C: + reportProgress(w, startTime, counter) + } } +} - return nil +func reportProgress(w io.Writer, startTime time.Time, counter *lines.Counter) { + processed := counter.FilesProcessed.Load() + found := counter.FilesFound.Load() + + elapsed := time.Since(startTime) + + inQueue := found - processed + if inQueue < 0 { + inQueue = 0 + } + + fmt.Fprintf(w, "\r\033[KProcessed: %d | In Queue: %d | Elapsed: %v", processed, inQueue, elapsed) } diff --git a/go.mod b/go.mod index 02edd52..2bfaef6 100644 --- a/go.mod +++ b/go.mod @@ -1,12 +1,13 @@ -module github.com/moderrek/lines +module github.com/Moderrek/lines -go 1.22.4 +go 1.25.0 -require github.com/orcaman/concurrent-map/v2 v2.0.1 +require ( + github.com/fatih/color v1.19.0 + github.com/mattn/go-isatty v0.0.24 +) require ( - github.com/fatih/color v1.17.0 - github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.20 - golang.org/x/sys v0.18.0 // indirect + github.com/mattn/go-colorable v0.1.15 // indirect + golang.org/x/sys v0.47.0 // indirect ) diff --git a/go.sum b/go.sum index 91faaae..2e098fe 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,8 @@ -github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= -github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c= -github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= +github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= +github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= +github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= diff --git a/pkg/lines/analysis.go b/pkg/lines/analysis.go new file mode 100644 index 0000000..ed33ec4 --- /dev/null +++ b/pkg/lines/analysis.go @@ -0,0 +1,93 @@ +package lines + +import ( + "bufio" + "bytes" + "fmt" + "io" + "os" + "path/filepath" +) + +// analyzeFile reads a file and counts non-blank, non-comment lines. +// Lines starting with '//', '#' or '--' are treated as comments and skipped. +// path specifies the file path to count non-blank lines. +// bufferInitialSize specifies the initial scanner buffer size. +// bufferMaxSize specifies the maximum scanner buffer size. +func analyzeFile(path string, readerInitialBufferSize int) (int, error) { + file, err := os.Open(path) + if err != nil { + return 0, fmt.Errorf("failed to analyze %q: %v", filepath.ToSlash(path), err) + } + + defer func() { + if err := file.Close(); err != nil { + fmt.Fprintf(os.Stderr, "warn: failed to close file %q: %v\n", filepath.ToSlash(path), err) + } + }() + + reader := newReader(file, readerInitialBufferSize) + lineCount := 0 + isInsideLongLine := false + + for { + line, err := readLine(reader, path, &isInsideLongLine) + if err != nil { + if err == io.EOF { + break + } + return 0, fmt.Errorf("failed to analyze %q: readLine: %v\n", filepath.ToSlash(path), err) + } + + if len(line) == 0 { + continue + } + + if isCommentLine(line) { + continue + } + + lineCount++ + } + + return lineCount, nil +} + +func readLine(reader *bufio.Reader, path string, isInsideLongLine *bool) ([]byte, error) { + line, isPrefix, err := reader.ReadLine() + if err != nil { + if err == io.EOF { + return nil, err + } + return nil, fmt.Errorf("failed to analyze %q: %v", filepath.ToSlash(path), err) + } + + if *isInsideLongLine { + if !isPrefix { + *isInsideLongLine = false + } + return nil, nil + } + + if isPrefix { + *isInsideLongLine = true + } + + trimmedLine := bytes.TrimSpace(line) + return trimmedLine, nil +} + +func isCommentLine(line []byte) bool { + doubleSlash := []byte("//") + hash := []byte("#") + doubleDash := []byte("--") + + return bytes.HasPrefix(line, doubleSlash) || bytes.HasPrefix(line, hash) || bytes.HasPrefix(line, doubleDash) +} + +func newReader(file *os.File, initialBufferSize int) *bufio.Reader { + if initialBufferSize > 0 { + return bufio.NewReaderSize(file, initialBufferSize) + } + return bufio.NewReader(file) +} diff --git a/pkg/lines/config.go b/pkg/lines/config.go new file mode 100644 index 0000000..b927f4b --- /dev/null +++ b/pkg/lines/config.go @@ -0,0 +1,21 @@ +package lines + +// TODO: consider storing ignored directories and extensions as slices instead of maps. +// We can convert them to maps during initialization. +// This would make the configuration more user-friendly while still allowing for efficient lookups during analysis. + +// Config holds settings for the line counting process. +type Config struct { + // IncludeHidden analyzes hidden files and directories (starting with '.'). + IncludeHidden bool + // IgnoredDirs are directories to skip during analysis. Defaults to ["node_modules", "vendor", ".git", "target"]. + IgnoredDirs map[string]struct{} + // IgnoredExtensions are file extensions to skip. Defaults to common binary and media formats. + IgnoredExtensions map[string]struct{} + // ReaderInitialBufferSize is the initial buffer size for the scanner. + ReaderInitialBufferSize int + // NumWorkers is the number of workers to use for file analysis. + NumWorkers int + // Verbose enables logging while file analysis. + Verbose bool +} diff --git a/pkg/lines/counter.go b/pkg/lines/counter.go new file mode 100644 index 0000000..1019cf8 --- /dev/null +++ b/pkg/lines/counter.go @@ -0,0 +1,170 @@ +package lines + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "sync" + "sync/atomic" +) + +// TODO: add handle log function. + +// Counter analyzes directories and counts non-blank lines of code. +type Counter struct { + Config Config + + linesLock sync.Mutex + lines map[string]int + + workers sync.WaitGroup + filesToAnalyze chan string + + FilesFound atomic.Int64 + FilesProcessed atomic.Int64 + + isWorking bool +} + +// NewCounter creates a new Counter with the given configuration. +// If IgnoredDirs or IgnoredExtensions are empty, sensible defaults are used. +func NewCounter(config Config) *Counter { + return &Counter{ + Config: configWithDefaults(config), + linesLock: sync.Mutex{}, + lines: make(map[string]int), + workers: sync.WaitGroup{}, + filesToAnalyze: nil, + FilesFound: atomic.Int64{}, + FilesProcessed: atomic.Int64{}, + } +} + +func configWithDefaults(config Config) Config { + if len(config.IgnoredDirs) == 0 { + config.IgnoredDirs = DefaultIgnoredDirs() + } + if len(config.IgnoredExtensions) == 0 { + config.IgnoredExtensions = DefaultIgnoredExtensions() + } + if config.ReaderInitialBufferSize == 0 { + config.ReaderInitialBufferSize = 64 * 1024 + } + if config.NumWorkers <= 0 { + config.NumWorkers = runtime.NumCPU() * 2 + } + return config +} + +// checkTargets checks does target exists and it is directory or file. +func checkTargets(targets []string) (map[string]bool, error) { + isDir := make(map[string]bool) + for _, target := range targets { + fileInfo, err := os.Stat(target) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("target does not exists %q: %v\n", filepath.ToSlash(target), err) + } + return nil, err + } + isDir[target] = fileInfo.IsDir() + } + return isDir, nil +} + +func (c *Counter) Run(targets []string) (*Result, error) { + if len(targets) == 0 { + targets = []string{"."} + } + + isDir, err := checkTargets(targets) + if err != nil { + return nil, err + } + + // initializes worker pool + c.reset() + + workersCount := c.Config.NumWorkers + maximumWaitingWork := workersCount * 4 + c.filesToAnalyze = make(chan string, maximumWaitingWork) + c.logVerbosef("creating %d workers with work queue of capacity %d", workersCount, maximumWaitingWork) + for range workersCount { + c.workers.Add(1) + go c.analyzeFilesWorker() + } + + for _, target := range targets { + if isDir[target] { + err := c.walkDir(target) + if err != nil { + close(c.filesToAnalyze) + c.workers.Wait() + return nil, err + } + } else { + c.FilesFound.Add(1) + c.filesToAnalyze <- target + } + + } + + close(c.filesToAnalyze) + c.workers.Wait() + + result := Result{LinesByExtension: make(map[string]int)} + c.linesLock.Lock() + for ext, count := range c.lines { + result.LinesByExtension[ext] = count + } + c.linesLock.Unlock() + + return &result, nil +} + +func (c *Counter) startAnalysis() { + c.reset() +} + +// analyzeDirectory analyzes the given directory and returns the results. +// It recursively walks the directory tree using goroutines for performance. +func (c *Counter) analyzeDirectory(dir string) (*Result, error) { + if _, err := os.Stat(dir); os.IsNotExist(err) { + return nil, fmt.Errorf("directory %q does not exist", dir) + } + + c.logVerbosef("starting analysis at: %q", filepath.ToSlash(dir)) + err := c.walkDir(dir) + if err != nil { + return nil, err + } + + // Makes copy of result. + c.logVerbosef("copying result") + linesByExt := make(map[string]int) + for ext, count := range c.lines { + linesByExt[ext] = count + } + + return &Result{ + LinesByExtension: linesByExt, + }, nil +} + +func (c *Counter) addLineCount(ext string, count int) { + c.linesLock.Lock() + c.lines[ext] += count + c.linesLock.Unlock() +} + +func (c *Counter) reset() { + c.workers = sync.WaitGroup{} + c.FilesFound.Store(0) + c.FilesProcessed.Store(0) + + c.linesLock.Lock() + c.lines = make(map[string]int) + c.linesLock.Unlock() +} diff --git a/pkg/lines/defaults.go b/pkg/lines/defaults.go index 6a8ee37..616f1f4 100644 --- a/pkg/lines/defaults.go +++ b/pkg/lines/defaults.go @@ -1,33 +1,74 @@ package lines -// DefaultIgnoredDirs returns default directories to ignore. -func DefaultIgnoredDirs() []string { - return []string{ - "node_modules", "vendor", ".git", "target", +import "strings" + +// DefaultIgnoredDirs returns the default directories to ignore. +func DefaultIgnoredDirs() map[string]struct{} { + return map[string]struct{}{ + "node_modules": {}, + "vendor": {}, + ".git": {}, + "target": {}, + } +} + +// DefaultIgnoredExtensions returns the default file extensions to ignore. +func DefaultIgnoredExtensions() map[string]struct{} { + return makeExtensionSet( + "exe", "dll", "so", "dylib", "msi", "mui", "mun", + "zip", "tar", "gz", "bz2", "xz", + "jpg", "jpeg", + "png", "dng", "heic", + "gif", "bmp", "webp", "svg", "ico", + "mp3", "wav", "flac", "ogg", "aac", + "mp4", "mkv", "avi", "mov", "wmv", + "pdf", "doc", "docx", "xls", "xlsx", + "icns", "ttf", "otf", "woff", "woff2", + "eot", "svgz", "uasset", "plist", + "url", "pbxproj", "sln", + "vcxproj", "csproj", "vcproj", "tlog", + "tmp", "filters", "idb", "lock", "rc", + "sqlite", "gdb", "node", "rmeta", + "rlib", "mcmeta", "iml", "map", "natvis", + "d", "dat_old", "storyboard", "ilk", "ppt", + "pptx", "odt", "ods", "odp", "odg", "mca", + "psd", "bin", "jar", "pdb", "dox", "db", + "schem", "lnk", "mod", "lib", "o", "obj", + "a", "class", "pyc", "pyo", "whl", "log", + "in", "dat", "TAG", "repositories", "MF", + ) +} + +// IgnoredDirSet builds a directory ignore set from a list of directory names. +func IgnoredDirSet(items ...string) map[string]struct{} { + return makeStringSet(items...) +} + +// IgnoredExtensionSet builds a file extension ignore set from a list of extensions. +// Extensions may be passed with or without a leading dot. +func IgnoredExtensionSet(items ...string) map[string]struct{} { + return makeExtensionSet(items...) +} + +// makeExtensionSet makes a set of file extensions for faster lookup. +// The extensions are stored in lowercase and prefixed with a dot. +// NOTE: Extensions are prefixed with dot to avoid stripping out the dot for every file. +func makeExtensionSet(items ...string) map[string]struct{} { + set := make(map[string]struct{}, len(items)) + for _, item := range items { + normalized := strings.TrimPrefix(strings.ToLower(item), ".") + if normalized == "" { + continue + } + set["."+normalized] = struct{}{} } + return set } -// DefaultIgnoredExtensions returns default file extensions to ignore. -func DefaultIgnoredExtensions() []string { - return []string{ - ".exe", ".dll", ".so", ".dylib", - ".zip", ".tar", ".gz", ".bz2", ".xz", - ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".ico", - ".mp3", ".wav", ".flac", ".ogg", ".aac", - ".mp4", ".mkv", ".avi", ".mov", ".wmv", - ".pdf", ".doc", ".docx", ".xls", ".xlsx", - ".icns", ".ttf", ".otf", ".woff", ".woff2", - ".eot", ".svgz", ".uasset", ".plist", - ".url", ".pbxproj", ".sln", - ".vcxproj", ".csproj", ".vcproj", ".tlog", - ".tmp", ".filters", ".idb", ".lock", ".rc", - ".sqlite", ".gdb", ".node", ".rmeta", - ".rlib", ".mcmeta", ".iml", ".map", ".natvis", - ".d", ".dat_old", ".storyboard", ".ilk", ".ppt", - ".pptx", ".odt", ".ods", ".odp", ".odg", ".mca", - ".psd", ".bin", ".jar", ".pdb", ".dox", ".db", - ".schem", ".lnk", ".mod", ".lib", ".o", ".obj", - ".a", ".class", ".pyc", ".pyo", ".whl", ".log", - ".in", ".dat", ".TAG", ".repositories", ".MF", +func makeStringSet(items ...string) map[string]struct{} { + set := make(map[string]struct{}, len(items)) + for _, item := range items { + set[item] = struct{}{} } + return set } diff --git a/pkg/lines/lines.go b/pkg/lines/lines.go deleted file mode 100644 index 4e257dc..0000000 --- a/pkg/lines/lines.go +++ /dev/null @@ -1,213 +0,0 @@ -package lines - -import ( - "bufio" - "fmt" - "os" - "path/filepath" - "strings" - "sync" - - cmap "github.com/orcaman/concurrent-map/v2" -) - -// Config holds settings for the line counting process. -type Config struct { - // IncludeHidden analyzes hidden files and directories (starting with '.'). - IncludeHidden bool - // IgnoredDirs are directories to skip during analysis. Defaults to ["node_modules", "vendor", ".git", "target"]. - IgnoredDirs []string - // IgnoredExtensions are file extensions to skip. Defaults to common binary and media formats. - IgnoredExtensions []string - // BufferInitialSize is the initial buffer size for the scanner. Defaults to 64KB. - BufferInitialSize int - // BufferMaxSize is the maximum buffer size for the scanner. Defaults to 1MB. - BufferMaxSize int -} - -// Represents the results of the line counting process. -type Result struct { - // LinesByExtension maps file extensions to their total line counts. - LinesByExtension map[string]int -} - -// Counter analyzes directories and counts non-blank lines of code. -// NOTE: Counter is safe for concurrent use and uses goroutines internally. -type Counter struct { - config Config - lines cmap.ConcurrentMap[string, int] - workers sync.WaitGroup -} - -// NewCounter creates a new Counter with the given configuration. -// If IgnoredDirs or IgnoredExtensions are empty, sensible defaults are used. -func NewCounter(config Config) *Counter { - // Use sensible defaults if lists are empty. - if len(config.IgnoredDirs) == 0 { - config.IgnoredDirs = DefaultIgnoredDirs() - } - if len(config.IgnoredExtensions) == 0 { - config.IgnoredExtensions = DefaultIgnoredExtensions() - } - if config.BufferInitialSize == 0 { - config.BufferInitialSize = 64 * 1024 - } - if config.BufferMaxSize == 0 { - config.BufferMaxSize = 1024 * 1024 - } - - return &Counter{ - config: config, - lines: cmap.New[int](), - } -} - -// isIgnoredDir checks if a directory should be ignored. -func (c *Counter) isIgnoredDir(dirname string) bool { - for _, ignored := range c.config.IgnoredDirs { - if dirname == ignored { - return true - } - } - return false -} - -// isIgnoredExtension checks if a file extension should be ignored. -// The comparison is case-insensitive. -func (c *Counter) isIgnoredExtension(ext string) bool { - ext = strings.ToLower(ext) - for _, ignored := range c.config.IgnoredExtensions { - if ext == strings.ToLower(ignored) { - return true - } - } - return false -} - -// Run analyzes the given directory and returns the results. -// It recursively walks the directory tree using goroutines for performance. -func (c *Counter) Run(dir string) (*Result, error) { - if _, err := os.Stat(dir); os.IsNotExist(err) { - return nil, fmt.Errorf("directory '%s' does not exist", dir) - } - - c.workers.Add(1) - go c.walkDir(dir) - c.workers.Wait() - - result := &Result{ - LinesByExtension: c.lines.Items(), - } - return result, nil -} - -// walkDir recursively walks the directory tree and counts lines in files. -// It spawns goroutines for each subdirectory to achieve parallel processing. -func (c *Counter) walkDir(dir string) { - defer c.workers.Done() - - visit := func(path string, f os.FileInfo, err error) error { - if err != nil { - // NOTE: Log access errors but continue with other directories. - fmt.Fprintf(os.Stderr, "ERROR: cannot access path %q: %v\n", path, err) - return err - } - if f.IsDir() && path != dir { - dirname := filepath.Base(path) - if !c.config.IncludeHidden && dirname[0] == '.' { - return filepath.SkipDir - } - if c.isIgnoredDir(dirname) { - return filepath.SkipDir - } - c.workers.Add(1) - go c.walkDir(path) - return filepath.SkipDir - } - if f.Mode().IsRegular() { - if c.needToAnalyze(path) { - c.fastLineCounter(path) - } - } - return nil - } - filepath.Walk(dir, visit) -} - -// needToAnalyze determines if a file should be analyzed. -// Returns false if the file is hidden (when IncludeHidden is false), -// has no extension, or has an ignored extension. -func (c *Counter) needToAnalyze(path string) bool { - if !c.config.IncludeHidden && filepath.Base(path)[0] == '.' { - return false - } - extension := filepath.Ext(path) - if len(extension) == 0 { - return false - } - if c.isIgnoredExtension(extension) { - return false - } - return true -} - -// fastLineCounter counts non-blank lines in a file and updates results. -// TODO: Consider caching results for frequently accessed files. -func (c *Counter) fastLineCounter(path string) { - extension := strings.ToLower(filepath.Ext(path)) - c.workers.Add(1) - go func() { - defer c.workers.Done() - countedLines, err := countNonBlankLines(path, c.config.BufferInitialSize, c.config.BufferMaxSize) - if err != nil { - // NOTE: Silently skip files with read/encoding issues. - fmt.Fprintf(os.Stderr, "WARNING: failed to count lines in %q: %v\n", path, err) - return - } - if countedLines > 0 { - c.lines.Upsert(extension, countedLines, func(exists bool, valueInMap int, newValue int) int { - if exists { - return valueInMap + newValue - } - return newValue - }) - } - }() -} - -// countNonBlankLines reads a file and counts non-blank, non-comment lines. -// Lines starting with '//' or '#' are treated as comments and skipped. -// bufferInitialSize specifies the initial scanner buffer size. -// bufferMaxSize specifies the maximum scanner buffer size. -func countNonBlankLines(path string, bufferInitialSize, bufferMaxSize int) (int, error) { - file, err := os.Open(path) - if err != nil { - return 0, err - } - defer file.Close() - - scanner := bufio.NewScanner(file) - buffer := make([]byte, 0, bufferInitialSize) - scanner.Buffer(buffer, bufferMaxSize) - - lineCounter := 0 - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - - // Skip empty lines. - if line == "" { - continue - } - // Skip comment lines: //, #, or --. - if strings.HasPrefix(line, "//") || strings.HasPrefix(line, "#") || strings.HasPrefix(line, "--") { - continue - } - lineCounter++ - } - - if err := scanner.Err(); err != nil { - return 0, err - } - - return lineCounter, nil -} diff --git a/pkg/lines/log.go b/pkg/lines/log.go new file mode 100644 index 0000000..00a1d17 --- /dev/null +++ b/pkg/lines/log.go @@ -0,0 +1,16 @@ +package lines + +import ( + "fmt" + "os" +) + +// TODO: use writer from config. + +// logVerbosef logs a formatted message if verbose mode is enabled in config. +func (c *Counter) logVerbosef(format string, v ...any) { + if !c.Config.Verbose { + return + } + fmt.Fprintf(os.Stderr, "info: "+format+"\n", v...) +} diff --git a/pkg/lines/result.go b/pkg/lines/result.go new file mode 100644 index 0000000..8d95302 --- /dev/null +++ b/pkg/lines/result.go @@ -0,0 +1,23 @@ +package lines + +// TODO: add processed files count. +// TODO: add total lines count. + +// Result represents the results of the line counting process. +type Result struct { + // LinesByExtension maps file extensions to their total line counts. + LinesByExtension map[string]int +} + +// MergeResults merges multiple results into single new result. +func MergeResults(results ...Result) Result { + merged := Result{ + LinesByExtension: make(map[string]int), + } + for _, result := range results { + for ext, count := range result.LinesByExtension { + merged.LinesByExtension[ext] += count + } + } + return merged +} diff --git a/pkg/lines/walker.go b/pkg/lines/walker.go new file mode 100644 index 0000000..5f4ee84 --- /dev/null +++ b/pkg/lines/walker.go @@ -0,0 +1,81 @@ +package lines + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// walkDir recursively walks the directory tree and enqueues files for the analyzeFilesWorker pool to analyze. +func (c *Counter) walkDir(dir string) error { + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + fmt.Fprintf(os.Stderr, "warn: failed to access path %q: %v\n", filepath.ToSlash(path), err) + return nil + } + + if d.IsDir() { + if path == dir { + return nil + } + name := d.Name() + if !c.Config.IncludeHidden && strings.HasPrefix(name, ".") { + c.logVerbosef("skipping hidden directory: %q", filepath.ToSlash(path)) + return filepath.SkipDir + } + if c.isIgnoredDir(name) { + c.logVerbosef("skipping ignored directory: %q", filepath.ToSlash(path)) + return filepath.SkipDir + } + + // continue walking + return nil + } + + if d.Type().IsRegular() && c.shouldAnalyzeFile(path, d.Name()) { + // Enqueue the file for analysis. + c.logVerbosef("found file to analyze: %q", filepath.ToSlash(path)) + c.FilesFound.Add(1) + c.filesToAnalyze <- path + } else { + // File is skipped because it is not a regular file or its extension is ignored. + c.logVerbosef("skipping file: %q", filepath.ToSlash(path)) + } + + return nil + }) + + return err +} + +// shouldAnalyzeFile determines if a file should be analyzed. +// Returns false if the file is hidden (when IncludeHidden is false), +// has no extension, or has an ignored extension. +func (c *Counter) shouldAnalyzeFile(path, filename string) bool { + if !c.Config.IncludeHidden && strings.HasPrefix(filename, ".") { + return false + } + + ext := filepath.Ext(path) + if len(ext) == 0 { + // probably its binary file + return false + } + + return !c.isIgnoredExtension(ext) +} + +// isIgnoredDir checks if a directory should be ignored. +func (c *Counter) isIgnoredDir(dirname string) bool { + _, ok := c.Config.IgnoredDirs[dirname] + return ok +} + +// isIgnoredExtension checks if a file extension should be ignored for line counting. +// The comparison is case-insensitive. Extension should begin with dot. +func (c *Counter) isIgnoredExtension(ext string) bool { + _, ok := c.Config.IgnoredExtensions[strings.ToLower(ext)] + return ok +} diff --git a/pkg/lines/worker.go b/pkg/lines/worker.go new file mode 100644 index 0000000..cdd1942 --- /dev/null +++ b/pkg/lines/worker.go @@ -0,0 +1,32 @@ +package lines + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +func (c *Counter) analyzeFilesWorker() { + defer c.workers.Done() + + for path := range c.filesToAnalyze { + lineCount, err := analyzeFile(path, c.Config.ReaderInitialBufferSize) + c.FilesProcessed.Add(1) + + if err != nil { + // logs the error and continues processing other files. + fmt.Fprintf(os.Stderr, "warn: failed to count lines in %q: %v\n", filepath.ToSlash(path), err) + continue + } + + if lineCount <= 0 { + // file is empty. + continue + } + + ext := strings.ToLower(filepath.Ext(path)) + c.addLineCount(ext, lineCount) + c.logVerbosef("file %q had %d lines", filepath.ToSlash(path), lineCount) + } +}