From 3c66aff754532d3155aaa09d8735f326f8265180 Mon Sep 17 00:00:00 2001 From: Viacheslav Poturaev Date: Mon, 15 Apr 2024 17:10:33 +0200 Subject: [PATCH] Use scan instead of copy in parallel mode --- .github/workflows/golangci-lint.yml | 6 +++--- .github/workflows/gorelease.yml | 2 +- .github/workflows/release-assets.yml | 4 ++-- .github/workflows/test-unit.yml | 4 ++-- .golangci.yml | 2 +- Makefile | 2 +- cmd/catp/catp/app.go | 24 +++++++++++------------- go.mod | 4 ++-- go.sum | 4 ++++ 9 files changed, 27 insertions(+), 25 deletions(-) diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 5e985cc..6239219 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -21,13 +21,13 @@ jobs: steps: - uses: actions/setup-go@v3 with: - go-version: 1.21.x + go-version: 1.22.x - uses: actions/checkout@v2 - name: golangci-lint - uses: golangci/golangci-lint-action@v3.7.0 + uses: golangci/golangci-lint-action@v4.0.0 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.55.2 + version: v1.56.2 # Optional: working directory, useful for monorepos # working-directory: somedir diff --git a/.github/workflows/gorelease.yml b/.github/workflows/gorelease.yml index 97ebe87..6356a9d 100644 --- a/.github/workflows/gorelease.yml +++ b/.github/workflows/gorelease.yml @@ -9,7 +9,7 @@ concurrency: cancel-in-progress: true env: - GO_VERSION: 1.21.x + GO_VERSION: 1.22.x jobs: gorelease: runs-on: ubuntu-latest diff --git a/.github/workflows/release-assets.yml b/.github/workflows/release-assets.yml index d510ad2..d322a95 100644 --- a/.github/workflows/release-assets.yml +++ b/.github/workflows/release-assets.yml @@ -8,13 +8,13 @@ on: - created env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GO_VERSION: '1.22.0-rc.1' + GO_VERSION: 1.22.x LINUX_AMD64_BUILD_OPTIONS: '-tags cgo_zstd' GOAMD64: v3 jobs: build: name: Upload Release Assets - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Install Go stable if: env.GO_VERSION != 'tip' diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 4214222..9b6e93d 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -15,13 +15,13 @@ concurrency: env: GO111MODULE: "on" RUN_BASE_COVERAGE: "on" # Runs test for PR base in case base test coverage is missing. - COV_GO_VERSION: 1.21.x # Version of Go to collect coverage + COV_GO_VERSION: 1.22.x # Version of Go to collect coverage TARGET_DELTA_COV: 90 # Target coverage of changed lines, in percents jobs: test: strategy: matrix: - go-version: [ 1.19.x, 1.20.x, 1.21.x ] + go-version: [ 1.20.x, 1.21.x, 1.22.x ] runs-on: ubuntu-latest steps: - name: Install Go stable diff --git a/.golangci.yml b/.golangci.yml index c2a2977..43850cc 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -19,7 +19,7 @@ linters-settings: funlen: lines: 80 cyclop: - max-complexity: 15 + max-complexity: 16 linters: enable-all: true diff --git a/Makefile b/Makefile index 4926457..caf1cf0 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -#GOLANGCI_LINT_VERSION := "v1.55.2" # Optional configuration to pinpoint golangci-lint version. +#GOLANGCI_LINT_VERSION := "v1.56.2" # Optional configuration to pinpoint golangci-lint version. # The head of Makefile determines location of dev-go to include standard targets. GO ?= go diff --git a/cmd/catp/catp/app.go b/cmd/catp/catp/app.go index 77d2208..3ebd9c0 100644 --- a/cmd/catp/catp/app.go +++ b/cmd/catp/catp/app.go @@ -279,7 +279,7 @@ func (r *runner) cat(filename string) (err error) { }) } - if len(r.pass) > 0 || len(r.skip) > 0 { + if len(r.pass) > 0 || len(r.skip) > 0 || r.parallel > 1 { r.scanFile(rd, out) } else { r.readFile(rd, out) @@ -364,6 +364,8 @@ func Main() error { //nolint:funlen,cyclop,gocognit,gocyclo skip stringFlags ) + r := &runner{} + flag.Var(&pass, "pass", "filter matching, may contain multiple AND patterns separated by ^,\n"+ "if filter matches, line is passed to the output (unless filtered out by -skip)\n"+ "each -pass value is added with OR logic,\n"+ @@ -374,20 +376,21 @@ func Main() error { //nolint:funlen,cyclop,gocognit,gocyclo "each -skip value is added with OR logic,\n"+ "for example, you can use \"-skip quux^baz -skip fooO\" to skip lines that have (quux AND baz) OR fooO") - parallel := flag.Int("parallel", 1, "number of parallel readers if multiple files are provided\n"+ + flag.IntVar(&r.parallel, "parallel", 1, "number of parallel readers if multiple files are provided\n"+ "lines from different files will go to output simultaneously (out of order of files, but in order of lines in each file)\n"+ "use 0 for multi-threaded zst decoder (slightly faster at cost of more CPU)") cpuProfile := flag.String("dbg-cpu-prof", "", "write first 10 seconds of CPU profile to file") memProfile := flag.String("dbg-mem-prof", "", "write heap profile to file after 10 seconds") output := flag.String("output", "", "output to file instead of STDOUT") - outDir := flag.String("out-dir", "", "output to directory instead of STDOUT\n"+ - "files will be written to out dir with original base names\n"+ - "disables output flag") noProgress := flag.Bool("no-progress", false, "disable progress printing") progressJSON := flag.String("progress-json", "", "write current progress to a file") ver := flag.Bool("version", false, "print version and exit") + flag.StringVar(&r.outDir, "out-dir", "", "output to directory instead of STDOUT\n"+ + "files will be written to out dir with original base names\n"+ + "disables output flag") + flag.Usage = func() { fmt.Println("catp", version.Module("github.com/bool64/progress").Version+",", version.Info().GoVersion, strings.Join(versionExtra, " ")) @@ -420,12 +423,7 @@ func Main() error { //nolint:funlen,cyclop,gocognit,gocyclo defer pprof.StopCPUProfile() } - r := &runner{} - - r.parallel = *parallel - r.outDir = *outDir - - if *output != "" && *outDir == "" { //nolint:nestif + if *output != "" && r.outDir == "" { //nolint:nestif out, err := os.Create(*output) if err != nil { return fmt.Errorf("failed to create output file %s: %w", *output, err) @@ -503,7 +501,7 @@ func Main() error { //nolint:funlen,cyclop,gocognit,gocyclo r.sizes[fn] = st.Size() } - if *parallel >= 2 { + if r.parallel >= 2 { pr := r.pr pr.Start(func(t *progress.Task) { t.TotalBytes = func() int64 { return r.totalBytes } @@ -513,7 +511,7 @@ func Main() error { //nolint:funlen,cyclop,gocognit,gocyclo t.PrintOnStart = true }) - sem := make(chan struct{}, *parallel) + sem := make(chan struct{}, r.parallel) errs := make(chan error, 1) for i := 0; i < flag.NArg(); i++ { diff --git a/go.mod b/go.mod index 538a66b..5df2d84 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.21 require ( github.com/DataDog/zstd v1.5.5 - github.com/bool64/dev v0.2.33 - github.com/klauspost/compress v1.17.4 + github.com/bool64/dev v0.2.34 + github.com/klauspost/compress v1.17.8 github.com/klauspost/pgzip v1.2.6 ) diff --git a/go.sum b/go.sum index 9c457dc..e2b52ff 100644 --- a/go.sum +++ b/go.sum @@ -2,7 +2,11 @@ github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/bool64/dev v0.2.33 h1:ETAcSa8H9w4talcCdSQCCnLX7PMHmuxdLcDl6TpSDj4= github.com/bool64/dev v0.2.33/go.mod h1:iJbh1y/HkunEPhgebWRNcs8wfGq7sjvJ6W5iabL8ACg= +github.com/bool64/dev v0.2.34 h1:P9n315P8LdpxusnYQ0X7MP1CZXwBK5ae5RZrd+GdSZE= +github.com/bool64/dev v0.2.34/go.mod h1:iJbh1y/HkunEPhgebWRNcs8wfGq7sjvJ6W5iabL8ACg= github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=