diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..36b6c15 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,20 @@ +version: 2 +updates: + - package-ecosystem: "gomod" + directory: "/" + schedule: + interval: "daily" + # security updates only + open-pull-requests-limit: 0 + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "daily" + # security updates only + open-pull-requests-limit: 0 + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + # security updates only + open-pull-requests-limit: 0 \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..b9c3154 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,41 @@ +name: continuous-integration + +on: + pull_request: + branches: + - '*' + push: + branches: + - main + - master # backward compatibility + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +env: + SHELL: /bin/bash + +defaults: + run: + shell: bash + +jobs: + test: + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: actions/setup-go@v3 + with: + go-version: 'stable' + - run: go test -v -cover ./... + - name: TruffleHog OSS + uses: trufflesecurity/trufflehog@v3.16.4 + with: + path: ./ + base: ${{ github.event.repository.default_branch }} + head: HEAD + extra_args: --debug --only-verified \ No newline at end of file diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..6a7f980 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,76 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ "main" ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "main" ] + schedule: + - cron: '32 16 * * 3' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'go' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Use only 'java' to analyze code written in Java, Kotlin or both + # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{matrix.language}}" \ No newline at end of file diff --git a/config.go b/config.go index 42c0b87..1e62c30 100644 --- a/config.go +++ b/config.go @@ -1,10 +1,9 @@ package main import ( - "io/ioutil" "os" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) type FileInputConfig struct { @@ -28,10 +27,10 @@ func ReadConfigFromFile(filename string) (*Config, error) { } defer file.Close() - data, err := ioutil.ReadAll(file) config := Config{} - err = yaml.UnmarshalStrict([]byte(data), &config) - + decoder := yaml.NewDecoder(file) + decoder.KnownFields(true) + err = decoder.Decode(&config) if err != nil { return &config, err } diff --git a/config_test.go b/config_test.go new file mode 100644 index 0000000..26eaa7f --- /dev/null +++ b/config_test.go @@ -0,0 +1,70 @@ +package main + +import ( + "os" + "path" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestReadConfigFromFile(t *testing.T) { + temporaryDirectory, err := os.MkdirTemp(os.TempDir(), "sentlog-*") + if err != nil { + t.Fatalf("creating temporary directory: %v", err) + } + + t.Cleanup(func() { + err := os.RemoveAll(temporaryDirectory) + if err != nil { + t.Log(err) + } + }) + + // Write temporary file + payload := []byte(`--- +# Sentry DSN (also can be configured via environment) +sentry_dsn: https://XXX@sentry.io/YYY +# Additional Grok pattern files +pattern_files: + - ./patterns1.txt + - ../patterns2.txt + +# List of files that we want to watch +inputs: + - file: /var/log/nginx/error.log + # Patterns to find and report + patterns: + - "%{NGINX_ERROR_LOG}" + # Additional tags that will be added to the Sentry event + tags: + pattern: nginx_error + custom: tag`) + fileName := path.Join(temporaryDirectory, "config.yml") + err = os.WriteFile(fileName, payload, 0644) + if err != nil { + t.Fatalf("writing temporary file: %v", err) + } + + config, err := ReadConfigFromFile(fileName) + if err != nil { + t.Error(err) + } + + assert.Equal(t, &Config{ + SentryDsn: "https://XXX@sentry.io/YYY", + PatternFiles: []string{"./patterns1.txt", "../patterns2.txt"}, + Inputs: []FileInputConfig{ + { + File: "/var/log/nginx/error.log", + Follow: nil, + FromLineNumber: nil, + Patterns: []string{"%{NGINX_ERROR_LOG}"}, + Tags: map[string]string{ + "pattern": "nginx_error", + "custom": "tag", + }, + }, + }, + }, config) +} diff --git a/go.mod b/go.mod index 1dee375..13f3267 100644 --- a/go.mod +++ b/go.mod @@ -1,18 +1,26 @@ module sentlog -go 1.12 +go 1.20 require ( - github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc // indirect - github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf // indirect - github.com/araddon/dateparse v0.0.0-20190622164848-0fb0a474d195 - github.com/getsentry/sentry-go v0.1.1-0.20190624124141-69c26e4dfca8 + github.com/alecthomas/kingpin/v2 v2.3.2 + github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de + github.com/getsentry/sentry-go v0.20.0 github.com/hpcloud/tail v1.0.1-0.20180514194441-a1dbeea552b7 - github.com/sirupsen/logrus v1.4.2 - github.com/vjeantet/grok v1.0.1-0.20180213041522-5a86c829f3c3 - golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 // indirect - gopkg.in/alecthomas/kingpin.v2 v2.2.6 + github.com/sirupsen/logrus v1.9.0 + github.com/stretchr/testify v1.8.2 + github.com/vjeantet/grok v1.0.1 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/xhit/go-str2duration/v2 v2.1.0 // indirect + golang.org/x/sys v0.6.0 // indirect + golang.org/x/text v0.8.0 // indirect gopkg.in/fsnotify/fsnotify.v1 v1.4.7 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect - gopkg.in/yaml.v2 v2.2.2 ) diff --git a/go.sum b/go.sum index af28490..b335ba8 100644 --- a/go.sum +++ b/go.sum @@ -1,34 +1,55 @@ -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/araddon/dateparse v0.0.0-20190622164848-0fb0a474d195 h1:c4mLfegoDw6OhSJXTd2jUEQgZUQuJWtocudb97Qn9EM= -github.com/araddon/dateparse v0.0.0-20190622164848-0fb0a474d195/go.mod h1:SLqhdZcd+dF3TEVL2RMoob5bBP5R1P1qkox+HtCBgGI= +github.com/alecthomas/kingpin/v2 v2.3.2 h1:H0aULhgmSzN8xQ3nX1uxtdlTHYoPLu5AhHxWrKI6ocU= +github.com/alecthomas/kingpin/v2 v2.3.2/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA= +github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/getsentry/sentry-go v0.1.1-0.20190624124141-69c26e4dfca8 h1:mAdAkMPKhAvYWrlrLRx7Ya+LdJt56L2EdjiAJPgLPJQ= -github.com/getsentry/sentry-go v0.1.1-0.20190624124141-69c26e4dfca8/go.mod h1:2QfSdvxz4IZGyB5izm1TtADFhlhfj1Dcesrg8+A/T9Y= -github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/getsentry/sentry-go v0.20.0 h1:bwXW98iMRIWxn+4FgPW7vMrjmbym6HblXALmhjHmQaQ= +github.com/getsentry/sentry-go v0.20.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/hpcloud/tail v1.0.1-0.20180514194441-a1dbeea552b7 h1:Ysi1UhrSyBltF8f+3RAt4UaqHc+53JJ0jyl0pY0sfck= github.com/hpcloud/tail v1.0.1-0.20180514194441-a1dbeea552b7/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/pingcap/errors v0.11.1/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/vjeantet/grok v1.0.1-0.20180213041522-5a86c829f3c3 h1:T3ATR84Xk4b9g0QbGgLJVpRYWm/jvixqLTWRsR108sI= -github.com/vjeantet/grok v1.0.1-0.20180213041522-5a86c829f3c3/go.mod h1:/FWYEVYekkm+2VjcFmO9PufDU5FgXHUz9oy2EGqmQBo= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 h1:HyfiK1WMnHj5FXFXatD+Qs1A/xC2Run6RzeW1SyHxpc= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/vjeantet/grok v1.0.1 h1:2rhIR7J4gThTgcZ1m2JY4TrJZNgjn985U28kT2wQrJ4= +github.com/vjeantet/grok v1.0.1/go.mod h1:ax1aAchzC6/QMXMcyzHQGZWaW1l195+uMYIkCWPCNIo= +github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= +github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify/fsnotify.v1 v1.4.7 h1:XNNYLJHt73EyYiCZi6+xjupS9CpvmiDgjPTAjrBlQbo= gopkg.in/fsnotify/fsnotify.v1 v1.4.7/go.mod h1:Fyux9zXlo4rWoMSIzpn9fDAYjalPqJ/K1qJ27s+7ltE= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/grok.go b/grok.go index 58d7134..999c430 100644 --- a/grok.go +++ b/grok.go @@ -9,10 +9,18 @@ import ( "github.com/vjeantet/grok" ) -func AddDefaultPatterns(g *grok.Grok) error { +func AddDefaultPatterns(g *grok.Grok) (err error) { // Nginx - g.AddPattern("NGINX_ERROR_DATESTAMP", `\d{4}/\d{2}/\d{2}[- ]%{TIME}`) - g.AddPattern("NGINX_ERROR_LOG", `%{NGINX_ERROR_DATESTAMP:timestamp} \[%{DATA:err_severity}\] (%{NUMBER:pid:int}#%{NUMBER}: \*%{NUMBER}|\*%{NUMBER}) %{DATA:message}(?:, client: "?%{IPORHOST:client}"?)(?:, server: %{IPORHOST:server})(?:, request: "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}")?(?:, upstream: "%{DATA:upstream}")?(?:, host: "%{URIHOST:host}")?(?:, referrer: "%{URI:referrer}")?`) + err = g.AddPattern("NGINX_ERROR_DATESTAMP", `\d{4}/\d{2}/\d{2}[- ]%{TIME}`) + if err != nil { + return err + } + + err = g.AddPattern("NGINX_ERROR_LOG", `%{NGINX_ERROR_DATESTAMP:timestamp} \[%{DATA:err_severity}\] (%{NUMBER:pid:int}#%{NUMBER}: \*%{NUMBER}|\*%{NUMBER}) %{DATA:message}(?:, client: "?%{IPORHOST:client}"?)(?:, server: %{IPORHOST:server})(?:, request: "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}")?(?:, upstream: "%{DATA:upstream}")?(?:, host: "%{URIHOST:host}")?(?:, referrer: "%{URI:referrer}")?`) + if err != nil { + return err + } + return nil } @@ -21,6 +29,7 @@ func ReadPatternsFromFile(g *grok.Grok, filename string) error { if err != nil { return err } + defer file.Close() log.Printf("Adding grok patterns from \"%s\"", filename) scanner := bufio.NewScanner(file) diff --git a/grok_test.go b/grok_test.go new file mode 100644 index 0000000..e4a2000 --- /dev/null +++ b/grok_test.go @@ -0,0 +1,98 @@ +package main + +import ( + "errors" + "fmt" + "os" + "path" + "testing" + + "github.com/vjeantet/grok" +) + +func TestAddDefaultPatterns(t *testing.T) { + g, err := grok.NewWithConfig(&grok.Config{NamedCapturesOnly: true}) + if err != nil { + t.Fatalf("Grok engine initialization failed: %v\n", err) + } + + err = AddDefaultPatterns(g) + if err != nil { + t.Error(err) + } +} + +func TestReadPatternsFromFile(t *testing.T) { + temporaryDirectory, err := os.MkdirTemp(os.TempDir(), "sentlog-*") + if err != nil { + t.Fatalf("creating temporary directory: %v", err) + } + + t.Cleanup(func() { + err := os.RemoveAll(temporaryDirectory) + if err != nil { + t.Log(err) + } + }) + + var tests = []struct { + testName string + fileName string + payload []byte + expectedError error + }{ + { + testName: "Normal case", + fileName: path.Join(temporaryDirectory, "normal.txt"), + payload: []byte(`postgres ^%{DATESTAMP:timestamp}.*FATAL:.*host" + +# This is a comment, should be skipped`), + expectedError: nil, + }, + { + testName: "Invalid length", + fileName: path.Join(temporaryDirectory, "invalid-length.txt"), + payload: []byte(`HelloWorld`), + expectedError: fmt.Errorf("Cannot parse patterns in \"%s\"", path.Join(temporaryDirectory, "invalid-length.txt")), + }, + { + testName: "Invalid pattern", + fileName: path.Join(temporaryDirectory, "invalid-pattern.txt"), + payload: []byte(`hello %{HELLO-WORLD}`), + expectedError: errors.New("no pattern found for %{HELLO-WORLD}"), + }, + { + testName: "Empty file", + fileName: path.Join(temporaryDirectory, "empty.txt"), + payload: nil, + expectedError: nil, + }, + } + for _, test := range tests { + t.Run(test.testName, func(t *testing.T) { + g, err := grok.NewWithConfig(&grok.Config{NamedCapturesOnly: true}) + if err != nil { + t.Fatalf("Grok engine initialization failed: %v\n", err) + } + + err = os.WriteFile(test.fileName, test.payload, 0644) + if err != nil { + t.Fatalf("writing temporary file: %v", err) + } + + err = ReadPatternsFromFile(g, test.fileName) + if test.expectedError == nil && err != nil { + t.Error(err) + } else if test.expectedError != nil { + if err != nil { + if test.expectedError.Error() != err.Error() { + t.Errorf("expecting %s, got %s", test.expectedError.Error(), err.Error()) + } + } else { + // err is nil + t.Errorf("expecting %s, got nil", test.expectedError.Error()) + } + } + }) + } +} diff --git a/main.go b/main.go index cc752be..58fd2c1 100644 --- a/main.go +++ b/main.go @@ -7,9 +7,9 @@ import ( "syscall" "time" + "github.com/alecthomas/kingpin/v2" "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" - "gopkg.in/alecthomas/kingpin.v2" ) type CmdArgs struct { @@ -63,7 +63,7 @@ func initSentry(config *Config) { // Catches Ctrl-C func catchInterrupt() { - c := make(chan os.Signal) + c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt, syscall.SIGINT) go func() { <-c diff --git a/process.go b/process.go index 3a1dc03..824cba1 100644 --- a/process.go +++ b/process.go @@ -122,7 +122,10 @@ func initGrokProcessor() *grok.Grok { if err != nil { log.Fatalf("Grok engine initialization failed: %v\n", err) } - AddDefaultPatterns(g) + + if err := AddDefaultPatterns(g); err != nil { + log.Fatalf("Processing default patterns: %v\n", err) + } return g } diff --git a/process_test.go b/process_test.go new file mode 100644 index 0000000..9c085da --- /dev/null +++ b/process_test.go @@ -0,0 +1,77 @@ +package main + +import ( + "sync" + "testing" + "time" + + "github.com/getsentry/sentry-go" +) + +func TestProcessLine(t *testing.T) { + _verbose = true + transportMock := &TransportMock{} + err := sentry.Init(sentry.ClientOptions{Debug: true, Transport: transportMock}) + if err != nil { + t.Fatal(err) + } + + g := initGrokProcessor() + + processLine( + "", + []string{"%{COMMONAPACHELOG}"}, + g, + sentry.CurrentHub().Clone(), + ) + // We expect it to not send anything to Sentry + if transportMock.lastEvent != nil { + t.Errorf("expecting nil, got %v", transportMock.lastEvent) + } + + processLine( + `127.0.0.1 - - [23/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207`, + []string{"%{COMMONAPACHELOG}"}, + g, + sentry.CurrentHub().Clone(), + ) + // We expect it to send something to Sentry + expectMessage := "127.0.0.1 - - [23/Apr/2014:22:58:32 +0200] \"GET /index.php HTTP/1.1\" 404 207" + if transportMock.lastEvent.Message != expectMessage { + t.Errorf("expecting lastEvent.Message to be %q, instead got %q", expectMessage, transportMock.lastEvent.Message) + } + + if transportMock.lastEvent.Level != sentry.LevelError { + t.Errorf("expecting lastEvent.Level to be %q, instead got %q", sentry.LevelError, transportMock.lastEvent.Level) + } + + if value, ok := transportMock.lastEvent.Extra["log_entry"]; ok && value != expectMessage { + t.Errorf("expecting transportMock.lastEvent.Extra[\"log_entry\"] to be %q, instead got %q", expectMessage, value) + } + + if value, ok := transportMock.lastEvent.Extra["pattern"]; ok && value != "%{COMMONAPACHELOG}" { + t.Errorf("expecting transportMock.lastEvent.Extra[\"log_entry\"] to be %q, instead got %q", "%{COMMONAPACHELOG}", value) + } +} + +type TransportMock struct { + mu sync.Mutex + events []*sentry.Event + lastEvent *sentry.Event +} + +func (t *TransportMock) Configure(options sentry.ClientOptions) {} +func (t *TransportMock) SendEvent(event *sentry.Event) { + t.mu.Lock() + defer t.mu.Unlock() + t.events = append(t.events, event) + t.lastEvent = event +} +func (t *TransportMock) Flush(timeout time.Duration) bool { + return true +} +func (t *TransportMock) Events() []*sentry.Event { + t.mu.Lock() + defer t.mu.Unlock() + return t.events +}