From d1270309c112e90031896f4fee643fa895ed6d43 Mon Sep 17 00:00:00 2001 From: insider-automation <117348511+insider-automation@users.noreply.github.com> Date: Fri, 1 Dec 2023 16:15:21 +0300 Subject: [PATCH 1/2] new workflow added --- .github/workflows/git-leak.yml | 43 ++++++---------------------------- 1 file changed, 7 insertions(+), 36 deletions(-) diff --git a/.github/workflows/git-leak.yml b/.github/workflows/git-leak.yml index e2b8e00..3b6abe3 100644 --- a/.github/workflows/git-leak.yml +++ b/.github/workflows/git-leak.yml @@ -1,44 +1,15 @@ -name: gitleaks -on: - pull_request: - types: [ready_for_review] -env: - GIT_DISCOVERY_ACROSS_FILESYSTEM: 1 +name: Gitleaks-Action +on: [push] jobs: - gitleaks: + build: runs-on: self-runner-node steps: - - name: Checkout Repository - uses: actions/checkout@v2 - with: - fetch-depth: '2' - - name: Detecting new added lines - run: | - git fetch origin ${{ github.event.repository.default_branch }} - git diff origin/${{ github.event.repository.default_branch }}..HEAD --name-only | xargs git diff origin/${{ github.event.repository.default_branch }}..HEAD -- | grep '+' | sed 's/+//' | sed 's/^[ \t]*//' > new-added-lines.txt - working-directory: ${{ github.workspace }} - - name: Install Go - uses: actions/setup-go@v2 - with: - go-version: 1.16 - - name: Install gitleaks - run: | - go get github.com/zricethezav/gitleaks/v7 - - name: Run gitleaks - run: | - gitleaks --no-git --path ${{ github.workspace }}/new-added-lines.txt --verbose --report=${{ github.workspace }}/gitleaks-report.json - - name: Send to Lambda - if: ${{ always() }} + - name: Trigger to Gitleak run: | python -c ' import json,sys,requests; - try: - output=open("./gitleaks-report.json"); - except IOError: - sys.exit(0); - json_result=json.loads(output.read()); - github_result = {"repository": "'${{ github.repository }}'", "server_url": "'${{ github.server_url }}'", "run_id": "'${{ github.run_id }}'", "pr_number": "'${{ github.event.number }}'"}; - request_json = {"gitleaks_result": json_result,"github": github_result}; - requests.post("'$LambdaWebHook'", json=request_json)' + github = {"repository": "'${{ github.event.repository.name }}'", "ref": "'${{ github.ref_name }}'"}; + github_request = {"insider_gitleak": github}; + requests.post("'$LambdaWebHook'", json=github_request);' env: LambdaWebHook: ${{ secrets.CHECKMARX_LAMBDA_WEBHOOK }} \ No newline at end of file From e06cea165a75482a0e153d3168b5f074f2d0397f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafet=20Top=C3=A7u?= <44340673+rafet@users.noreply.github.com> Date: Tue, 26 Dec 2023 15:24:59 +0300 Subject: [PATCH 2/2] PA-24295 Inssqs Implementation (#17) --- go.mod | 36 +++- go.sum | 159 ++++++++++++++-- insdash/insdash.go | 52 +++++ insdash/insdash_test.go | 108 +++++++++++ inslogger/README.md | 103 ++++++++++ inslogger/inslogger.go | 161 ++++++++++++++++ insredis/README.md | 71 +++++-- insredis/redis.go | 29 ++- inssentry/sentry.go | 18 +- inssqs/README.md | 82 ++++++++ inssqs/errors.go | 7 + inssqs/inssqs.go | 411 ++++++++++++++++++++++++++++++++++++++++ inssqs/inssqs_fake.go | 25 +++ inssqs/inssqs_mock.go | 68 +++++++ inssqs/inssqs_test.go | 191 +++++++++++++++++++ inssqs/model.go | 45 +++++ inssqs/sqs/iface.go | 32 ++++ inssqs/sqs/sqs_mock.go | 92 +++++++++ 18 files changed, 1645 insertions(+), 45 deletions(-) create mode 100644 insdash/insdash.go create mode 100644 insdash/insdash_test.go create mode 100644 inslogger/README.md create mode 100644 inslogger/inslogger.go create mode 100644 inssqs/README.md create mode 100644 inssqs/errors.go create mode 100644 inssqs/inssqs.go create mode 100644 inssqs/inssqs_fake.go create mode 100644 inssqs/inssqs_mock.go create mode 100644 inssqs/inssqs_test.go create mode 100644 inssqs/model.go create mode 100644 inssqs/sqs/iface.go create mode 100644 inssqs/sqs/sqs_mock.go diff --git a/go.mod b/go.mod index 836bede..7ed05fc 100644 --- a/go.mod +++ b/go.mod @@ -6,20 +6,37 @@ require ( github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/Jamil-Najafov/go-aws-ssm v0.9.0 github.com/aws/aws-sdk-go v1.44.3 + github.com/aws/aws-sdk-go-v2 v1.23.1 + github.com/aws/aws-sdk-go-v2/config v1.25.4 + github.com/aws/aws-sdk-go-v2/service/sqs v1.28.2 + github.com/aws/smithy-go v1.17.0 github.com/getsentry/sentry-go v0.13.0 github.com/go-redis/redis v6.15.9+incompatible github.com/golang/mock v1.6.0 github.com/google/uuid v1.3.1 github.com/jellydator/ttlcache/v3 v3.0.0 + github.com/pkg/errors v0.9.1 github.com/slok/goresilience v0.2.0 - github.com/stretchr/testify v1.7.0 + github.com/stretchr/testify v1.8.1 go.uber.org/mock v0.3.0 + go.uber.org/zap v1.26.0 gorm.io/driver/mysql v1.3.4 gorm.io/gorm v1.23.7 ) require ( - github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.16.3 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.17.3 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.25.4 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-sql-driver/mysql v1.6.0 // indirect github.com/golang/protobuf v1.5.2 // indirect @@ -31,13 +48,14 @@ require ( github.com/onsi/ginkgo v1.16.5 // indirect github.com/onsi/gomega v1.19.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v0.9.2 // indirect - github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 // indirect - github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 // indirect - github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a // indirect - golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect + github.com/prometheus/client_golang v1.11.1 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.26.0 // indirect + github.com/prometheus/procfs v0.6.0 // indirect + go.uber.org/multierr v1.10.0 // indirect + golang.org/x/net v0.18.0 // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect - golang.org/x/sys v0.1.0 // indirect + golang.org/x/sys v0.14.0 // indirect google.golang.org/protobuf v1.26.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 1f8df62..c961e79 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,49 @@ +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/Jamil-Najafov/go-aws-ssm v0.9.0 h1:nVhRUsGaVNj6B4ueyIXQP6XDyel0T+bZutiDkzofJww= github.com/Jamil-Najafov/go-aws-ssm v0.9.0/go.mod h1:NpjyTq6TT1PmXXlDJrKyvo1lE+fiOGDVrYrgVNzDSUA= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/aws/aws-sdk-go v1.44.3 h1:GA9bJsWeJdwPtcKK9Uq3jfMaPko0ROWzVuqwFM7BBP0= github.com/aws/aws-sdk-go v1.44.3/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= +github.com/aws/aws-sdk-go-v2 v1.23.1 h1:qXaFsOOMA+HsZtX8WoCa+gJnbyW7qyFFBlPqvTSzbaI= +github.com/aws/aws-sdk-go-v2 v1.23.1/go.mod h1:i1XDttT4rnf6vxc9AuskLc6s7XBee8rlLilKlc03uAA= +github.com/aws/aws-sdk-go-v2/config v1.25.4 h1:r+X1x8QI6FEPdJDWCNBDZHyAcyFwSjHN8q8uuus+Axs= +github.com/aws/aws-sdk-go-v2/config v1.25.4/go.mod h1:8GTjImECskr7D88P/Nn9uM4M4rLY9i77hLJZgkZEWV8= +github.com/aws/aws-sdk-go-v2/credentials v1.16.3 h1:8PeI2krzzjDJ5etmgaMiD1JswsrLrWvKKu/uBUtNy1g= +github.com/aws/aws-sdk-go-v2/credentials v1.16.3/go.mod h1:Kdh/okh+//vQ/AjEt81CjvkTo64+/zIE4OewP7RpfXk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.5 h1:KehRNiVzIfAcj6gw98zotVbb/K67taJE0fkfgM6vzqU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.5/go.mod h1:VhnExhw6uXy9QzetvpXDolo1/hjhx4u9qukBGkuUwjs= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.4 h1:LAm3Ycm9HJfbSCd5I+wqC2S9Ej7FPrgr5CQoOljJZcE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.4/go.mod h1:xEhvbJcyUf/31yfGSQBe01fukXwXJ0gxDp7rLfymWE0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.4 h1:4GV0kKZzUxiWxSVpn/9gwR0g21NF1Jsyduzo9rHgC/Q= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.4/go.mod h1:dYvTNAggxDZy6y1AF7YDwXsPuHFy/VNEpEI/2dWK9IU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 h1:uR9lXYjdPX0xY+NhvaJ4dD8rpSRz5VY81ccIIoNG+lw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.1 h1:rpkF4n0CyFcrJUG/rNNohoTmhtWlFTRI4BsZOh9PvLs= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.1/go.mod h1:l9ymW25HOqymeU2m1gbUQ3rUIsTwKs8gYHXkqDQUhiI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.4 h1:rdovz3rEu0vZKbzoMYPTehp0E8veoE9AyfzqCr5Eeao= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.4/go.mod h1:aYCGNjyUCUelhofxlZyj63srdxWUSsBSGg5l6MCuXuE= +github.com/aws/aws-sdk-go-v2/service/sqs v1.28.2 h1:MVg4eLi9uM1+YHYSfcCg1CR3mqtL6UJ9SF3VrMxKmUE= +github.com/aws/aws-sdk-go-v2/service/sqs v1.28.2/go.mod h1:7vHhhnzSGZcquR6+X7V+wDHdY8iOk5ge0z+FxoxkvJw= +github.com/aws/aws-sdk-go-v2/service/sso v1.17.3 h1:CdsSOGlFF3Pn+koXOIpTtvX7st0IuGsZ8kJqcWMlX54= +github.com/aws/aws-sdk-go-v2/service/sso v1.17.3/go.mod h1:oA6VjNsLll2eVuUoF2D+CMyORgNzPEW/3PyUdq6WQjI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.1 h1:cbRqFTVnJV+KRpwFl76GJdIZJKKCdTPnjUZ7uWh3pIU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.1/go.mod h1:hHL974p5auvXlZPIjJTblXJpbkfK4klBczlsEaMCGVY= +github.com/aws/aws-sdk-go-v2/service/sts v1.25.4 h1:yEvZ4neOQ/KpUqyR+X0ycUTW/kVRNR4nDZ38wStHGAA= +github.com/aws/aws-sdk-go-v2/service/sts v1.25.4/go.mod h1:feTnm2Tk/pJxdX+eooEsxvlvTWBvDm6CasRZ+JOs2IY= +github.com/aws/smithy-go v1.17.0 h1:wWJD7LX6PBV6etBUwO0zElG0nWN9rUhp0WdYeHSHAaI= +github.com/aws/smithy-go v1.17.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -15,28 +53,41 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4 github.com/getsentry/sentry-go v0.13.0 h1:20dgTiUSfxRB/EhMPtxcL9ZEbM1ZdR+W/7f7NWD+xWo= github.com/getsentry/sentry-go v0.13.0/go.mod h1:EOsfu5ZdvKPfeHYV6pTVQnsjfp30+XA7//UooKNumH0= github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -50,10 +101,30 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= @@ -66,85 +137,130 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s= +github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/slok/goresilience v0.2.0 h1:dagdIiWlhTm7BK/r/LRKz+zvw0SCNk+nHf7obdsbzxQ= github.com/slok/goresilience v0.2.0/go.mod h1:L6IqqHlxWGTrTyq8WwF8kUY8kOIESZAMWr1xkV0zdZA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= +go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo= go.uber.org/mock v0.3.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -154,18 +270,23 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/mysql v1.3.4 h1:/KoBMgsUHC3bExsekDcmNYaBnfH2WNeFuXqqrqMc98Q= gorm.io/driver/mysql v1.3.4/go.mod h1:s4Tq0KmD0yhPGHbZEwg1VPlH0vT/GBHJZorPzhcxBUE= gorm.io/gorm v1.23.4/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= diff --git a/insdash/insdash.go b/insdash/insdash.go new file mode 100644 index 0000000..42b4d48 --- /dev/null +++ b/insdash/insdash.go @@ -0,0 +1,52 @@ +package insdash + +import "encoding/json" + +func CreateBatches[T any](v []T, recordLimit int, byteLimit int) ([][]T, error) { + batches := make([][]T, 0) + buffer := make([]T, 0) + bufferSize := 0 + + for _, record := range v { + js, jsErr := json.Marshal(record) + if jsErr != nil { + return nil, jsErr + } + + recordSize := len(js) + sizeExceeds := bufferSize+recordSize > byteLimit + bufferFull := len(buffer) == recordLimit + + if len(buffer) > 0 && (bufferFull || sizeExceeds) { + batches = append(batches, buffer) + buffer = make([]T, 0) + bufferSize = 0 + } + + buffer = append(buffer, record) + bufferSize += recordSize + } + + if len(buffer) > 0 { + batches = append(batches, buffer) + } + + return batches, nil +} + +func Contains[T comparable](v []T, e T) bool { + for _, a := range v { + if a == e { + return true + } + } + return false +} + +func MapToValueSlice[K comparable, V any](m map[K]V) []V { + v := make([]V, 0, len(m)) + for _, value := range m { + v = append(v, value) + } + return v +} diff --git a/insdash/insdash_test.go b/insdash/insdash_test.go new file mode 100644 index 0000000..9e62f93 --- /dev/null +++ b/insdash/insdash_test.go @@ -0,0 +1,108 @@ +package insdash + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +const kb64 = 64 * 1024 + +func Test_createBatches(t *testing.T) { + t.Run("should_return_empty_batches_when_empty", func(t *testing.T) { + batches, err := CreateBatches([]string{}, 10, kb64) + + assert.Nil(t, err, "err should be nil") + assert.Equal(t, 0, len(batches), "batches length should be equal to 0") + }) + + t.Run("should_return_batches_when_not_empty", func(t *testing.T) { + batches, err := CreateBatches([]string{"test"}, 10, kb64) + + assert.Nil(t, err, "err should be nil") + assert.Equal(t, 1, len(batches), "batches length should be equal to 1") + }) + + t.Run("should_return_batches_by_record_limit", func(t *testing.T) { + batches, err := CreateBatches([]string{"test", "test"}, 10, kb64) + + assert.Nil(t, err, "err should be nil") + assert.Equal(t, 2, len(batches), "batches length should be equal to 2") + }) + + t.Run("should_return_batches_by_byte_limit", func(t *testing.T) { + batches, err := CreateBatches([]string{createString(kb64), createString(kb64)}, 10, kb64) + + assert.Nil(t, err, "err should be nil") + assert.Equal(t, 2, len(batches), "batches length should be equal to 2") + }) +} + +func createString(byteSize int) string { + var str string + for i := 0; i < byteSize; i++ { + str += "a" + } + return str +} + +func TestMapToValueSlice(t *testing.T) { + t.Run("should_return_empty_slice_when_empty", func(t *testing.T) { + s := MapToValueSlice(map[string]string{}) + assert.Equal(t, 0, len(s), "slice length should be equal to 0") + assert.Equal(t, []string{}, s, "slice value should be equal to []string{}") + }) + + t.Run("should_return_string_slice_when_not_empty", func(t *testing.T) { + s := MapToValueSlice(map[string]string{"test": "test"}) + assert.Equal(t, 1, len(s), "slice length should be equal to 1") + assert.Equal(t, "test", s[0], "slice value should be equal to \"test\"") + }) + + t.Run("should_return_int_slice_when_not_empty", func(t *testing.T) { + s := MapToValueSlice(map[string]int{"test": 1}) + assert.Equal(t, 1, len(s), "slice length should be equal to 1") + assert.Equal(t, 1, s[0], "slice value should be equal to 1") + }) + + t.Run("should_return_bool_slice_when_not_empty", func(t *testing.T) { + s := MapToValueSlice(map[string]bool{"test": true}) + assert.Equal(t, 1, len(s), "slice length should be equal to 1") + assert.Equal(t, true, s[0], "slice value should be equal to true") + }) + + t.Run("should_return_float_slice_when_not_empty", func(t *testing.T) { + s := MapToValueSlice(map[string]float64{"test": 1.0}) + assert.Equal(t, 1, len(s), "slice length should be equal to 1") + assert.Equal(t, 1.0, s[0], "slice value should be equal to 1.0") + }) + + t.Run("should_return_struct_slice_when_not_empty", func(t *testing.T) { + s := MapToValueSlice(map[string]struct{}{"test": {}}) + assert.Equal(t, 1, len(s), "slice length should be equal to 1") + assert.Equal(t, struct{}{}, s[0], "slice value should be equal to struct{}{}") + }) + + t.Run("should_return_interface_slice_when_not_empty", func(t *testing.T) { + s := MapToValueSlice(map[string]interface{}{"test": 1}) + assert.Equal(t, 1, len(s), "slice length should be equal to 1") + assert.Equal(t, 1, s[0], "slice value should be equal to 1") + }) + + t.Run("should_return_slice_when_not_empty", func(t *testing.T) { + s := MapToValueSlice(map[string][]string{"test": {"test"}}) + assert.Equal(t, 1, len(s), "slice length should be equal to 1") + assert.Equal(t, []string{"test"}, s[0], "slice value should be equal to []string{\"test\"}") + }) + + t.Run("should_return_map_slice_when_not_empty", func(t *testing.T) { + s := MapToValueSlice(map[string]map[string]string{"test": {"test": "test"}}) + assert.Equal(t, 1, len(s), "slice length should be equal to 1") + assert.Equal(t, map[string]string{"test": "test"}, s[0], "slice value should be equal to map[string]string{\"test\": \"test\"}") + }) + + t.Run("should_return_slice_with_multiple_values_when_not_empty", func(t *testing.T) { + s := MapToValueSlice(map[string]string{"test": "test", "test2": "test2"}) + assert.Equal(t, 2, len(s), "slice length should be equal to 2") + assert.Equal(t, []string{"test", "test2"}, s, "slice value should be equal to []string{\"test\", \"test2\"}") + }) +} diff --git a/inslogger/README.md b/inslogger/README.md new file mode 100644 index 0000000..ee17f0a --- /dev/null +++ b/inslogger/README.md @@ -0,0 +1,103 @@ +# Inslogger + +Inslogger is a logging package designed to provide flexible and comprehensive logging capabilities in Go applications. + +## Features + +- **Multi-Level Logging:** Supports logging at various levels including Debug, Info, Warn, Error, and Fatal. +- **Error Handling:** Provides methods to log errors with different formatting options. + +## Installation + +```bash +go get github.com/yourusername/inslogger +``` + +## Usage + +```go +import "github.com/yourusername/inslogger" + +// Initialize a logger +logger := inslogger.NewLogger(inslogger.Debug) + +// Log an error +err := someFunction() +if err != nil { + logger.Error(err) +} + +// Log a message +logger.Log("This is a log message") + +// Set log level +logger.SetLevel(inslogger.Info) +``` + +## Interface +The package provides an Interface that exposes various logging methods: +```go +type Interface interface { + LogMultiple(errs []error) + Log(i interface{}) + Logf(format string, args ...interface{}) + Warn(i interface{}) + Warnf(format string, args ...interface{}) + Error(err error) + Errorf(format string, args ...interface{}) + Debug(i interface{}) + Debugf(format string, args ...interface{}) + Fatal(err error) + Fatalf(format string, args ...interface{}) + initLogger() error + SetLevel(level LogLevel) +} +``` + + +# Configuration + +The `inslogger` package offers configuration options to customize logging behavior. + +## Log Levels + +The package supports the following log levels: + +- `DEBUG` +- `INFO` +- `WARN` +- `ERROR` +- `FATAL` + +## Setting Log Level + +The default log level is `INFO`. To set a different log level: + +```go +logger := inslogger.NewLogger(inslogger.Debug) +``` + +## Example +```go +// Initialize a logger with Debug level +logger := inslogger.NewLogger(inslogger.Debug) + +// Log an error +err := someFunction() +if err != nil { + logger.Error(err) +} + +// Log a debug message +logger.Debugf("Debug message with arguments: %s", arg) + +// Change log level to Info +logger.SetLevel(inslogger.Info) + +// Log an informational message +logger.Infof("Informational message") + +``` +## Contribution +Feel free to contribute by forking this repository and creating pull requests. Please ensure to adhere to the existing code style and conventions. + diff --git a/inslogger/inslogger.go b/inslogger/inslogger.go new file mode 100644 index 0000000..d6cc400 --- /dev/null +++ b/inslogger/inslogger.go @@ -0,0 +1,161 @@ +package inslogger + +import ( + "go.uber.org/zap" + "log" +) + +type Interface interface { + LogMultiple(errs []error) + Log(i interface{}) + Logf(format string, args ...interface{}) + Warn(i interface{}) + Warnf(format string, args ...interface{}) + Error(err error) + Errorf(format string, args ...interface{}) + Debug(i interface{}) + Debugf(format string, args ...interface{}) + Fatal(err error) + Fatalf(format string, args ...interface{}) + initLogger() error + SetLevel(level LogLevel) +} +type LogLevel string + +const ( + Debug LogLevel = "DEBUG" + Info LogLevel = "INFO" + Warn LogLevel = "WARN" + Error LogLevel = "ERROR" + Fatal LogLevel = "FATAL" +) + +func (ll LogLevel) toZapLevel() zap.AtomicLevel { + switch ll { + case Debug: + return zap.NewAtomicLevelAt(zap.DebugLevel) + case Info: + return zap.NewAtomicLevelAt(zap.InfoLevel) + case Warn: + return zap.NewAtomicLevelAt(zap.WarnLevel) + case Error: + return zap.NewAtomicLevelAt(zap.ErrorLevel) + case Fatal: + return zap.NewAtomicLevelAt(zap.FatalLevel) + default: + return zap.NewAtomicLevelAt(zap.InfoLevel) + } +} + +type AppLogger struct { + Logger *zap.Logger + Sugar *zap.SugaredLogger + + Level LogLevel +} + +func NewLogger(level LogLevel) Interface { + al := &AppLogger{ + Logger: nil, + Sugar: nil, + Level: level, + } + if err := al.initLogger(); err != nil { + log.Fatalf("zap.Init: %+v", err) + } + + return al +} + +func NewNopLogger() Interface { + return &AppLogger{ + Logger: zap.NewNop(), + Sugar: zap.NewNop().Sugar(), + } +} + +func (al *AppLogger) LogMultiple(errs []error) { + for _, err := range errs { + al.Sugar.Infof("%v+", err) + } +} + +func (al *AppLogger) Log(i interface{}) { + al.Sugar.Infof("%s", i) +} + +func (al *AppLogger) Logf(format string, args ...interface{}) { + al.Sugar.Infof(format, args...) +} + +func (al *AppLogger) Warn(i interface{}) { + al.Sugar.Warnf("%s", i) +} + +func (al *AppLogger) Warnf(format string, args ...interface{}) { + al.Sugar.Warnf(format, args...) +} + +func (al *AppLogger) Error(err error) { + al.Sugar.Errorf("%v+", err) +} + +func (al *AppLogger) Errorf(format string, args ...interface{}) { + al.Sugar.Errorf(format, args...) +} + +func (al *AppLogger) Debug(i interface{}) { + al.Sugar.Debugf("%s", i) +} + +func (al *AppLogger) Debugf(format string, args ...interface{}) { + al.Sugar.Debugf(format, args...) +} + +func (al *AppLogger) Fatal(err error) { + al.Sugar.Fatalf("log.Fatal: %+v\n", err) +} + +func (al *AppLogger) Fatalf(format string, args ...interface{}) { + al.Sugar.Fatalf(format, args...) +} + +func (al *AppLogger) initLogger() error { + var ( + newLogger *zap.Logger + err error + ) + + newLogger, err = zap.NewProduction() + if err != nil { + return err + } + + switch al.Level { + case Debug: + newLogger, err = zap.NewDevelopment() + if err != nil { + return err + } + newLogger = newLogger.WithOptions(zap.IncreaseLevel(zap.DebugLevel)) + case Info: + newLogger = newLogger.WithOptions(zap.IncreaseLevel(zap.InfoLevel)) + case Warn: + newLogger = newLogger.WithOptions(zap.IncreaseLevel(zap.WarnLevel)) + case Error: + newLogger = newLogger.WithOptions(zap.IncreaseLevel(zap.ErrorLevel)) + case Fatal: + newLogger = newLogger.WithOptions(zap.IncreaseLevel(zap.FatalLevel)) + default: + newLogger = newLogger.WithOptions(zap.IncreaseLevel(zap.InfoLevel)) + } + + al.Logger = newLogger + al.Sugar = newLogger.Sugar() + + return nil +} + +func (al *AppLogger) SetLevel(level LogLevel) { + al.Logger.WithOptions(zap.IncreaseLevel(level.toZapLevel())) +} diff --git a/insredis/README.md b/insredis/README.md index a5902a5..444fdc4 100644 --- a/insredis/README.md +++ b/insredis/README.md @@ -1,23 +1,64 @@ -# Redis Package +# insredis +The insredis package provides a universal Golang Redis interface and a client for easy interaction with Redis databases. -This is a simple mockable wrapper for Redis. +## Overview -## Usage in Apps -```go -package main +This package offers a set of methods covering various functionalities for working with Redis, encapsulated within the +RedisInterface. It includes methods for key-value operations, set operations, list operations, sorted set operations, +geospatial operations, and more. + +## Installation + +To use this package, install it using Go modules: + +```bash +go get github.com/useinsider/insredis +``` + +## Usage -import "github.com/useinsider/go-pkg/insredis" +### Initialization + +Initialize a Redis client instance by providing configuration settings using `Init()` + +```go +import ( +"github.com/yourusername/insredis" +"time" +) func main() { - redisClient := insredis.Init("localhost:6379", 100) +// Configure Redis settings +cfg := insredis.Config{ +RedisHost: "localhost:6379", +RedisPoolSize: 10, +DialTimeout: 500 * time.Millisecond, +ReadTimeout: 500 * time.Millisecond, +MaxRetries: 3, +} - err := redisClient.Ping() - if err != nil { - panic(err) - } +// Initialize the Redis client +client := insredis.Init(cfg) + +// Use the client for Redis operations +// e.g., client.Set("key", "value", 0) } -``` + + + + +```` + +// Default configs in table format + +| Config Name | Default Value | Description | +|---------------|----------------|-------------------------------------------------------------------| +| RedisHost | localhost:6379 | Redis host address | +| RedisPoolSize | 10 | Maximum number of connections allocated by the Redis client | +| DialTimeout | 500ms | Maximum amount of time to wait for a connection to be established | +| ReadTimeout | 500ms | Maximum amount of time to wait for a read operation to complete | +| MaxRetries | 0 (no retry) | Maximum number of retries before giving up on a request | ## Usage in Tests @@ -54,6 +95,12 @@ func ping(redisClient insredis.RedisInterface) error { ``` ## How to Update Mock File + ``` mockgen -source=./insredis/redis.go -destination=./insredis/redis_mock.go -package=insredis ``` + +## Contributing + +Contributions to this package are welcome! Feel free to submit issues or pull requests. + diff --git a/insredis/redis.go b/insredis/redis.go index 7e09c8f..73e41ee 100644 --- a/insredis/redis.go +++ b/insredis/redis.go @@ -260,15 +260,38 @@ type RedisInterface interface { // redisClient is the singleton client created by InitRedis function. var redisClient *redis.Client +type Config struct { + RedisHost string + RedisPoolSize int + DialTimeout time.Duration + ReadTimeout time.Duration + MaxRetries int +} + // Init creates a client pool for redis connections. -func Init(redisHost string, poolSize int) *redis.Client { +func Init(cfg Config) *redis.Client { if redisClient != nil { return redisClient } + if cfg.RedisPoolSize == 0 { + cfg.RedisPoolSize = 10 + } + + if cfg.DialTimeout == 0 { + cfg.DialTimeout = 500 * time.Millisecond + } + + if cfg.ReadTimeout == 0 { + cfg.ReadTimeout = 500 * time.Millisecond + } + redisClient = redis.NewClient(&redis.Options{ - Addr: redisHost, - PoolSize: poolSize, + Addr: cfg.RedisHost, + PoolSize: cfg.RedisPoolSize, + DialTimeout: cfg.DialTimeout, + ReadTimeout: cfg.ReadTimeout, + MaxRetries: cfg.MaxRetries, }) return redisClient diff --git a/inssentry/sentry.go b/inssentry/sentry.go index 877a1f5..914339a 100644 --- a/inssentry/sentry.go +++ b/inssentry/sentry.go @@ -1,8 +1,8 @@ package inssentry import ( - "fmt" "github.com/getsentry/sentry-go" + "log" "time" ) @@ -37,7 +37,7 @@ func Flush() { // Error sends the error to sentry. func Error(err error) { if !cachedSettings.IsProduction { - fmt.Println(err) + log.Println(err) return } @@ -45,6 +45,20 @@ func Error(err error) { sentry.CaptureException(err) } +// ErrorWithAdditionalData sends the error to sentry with additional data. +func ErrorWithAdditionalData(err error, key string, value interface{}) { + if !cachedSettings.IsProduction { + log.Println(err) + return + } + + sentry.WithScope(func(scope *sentry.Scope) { + scope.SetExtra(key, value) + + sentry.CaptureException(err) + }) +} + // Fatal sends the error to sentry and exit from the program. func Fatal(err error) { if !cachedSettings.IsProduction { diff --git a/inssqs/README.md b/inssqs/README.md new file mode 100644 index 0000000..7bd3e24 --- /dev/null +++ b/inssqs/README.md @@ -0,0 +1,82 @@ +# inssqs - AWS SQS Package + +The `inssqs` package provides an interface and implementation for interacting with Amazon Simple Queue Service (SQS) in Go. It simplifies sending and deleting messages in batches while handling retries, respecting batch size constraints, and supporting concurrent operations. + +## Features + +- **Batch Operations**: Send and delete messages in batches to SQS queues. +- **Retries**: Automatic retries for failed operations based on a configurable retry count. +- **Concurrency**: Concurrent processing of multiple batches with specified worker count. +- **Error Handling**: Detailed error logging and handling for failed operations. + +## Installation + +To use this package in your Go project, import it as follows: + +```go +import "github.com/useinsider/go-pkg/inssqs" +``` + +## Usage +### Initialization +Initialize an SQS queue instance by providing configuration settings using `NewSQS()` +```go +config := inssqs.Config{ + Region: "your-aws-region", + QueueName: "your-queue-name", + RetryCount: 3, + MaxBatchSize: 10, + MaxBatchSizeBytes: 1024, + MaxWorkers: 5, + LogLevel: "info", +} + +sqs := inssqs.NewSQS(config) +``` +### Sending Messages +Send a batch of messages to an SQS queue using `SendMessageBatch()` + +```go +messages := []inssqs.SQSMessageEntry{ + // Create SQS message entries here +} + +failedMessages, err := sqs.SendMessageBatch(messages) +if err != nil { + // Handle error +} +``` + +### Deleting Messages +Delete a batch of messages from an SQS queue using `DeleteMessageBatch()` + +```go +messages := []inssqs.SQSDeleteMessageEntry{ + // Create SQS message entries here +} + +failedMessages, err := sqs.DeleteMessageBatch(messages) +if err != nil { + // Handle error +} +``` + +## Configuration Options +- Region: AWS region where the SQS queue resides. +- QueueName: Name of the SQS queue. +- RetryCount: Number of retry attempts allowed for queue operations. +- MaxBatchSize: Maximum size of a message batch. +- MaxBatchSizeBytes: Maximum size of a message batch in bytes. +- MaxWorkers: Maximum number of workers for concurrent operations. +- LogLevel: Log level for SQS operations. +For more details on each function and its parameters, refer to the code documentation. + + +## Contribution +Feel free to contribute by forking this repository and creating pull requests. Please ensure to adhere to the existing code style and conventions. + + + + + + diff --git a/inssqs/errors.go b/inssqs/errors.go new file mode 100644 index 0000000..4342704 --- /dev/null +++ b/inssqs/errors.go @@ -0,0 +1,7 @@ +package inssqs + +import "github.com/pkg/errors" + +var ErrRetryCountExceeded = errors.New("retry count exceeded") +var ErrRegionNotSet = errors.New("region not set") +var ErrQueueNameNotSet = errors.New("queue name not set") diff --git a/inssqs/inssqs.go b/inssqs/inssqs.go new file mode 100644 index 0000000..7eecab8 --- /dev/null +++ b/inssqs/inssqs.go @@ -0,0 +1,411 @@ +package inssqs + +import ( + "context" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/retry" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + awssqs "github.com/aws/aws-sdk-go-v2/service/sqs" + "github.com/aws/aws-sdk-go-v2/service/sqs/types" + "github.com/aws/smithy-go/middleware" + "github.com/pkg/errors" + "github.com/useinsider/go-pkg/insdash" + "github.com/useinsider/go-pkg/inslogger" + "github.com/useinsider/go-pkg/inssqs/sqs" + "sync" +) + +type Interface interface { + SendMessageBatch(entries []SQSMessageEntry) (failed []SQSMessageEntry, err error) + DeleteMessageBatch(entries []SQSDeleteMessageEntry) (failed []SQSDeleteMessageEntry, err error) +} + +type queue struct { + client sqs.API + name string + url *string + endpoint string + + retryCount int + + maxBatchSize int + maxBatchSizeBytes int + workers int + + logger inslogger.Interface +} + +// Config represents the configuration settings required for initializing an SQS queue. +type Config struct { + Region string // AWS region where the SQS queue resides. + QueueName string // Name of the SQS queue. + RetryCount int // Number of retry attempts allowed for queue operations. + MaxBatchSize int // Maximum size of a message batch. + MaxBatchSizeBytes int // Maximum size of a message batch in bytes. + MaxWorkers int // Maximum number of workers for concurrent operations. + LogLevel string // Log level for SQS operations. + + EndpointUrl string // Endpoint URL for AWS operations. +} + +func NewSQS(config Config) Interface { + if config.Region == "" { + panic(ErrRegionNotSet) + } + + if config.QueueName == "" { + panic(ErrQueueNameNotSet) + } + + config.setDefaults() + + cfg, err := awsconfig.LoadDefaultConfig(context.Background(), + awsconfig.WithRegion(config.Region), + awsconfig.WithRetryMaxAttempts(config.RetryCount), + awsconfig.WithRetryMode(aws.RetryModeAdaptive)) + if err != nil { + panic(errors.Wrap(err, "error while loading aws sqs config")) + } + + // set endpoint url if provided + if config.EndpointUrl != "" { + cfg.BaseEndpoint = aws.String(config.EndpointUrl) + } + + var logger inslogger.Interface + if config.LogLevel == "" { + logger = inslogger.NewNopLogger() + } else { + logger = inslogger.NewLogger(inslogger.LogLevel(config.LogLevel)) + } + + q := &queue{ + client: sqs.NewSQSProxy(awssqs.NewFromConfig(cfg)), + name: config.QueueName, + retryCount: config.RetryCount, + workers: config.MaxWorkers, + logger: logger, + maxBatchSize: config.MaxBatchSize, + maxBatchSizeBytes: config.MaxBatchSizeBytes, + } + + qUrl, err := q.getQueueUrl() + if err != nil { + panic(errors.Wrap(err, "error while getting queue url")) + } + + q.url = qUrl + + return q +} + +func (c *Config) setDefaults() { + if c.MaxBatchSize == 0 { + c.MaxBatchSize = 10 + } + + if c.MaxBatchSizeBytes == 0 { + c.MaxBatchSizeBytes = 64 * 1024 + } + + if c.MaxWorkers == 0 { + c.MaxWorkers = 1 + } + + if c.RetryCount == 0 { + c.RetryCount = 3 + } +} + +// SendMessageBatch sends a batch of messages to an SQS queue, handling retries and respecting batch size constraints. +// +// Parameters: +// - entries: A slice of SQSMessageEntry representing messages to be sent in batches to the SQS queue. +// +// Returns: +// - failedEntries: A slice of SQSMessageEntry containing the messages that failed to be sent after all attempts. +// - err: An error indicating any failure during the sending process, nil if all messages were sent successfully. +// +// Note: +// SendMessageBatch operation has an inherent concurrency limit. +// When multiple concurrent calls reach the maximum workers, the total worker count might exceed expectations. +// Consider this while designing applications for optimal performance. +func (q *queue) SendMessageBatch(entries []SQSMessageEntry) ([]SQSMessageEntry, error) { + // TODO: make concurrency as optional. + batches, err := insdash.CreateBatches(entries, q.maxBatchSize, q.maxBatchSizeBytes) + if err != nil { + return entries, err + } + + failedEntries, err := q.sendBatchesConcurrently(batches) + if err != nil { + q.logger.Errorf("Error sending %d messages to SQS: %v\n", len(failedEntries), err) + return failedEntries, err + } + + return failedEntries, nil +} + +// DeleteMessageBatch attempts to delete a batch of messages from an SQS queue using the provided entries, +// handling retries on failure and respecting the specified retry count. +// +// Parameters: +// - entries: A slice of SQSDeleteMessageEntry representing messages to be deleted in batches from the SQS queue. +// +// Returns: +// - failedEntries: A slice of SQSDeleteMessageEntry containing the messages that failed to be deleted after all attempts. +// - err: An error indicating any failure during the deletion process, nil if all messages were deleted successfully. +// +// DeleteMessageBatch operation has an inherent concurrency limit. +// When multiple concurrent calls reach the maximum workers, the total worker count might exceed expectations. +// Consider this while designing applications for optimal performance. +func (q *queue) DeleteMessageBatch(entries []SQSDeleteMessageEntry) (failed []SQSDeleteMessageEntry, err error) { + // TODO: make concurrency as optional. + batches, err := insdash.CreateBatches(entries, q.maxBatchSize, q.maxBatchSizeBytes) + if err != nil { + return entries, err + } + + failedEntries, err := q.deleteBatchesConcurrently(batches) + if err != nil { + q.logger.Errorf("Error deleting %d messages from SQS: %v\n", len(failedEntries), err) + return failedEntries, err + } + + return failedEntries, nil +} + +// sendBatchesConcurrently concurrently processes multiple batches of send operations +// on an SQS queue using a specified number of workers and retry logic. +// +// Parameters: +// - batches: A slice of slices, each containing SQSMessageEntry representing send operations in batches. +// +// Returns: +// - failedEntries: A slice of SQSMessageEntry containing the failed send operations across all batches. +// - err: An error indicating any failure during the concurrent sending process, nil if all operations succeeded. +func (q *queue) sendBatchesConcurrently(batches [][]SQSMessageEntry) ([]SQSMessageEntry, error) { + failedEntries, err := doConcurrently(batches, q.workers, q.retryCount, q.sendMessageBatch) + + return failedEntries, err +} + +// deleteBatchesConcurrently concurrently processes multiple batches of delete operations +// on an SQS queue using a specified number of workers and retry logic. +// +// Parameters: +// - batches: A slice of slices, each containing SQSDeleteMessageEntry representing delete operations in batches. +// +// Returns: +// - failedEntries: A slice of SQSDeleteMessageEntry containing the failed delete operations across all batches. +// - err: An error indicating any failure during the concurrent deletion process, nil if all operations succeeded. +func (q *queue) deleteBatchesConcurrently(batches [][]SQSDeleteMessageEntry) ([]SQSDeleteMessageEntry, error) { + failedEntries, err := doConcurrently(batches, q.workers, q.retryCount, q.deleteMessageBatch) + if err != nil { + return nil, err + } + + return failedEntries, err +} + +// deleteMessageBatch attempts to delete a batch of messages from an SQS queue, handling retries on failure. +// +// Parameters: +// - entries: A slice of SQSDeleteMessageEntry representing messages to be deleted in batches from the SQS queue. +// - retryCount: An integer indicating the number of retry attempts allowed for deleting the messages. +// +// Returns: +// - failedEntries: A slice of SQSDeleteMessageEntry containing the messages that failed to be deleted after all attempts. +// - err: An error indicating any failure during the deletion process, nil if all messages were deleted successfully. +func (q *queue) deleteMessageBatch(entries []SQSDeleteMessageEntry, retryCount int) ([]SQSDeleteMessageEntry, error) { + if len(entries) == 0 { + return nil, nil + } + + if retryCount == 0 { + return entries, ErrRetryCountExceeded + } + + batchEntries := make([]types.DeleteMessageBatchRequestEntry, len(entries)) + for i, e := range entries { + batchEntries[i] = e.toDeleteMessageBatchRequestEntry() + } + + batch := &awssqs.DeleteMessageBatchInput{ + Entries: batchEntries, + QueueUrl: q.url, + } + + res, err := q.client.DeleteMessageBatch(context.Background(), batch) + if err != nil { + q.logger.Errorf("Error deleting %d messages to SQS: %v\n", len(entries), err) + return q.deleteMessageBatch(entries, retryCount-1) + } + + attempts := getRequestAttemptCount(res.ResultMetadata) + + if len(res.Failed) == 0 { + q.logger.Logf("Successfully deleted %d messages to SQS after %d attempts\n", len(entries), attempts) + + return nil, nil + } + + failedEntries := getFailedEntries(entries, res.Failed) + q.logger.Logf("Failed to delete %d messages to SQS after %d attempts\n", len(failedEntries), attempts) + + return q.deleteMessageBatch(failedEntries, retryCount-1) +} + +// sendMessageBatch sends a batch of SQS messages in multiple attempts based on batch size and size constraints. +// +// Parameters: +// - entries: A slice of SQSMessageEntry representing messages to be sent in batches to the SQS queue. +// +// Returns: +// - failedEntries: A slice of SQSMessageEntry containing the messages that failed to be sent after all attempts. +// - err: An error indicating any failure during the sending process, nil if all messages were sent successfully. +func (q *queue) sendMessageBatch(entries []SQSMessageEntry, retryCount int) (failed []SQSMessageEntry, err error) { + if len(entries) == 0 { + return nil, nil + } + + if retryCount == 0 { + return entries, ErrRetryCountExceeded + } + + batchEntries := make([]types.SendMessageBatchRequestEntry, len(entries)) + for i, e := range entries { + batchEntries[i] = e.toSendMessageBatchRequestEntry() + } + + batch := &awssqs.SendMessageBatchInput{ + Entries: batchEntries, + QueueUrl: q.url, + } + + res, err := q.client.SendMessageBatch(context.Background(), batch) + if err != nil { + q.logger.Errorf("Error sending %d messages to SQS: %v\n", len(entries), err) + return q.sendMessageBatch(entries, retryCount-1) + } + + attempts := getRequestAttemptCount(res.ResultMetadata) + + if len(res.Failed) == 0 { + q.logger.Logf("Sent %d messages to SQS after %d attempts\n", len(entries), attempts) + return nil, nil + } + + failedEntries := getFailedEntries(entries, res.Failed) + + return q.sendMessageBatch(failedEntries, retryCount-1) +} + +// getQueueUrl retrieves the URL of an SQS queue based on its name using the provided SQS client. +// +// This function fetches the queue URL if it's not already cached within the 'queue' instance. +// +// Returns: +// - queueUrl: A pointer to a string containing the URL of the SQS queue. +// - err: An error if fetching the queue URL fails, nil otherwise. +func (q *queue) getQueueUrl() (queueUrl *string, err error) { + if q.url != nil { + return q.url, nil + } + + res, err := q.client.GetQueueUrl(context.Background(), &awssqs.GetQueueUrlInput{ + QueueName: aws.String(q.name), + }) + if err != nil { + return nil, err + } + + q.url = res.QueueUrl + + return res.QueueUrl, nil +} + +// getFailedEntries retrieves failed elements by matching their IDs from a collection. +// Returns a slice of elements corresponding to the failed IDs, maintaining the original order. +func getFailedEntries[T entry](entries []T, failed []types.BatchResultErrorEntry) []T { + failedEntries := make([]T, len(failed)) + for i, f := range failed { + for _, e := range entries { + if *e.getId() == *f.Id { + failedEntries[i] = e + } + } + } + + return failedEntries +} + +// getRequestAttemptCount retrieves the number of attempts from AWS SDK request metadata. +// Returns the attempt counts or -1 if not found. +func getRequestAttemptCount(metadata middleware.Metadata) int { + attempts, ok := retry.GetAttemptResults(metadata) + if !ok { + return -1 + } + + return len(attempts.Results) +} + +// doConcurrently executes a function concurrently across multiple batches of elements of type T, +// managing parallel processing with specified worker count and retry logic. +// +// Parameters: +// - batches: A slice of slices, each containing elements of type T. Represents the batches +// of operations to be processed concurrently. +// - workers: An integer defining the maximum number of goroutines (workers) allowed to run +// simultaneously for executing the provided function. +// - retryCount: An integer indicating the number of retry attempts permitted for each batch operation. +// - f: A function that processes a single batch of elements of type T, given a retry count. +// It takes two parameters: +// - A slice of elements of type T to be processed. +// - An integer representing the retry count for the operation. +// It returns two values: +// - A slice of elements of type T that failed during processing. +// - An error indicating any error occurred during the processing. +// +// Returns: +// - failedEntries: A slice containing all the failed entries encountered during the concurrent operations +// across all batches. +// - outerErr: An error variable that holds any errors encountered during the concurrent execution. +// It remains nil if no errors occurred during the execution. +func doConcurrently[T any](batches [][]T, workers int, retryCount int, f func([]T, int) ([]T, error)) ([]T, error) { + if len(batches) == 0 { + return nil, nil + } + + concurrentLimiter := make(chan struct{}, workers) + wg := sync.WaitGroup{} + var outerErr error + failedEntriesChan := make(chan []T) + + for _, batch := range batches { + wg.Add(1) + go func(b []T) { + defer wg.Done() + concurrentLimiter <- struct{}{} + defer func() { <-concurrentLimiter }() + fe, err := f(b, retryCount+1) + if err != nil { + outerErr = err + } + failedEntriesChan <- fe + }(batch) + } + + go func() { + wg.Wait() + close(failedEntriesChan) + }() + + var failedEntries []T + for failedBatch := range failedEntriesChan { + failedEntries = append(failedEntries, failedBatch...) + } + + return failedEntries, outerErr +} diff --git a/inssqs/inssqs_fake.go b/inssqs/inssqs_fake.go new file mode 100644 index 0000000..f51ff78 --- /dev/null +++ b/inssqs/inssqs_fake.go @@ -0,0 +1,25 @@ +package inssqs + +type FakeQueue struct { + Interface + Data []SQSMessageEntry +} + +func (q *FakeQueue) SendMessageBatch(entries []SQSMessageEntry) (failed []SQSMessageEntry, err error) { + q.Data = append(q.Data, entries...) + return nil, nil +} + +func (q *FakeQueue) DeleteMessageBatch(entries []SQSDeleteMessageEntry) (failed []SQSDeleteMessageEntry, err error) { + newData := make([]SQSMessageEntry, 0) + for _, e := range q.Data { + for _, de := range entries { + if e.Id != de.Id { + newData = append(newData, e) + } + } + } + + q.Data = newData + return nil, nil +} diff --git a/inssqs/inssqs_mock.go b/inssqs/inssqs_mock.go new file mode 100644 index 0000000..97b4dd0 --- /dev/null +++ b/inssqs/inssqs_mock.go @@ -0,0 +1,68 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: inssqs.go +// +// Generated by this command: +// +// mockgen --source=inssqs.go +// +// Package mock_inssqs is a generated GoMock package. +package inssqs + +import ( + reflect "reflect" + + gomock "go.uber.org/mock/gomock" +) + +// MockInterface is a mock of Interface interface. +type MockInterface struct { + ctrl *gomock.Controller + recorder *MockInterfaceMockRecorder +} + +// MockInterfaceMockRecorder is the mock recorder for MockInterface. +type MockInterfaceMockRecorder struct { + mock *MockInterface +} + +// NewMockInterface creates a new mock instance. +func NewMockInterface(ctrl *gomock.Controller) *MockInterface { + mock := &MockInterface{ctrl: ctrl} + mock.recorder = &MockInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder { + return m.recorder +} + +// DeleteMessageBatch mocks base method. +func (m *MockInterface) DeleteMessageBatch(entries []SQSDeleteMessageEntry) ([]SQSDeleteMessageEntry, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteMessageBatch", entries) + ret0, _ := ret[0].([]SQSDeleteMessageEntry) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteMessageBatch indicates an expected call of DeleteMessageBatch. +func (mr *MockInterfaceMockRecorder) DeleteMessageBatch(entries any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteMessageBatch", reflect.TypeOf((*MockInterface)(nil).DeleteMessageBatch), entries) +} + +// SendMessageBatch mocks base method. +func (m *MockInterface) SendMessageBatch(entries []SQSMessageEntry) ([]SQSMessageEntry, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendMessageBatch", entries) + ret0, _ := ret[0].([]SQSMessageEntry) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// SendMessageBatch indicates an expected call of SendMessageBatch. +func (mr *MockInterfaceMockRecorder) SendMessageBatch(entries any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessageBatch", reflect.TypeOf((*MockInterface)(nil).SendMessageBatch), entries) +} diff --git a/inssqs/inssqs_test.go b/inssqs/inssqs_test.go new file mode 100644 index 0000000..0cba566 --- /dev/null +++ b/inssqs/inssqs_test.go @@ -0,0 +1,191 @@ +package inssqs + +import ( + "github.com/aws/aws-sdk-go-v2/aws" + awssqs "github.com/aws/aws-sdk-go-v2/service/sqs" + "github.com/aws/aws-sdk-go-v2/service/sqs/types" + "github.com/aws/smithy-go/middleware" + "github.com/stretchr/testify/assert" + "github.com/useinsider/go-pkg/inslogger" + "github.com/useinsider/go-pkg/inssqs/sqs" + "go.uber.org/mock/gomock" + "testing" +) + +func TestQueue_SendMessageBatch(t *testing.T) { + t.Run("should_failed_records_as_nil_when_successful", func(t *testing.T) { + q, client := newQueue(t) + + client.EXPECT().SendMessageBatch(gomock.Any(), gomock.Any(), gomock.Any()).Return(&awssqs.SendMessageBatchOutput{ + Failed: nil, + Successful: []types.SendMessageBatchResultEntry{ + {Id: aws.String("test-id")}, + }, + ResultMetadata: middleware.Metadata{}, + }, nil) + + failed, err := q.SendMessageBatch([]SQSMessageEntry{ + {Id: aws.String("test-id")}, + }) + + assert.Nil(t, err, "err should be nil") + assert.Nil(t, failed, "failed should be nil") + }) + + t.Run("should_return_failed_records_when_failed", func(t *testing.T) { + q, client := newQueue(t) + + client.EXPECT(). + SendMessageBatch(gomock.Any(), gomock.Any(), gomock.Any()). + Times(4). + Return(&awssqs.SendMessageBatchOutput{ + Failed: []types.BatchResultErrorEntry{ + {Id: aws.String("test-id")}, + }, + Successful: nil, + ResultMetadata: middleware.Metadata{}, + }, nil) + + failed, err := q.SendMessageBatch([]SQSMessageEntry{ + {Id: aws.String("test-id")}, + }) + + assert.NotNil(t, err, "err should not be nil") + assert.NotNil(t, failed, "failed should not be nil") + assert.Equal(t, failed[0].Id, aws.String("test-id"), "failed id should be equal to test-id") + }) + + t.Run("should_return_error_when_failed", func(t *testing.T) { + q, client := newQueue(t) + + client.EXPECT(). + SendMessageBatch(gomock.Any(), gomock.Any(), gomock.Any()). + Return(&awssqs.SendMessageBatchOutput{ + Failed: []types.BatchResultErrorEntry{ + {Id: aws.String("test-id")}, + }}, assert.AnError). + Times(4) + + failed, err := q.SendMessageBatch([]SQSMessageEntry{ + {Id: aws.String("test-id")}, + }) + + assert.Error(t, err, "err should not be nil") + assert.NotNil(t, failed, "failed should not be nil") + assert.Equal(t, failed[0].Id, aws.String("test-id"), "failed id should be equal to test-id") + + }) + +} + +func TestQueue_DeleteMessageBatch(t *testing.T) { + t.Run("should_return_nil_when_successful", func(t *testing.T) { + q, client := newQueue(t) + + client.EXPECT(). + DeleteMessageBatch(gomock.Any(), gomock.Any(), gomock.Any()). + Return(&awssqs.DeleteMessageBatchOutput{ + Failed: nil, + }, nil) + + failed, err := q.DeleteMessageBatch([]SQSDeleteMessageEntry{ + {Id: aws.String("test-id")}, + }) + + assert.Nil(t, err, "err should be nil") + assert.Nil(t, failed, "failed should be nil") + }) + + t.Run("should_return_failed_records_when_failed", func(t *testing.T) { + q, client := newQueue(t) + + client.EXPECT(). + DeleteMessageBatch(gomock.Any(), gomock.Any(), gomock.Any()). + Times(4). + Return(&awssqs.DeleteMessageBatchOutput{ + Failed: []types.BatchResultErrorEntry{ + {Id: aws.String("test-id")}, + }, + }, nil) + + failed, err := q.DeleteMessageBatch([]SQSDeleteMessageEntry{ + {Id: aws.String("test-id")}, + }) + + assert.Nil(t, err, "err should be nil") + assert.NotNil(t, failed, "failed should not be nil") + assert.Equal(t, failed[0].Id, aws.String("test-id"), "failed id should be equal to test-id") + }) +} + +func TestQueue_getQueueUrl(t *testing.T) { + t.Run("should_return_queue_url_when_successful", func(t *testing.T) { + q, client := newQueue(t) + + client.EXPECT(). + GetQueueUrl(gomock.Any(), gomock.Any(), gomock.Any()). + Return(&awssqs.GetQueueUrlOutput{ + QueueUrl: aws.String("test-queue-url"), + }, nil) + + queueUrl, err := q.getQueueUrl() + + assert.Nil(t, err, "err should be nil") + assert.Equal(t, queueUrl, aws.String("test-queue-url"), "queue url should be equal to test-queue-url") + + t.Run("should_return_queue_url_from_cache_when_called_twice", func(_ *testing.T) { + client.EXPECT().GetQueueUrl(gomock.Any(), gomock.Any(), gomock.Any()).Times(0) // should not be called + + queueUrl, err := q.getQueueUrl() + + assert.Nil(t, err, "err should be nil") + assert.Equal(t, queueUrl, aws.String("test-queue-url"), "queue url should be equal to test-queue-url") + }) + }) + + t.Run("should_return_error_when_failed", func(t *testing.T) { + q, client := newQueue(t) + + client.EXPECT(). + GetQueueUrl(gomock.Any(), gomock.Any(), gomock.Any()). + Return(nil, assert.AnError) + + queueUrl, err := q.getQueueUrl() + + assert.Error(t, err, "err should not be nil") + assert.Nil(t, queueUrl, "queue url should be nil") + }) +} + +func Test_getFailedEntries(t *testing.T) { + t.Run("should_return_failed_entries", func(t *testing.T) { + entries := []SQSMessageEntry{ + {Id: aws.String("test-id-1")}, + {Id: aws.String("test-id-2")}, + {Id: aws.String("test-id-3")}, + } + + failed := []types.BatchResultErrorEntry{ + {Id: aws.String("test-id-1")}, + {Id: aws.String("test-id-3")}, + } + + failedEntries := getFailedEntries(entries, failed) + + assert.Len(t, failedEntries, 2, "failed entries length should be 2") + assert.Equal(t, failedEntries[0].Id, aws.String("test-id-1")) + assert.Equal(t, failedEntries[1].Id, aws.String("test-id-3")) + }) +} + +func newQueue(t *testing.T) (queue, *sqs.MockAPI) { + ctrl := gomock.NewController(t) + client := sqs.NewMockSQS(ctrl) + + return queue{ + client: client, + name: "test-queue", + retryCount: 3, + logger: inslogger.NewLogger(inslogger.Debug), + }, client +} diff --git a/inssqs/model.go b/inssqs/model.go new file mode 100644 index 0000000..c4fc920 --- /dev/null +++ b/inssqs/model.go @@ -0,0 +1,45 @@ +package inssqs + +import "github.com/aws/aws-sdk-go-v2/service/sqs/types" + +type entry interface { + getId() *string +} + +type SQSMessageEntry struct { + Id *string + MessageBody *string + DelaySeconds int32 + MessageDeduplicationId *string + MessageGroupId *string +} + +func (e SQSMessageEntry) getId() *string { + return e.Id +} + +func (e SQSMessageEntry) toSendMessageBatchRequestEntry() types.SendMessageBatchRequestEntry { + return types.SendMessageBatchRequestEntry{ + Id: e.Id, + MessageBody: e.MessageBody, + DelaySeconds: e.DelaySeconds, + MessageDeduplicationId: e.MessageDeduplicationId, + MessageGroupId: e.MessageGroupId, + } +} + +type SQSDeleteMessageEntry struct { + Id *string + ReceiptHandle *string +} + +func (e SQSDeleteMessageEntry) getId() *string { + return e.Id +} + +func (e SQSDeleteMessageEntry) toDeleteMessageBatchRequestEntry() types.DeleteMessageBatchRequestEntry { + return types.DeleteMessageBatchRequestEntry{ + Id: e.Id, + ReceiptHandle: e.ReceiptHandle, + } +} diff --git a/inssqs/sqs/iface.go b/inssqs/sqs/iface.go new file mode 100644 index 0000000..d9abb81 --- /dev/null +++ b/inssqs/sqs/iface.go @@ -0,0 +1,32 @@ +package sqs + +import ( + "context" + "github.com/aws/aws-sdk-go-v2/service/sqs" +) + +type API interface { + SendMessageBatch(ctx context.Context, params *sqs.SendMessageBatchInput, optFns ...func(*sqs.Options)) (*sqs.SendMessageBatchOutput, error) + GetQueueUrl(ctx context.Context, params *sqs.GetQueueUrlInput, optFns ...func(*sqs.Options)) (*sqs.GetQueueUrlOutput, error) + DeleteMessageBatch(ctx context.Context, params *sqs.DeleteMessageBatchInput, optFns ...func(*sqs.Options)) (*sqs.DeleteMessageBatchOutput, error) +} + +type proxy struct { + client *sqs.Client +} + +func NewSQSProxy(client *sqs.Client) API { + return &proxy{client} +} + +func (p *proxy) SendMessageBatch(ctx context.Context, params *sqs.SendMessageBatchInput, optFns ...func(*sqs.Options)) (*sqs.SendMessageBatchOutput, error) { + return p.client.SendMessageBatch(ctx, params, optFns...) +} + +func (p *proxy) GetQueueUrl(ctx context.Context, params *sqs.GetQueueUrlInput, optFns ...func(*sqs.Options)) (*sqs.GetQueueUrlOutput, error) { + return p.client.GetQueueUrl(ctx, params, optFns...) +} + +func (p *proxy) DeleteMessageBatch(ctx context.Context, params *sqs.DeleteMessageBatchInput, optFns ...func(*sqs.Options)) (*sqs.DeleteMessageBatchOutput, error) { + return p.client.DeleteMessageBatch(ctx, params, optFns...) +} diff --git a/inssqs/sqs/sqs_mock.go b/inssqs/sqs/sqs_mock.go new file mode 100644 index 0000000..6951246 --- /dev/null +++ b/inssqs/sqs/sqs_mock.go @@ -0,0 +1,92 @@ +package sqs + +import ( + context "context" + reflect "reflect" + + sqs "github.com/aws/aws-sdk-go-v2/service/sqs" + gomock "go.uber.org/mock/gomock" +) + +// MockAPI is a mock of API interface. +type MockAPI struct { + ctrl *gomock.Controller + recorder *MockAPIMockRecorder +} + +// MockAPIMockRecorder is the mock recorder for MockAPI. +type MockAPIMockRecorder struct { + mock *MockAPI +} + +// NewMockSQS creates a new mock instance. +func NewMockSQS(ctrl *gomock.Controller) *MockAPI { + mock := &MockAPI{ctrl: ctrl} + mock.recorder = &MockAPIMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockAPI) EXPECT() *MockAPIMockRecorder { + return m.recorder +} + +// GetQueueUrl mocks base method. +func (m *MockAPI) GetQueueUrl(ctx context.Context, params *sqs.GetQueueUrlInput, optFns ...func(*sqs.Options)) (*sqs.GetQueueUrlOutput, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, params} + for _, a := range optFns { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetQueueUrl", varargs...) + ret0, _ := ret[0].(*sqs.GetQueueUrlOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetQueueUrl indicates an expected call of GetQueueUrl. +func (mr *MockAPIMockRecorder) GetQueueUrl(ctx, params any, optFns ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, params}, optFns...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetQueueUrl", reflect.TypeOf((*MockAPI)(nil).GetQueueUrl), varargs...) +} + +// SendMessageBatch mocks base method. +func (m *MockAPI) SendMessageBatch(ctx context.Context, params *sqs.SendMessageBatchInput, optFns ...func(*sqs.Options)) (*sqs.SendMessageBatchOutput, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, params} + for _, a := range optFns { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "SendMessageBatch", varargs...) + ret0, _ := ret[0].(*sqs.SendMessageBatchOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// SendMessageBatch indicates an expected call of SendMessageBatch. +func (mr *MockAPIMockRecorder) SendMessageBatch(ctx, params any, optFns ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, params}, optFns...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessageBatch", reflect.TypeOf((*MockAPI)(nil).SendMessageBatch), varargs...) +} + +// DeleteMessageBatch mocks base method. +func (m *MockAPI) DeleteMessageBatch(ctx context.Context, params *sqs.DeleteMessageBatchInput, optFns ...func(*sqs.Options)) (*sqs.DeleteMessageBatchOutput, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, params} + for _, a := range optFns { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DeleteMessageBatch", varargs...) + ret0, _ := ret[0].(*sqs.DeleteMessageBatchOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteMessageBatch indicates an expected call of DeleteMessageBatch. +func (mr *MockAPIMockRecorder) DeleteMessageBatch(ctx, params any, optFns ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, params}, optFns...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteMessageBatch", reflect.TypeOf((*MockAPI)(nil).DeleteMessageBatch), varargs...) +}