From 4bab8bd25dea4b031d684f328adc3b1721d7d29c Mon Sep 17 00:00:00 2001 From: ygsa Date: Thu, 17 Aug 2023 14:47:17 +0800 Subject: [PATCH 1/7] Switch MongoDB libraries. https://github.com/influxdata/telegraf/pull/9493 (cherry picked from commit a5b149174514a313511e3377763262c3470ecd83) --- go.mod | 6 +- go.sum | 80 +++++++- plugins/inputs/mongodb/mongodb.go | 179 ++++++++---------- plugins/inputs/mongodb/mongodb_data.go | 157 ++++++++------- plugins/inputs/mongodb/mongodb_server.go | 150 +++++++++++---- plugins/inputs/mongodb/mongodb_server_test.go | 8 +- plugins/inputs/mongodb/mongodb_test.go | 55 ++---- 7 files changed, 386 insertions(+), 249 deletions(-) diff --git a/go.mod b/go.mod index 4a2bef6d27329..6ab0e049beec5 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.1 github.com/aws/aws-sdk-go-v2/service/ec2 v1.1.0 github.com/aws/smithy-go v1.0.0 - github.com/beevik/ntp v0.3.0 // indirect + github.com/beevik/ntp v0.3.0 github.com/benbjohnson/clock v1.0.3 github.com/bitly/go-hostpool v0.1.0 // indirect github.com/bmatcuk/doublestar/v3 v3.0.0 @@ -132,13 +132,14 @@ require ( github.com/wvanbergen/kazoo-go v0.0.0-20180202103751-f72d8611297a // indirect github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c github.com/yuin/gopher-lua v0.0.0-20180630135845-46796da1b0b4 // indirect + go.mongodb.org/mongo-driver v1.5.3 go.starlark.net v0.0.0-20210312235212-74c10e2c17dc go.uber.org/multierr v1.6.0 // indirect golang.org/x/net v0.0.0-20201209123823-ac852fbbde11 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa - golang.org/x/text v0.3.4 + golang.org/x/text v0.3.5 golang.zx2c4.com/wireguard/wgctrl v0.0.0-20200205215550-e35592f146e4 google.golang.org/api v0.20.0 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 @@ -147,7 +148,6 @@ require ( gopkg.in/fatih/pool.v2 v2.0.0 // indirect gopkg.in/gorethink/gorethink.v3 v3.0.5 gopkg.in/ldap.v3 v3.1.0 - gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 gopkg.in/olivere/elastic.v5 v5.0.70 gopkg.in/yaml.v2 v2.3.0 gotest.tools v2.2.0+incompatible diff --git a/go.sum b/go.sum index c49003bf243a5..c6d9d60acee44 100644 --- a/go.sum +++ b/go.sum @@ -150,6 +150,7 @@ github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/aws/aws-sdk-go v1.34.34 h1:5dC0ZU0xy25+UavGNEkQ/5MOQwxXDA2YXtjCL1HfYKI= github.com/aws/aws-sdk-go v1.34.34 h1:5dC0ZU0xy25+UavGNEkQ/5MOQwxXDA2YXtjCL1HfYKI= github.com/aws/aws-sdk-go v1.34.34/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= @@ -349,6 +350,42 @@ github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gG github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd h1:hSkbZ9XSyjyBirMeqSqUrK+9HboWrweVlzRNqoBi2d4= +github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= +github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= +github.com/gobuffalo/depgen v0.1.0 h1:31atYa/UW9V5q8vMJ+W6wd64OaaTHUrCUXER358zLM4= +github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= +github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/envy v1.7.0 h1:GlXgaiBkmrYMHco6t4j7SacKO4XUjvh5pwXh0f4uxXU= +github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= +github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/flect v0.1.3 h1:3GQ53z7E3o00C/yy7Ko8VXqQXoJGLkrTQCLTF1EjoXU= +github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= +github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= +github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= +github.com/gobuffalo/genny v0.1.1 h1:iQ0D6SpNXIxu52WESsD+KoQ7af2e3nCfnSBoSF/hKe0= +github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= +github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211 h1:mSVZ4vj4khv+oThUfS+SQU3UuFIZ5Zo6UNcvK8E8Mz8= +github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= +github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= +github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= +github.com/gobuffalo/gogen v0.1.1 h1:dLg+zb+uOyd/mKeQUYIbwbNmfRsr9hd/WtYWepmayhI= +github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= +github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2 h1:8thhT+kUJMTMy3HlX4+y9Da+BNJck+p109tqqKp7WDs= +github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= +github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/mapi v1.0.2 h1:fq9WcL1BYrm36SzK6+aAnZ8hcp+SrmnDyAxhNx8dvJk= +github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packd v0.1.0 h1:4sGKOD8yaYJ+dek1FDkwcxCHA40M4kfKgFHx8N2kwbU= +github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= +github.com/gobuffalo/packr/v2 v2.2.0 h1:Ir9W9XIm9j7bhhkKE9cokvtTl1vBm62A/fene/ZCj6A= +github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= +github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754 h1:tpom+2CJmpzAWj5/VEHync2rJGi+epHNIeRSWjzGA+4= +github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/goburrow/modbus v0.1.0 h1:DejRZY73nEM6+bt5JSP6IsFolJ9dVcqxsYbpLbeW/ro= github.com/goburrow/modbus v0.1.0/go.mod h1:Kx552D5rLIS8E7TyUwQ/UdHEqvX5T8tyiGBTlzMcZBg= github.com/goburrow/serial v0.1.0 h1:v2T1SQa/dlUqQiYIT8+Cu7YolfqAi3K96UmhwYyuSrA= @@ -516,6 +553,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6 h1:UDMh68U github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/apcupsd v0.0.0-20210427145308-694d5caead0e h1:3J1OB4RDKwXs5l8uEV6BP/tucOJOPDQysiT7/9cuXzA= github.com/influxdata/apcupsd v0.0.0-20210427145308-694d5caead0e/go.mod h1:WYK/Z/aXq9cbMFIL5ihcA4sX/r/3/WCas/Qvs/2fXcA= github.com/influxdata/go-syslog/v2 v2.0.1 h1:l44S4l4Q8MhGQcoOxJpbo+QQYxJqp0vdgIVHh4+DO0s= @@ -568,6 +607,8 @@ github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5 h1:PJr+ZMXIecYc github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/kardianos/service v1.0.0 h1:HgQS3mFfOlyntWX8Oke98JcJLqt1DBcHR4kxShpYef0= github.com/kardianos/service v1.0.0/go.mod h1:8CzDhVuCuugtsHyZoTvsOBuvonN/UDBvl0kH+BUxvbo= +github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= +github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= @@ -577,9 +618,11 @@ github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHz github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.0 h1:wJbzvpYMVGG9iTI9VxpnNZfd4DzMPoCWze3GgSqz8yg= github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= @@ -607,6 +650,10 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2 h1:JgVTCPf0uBVcUSWpyXmGpgOc62nK5HWUBKAGc3Qqa5k= +github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= +github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI= +github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE= @@ -651,6 +698,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/multiplay/go-ts3 v1.0.0 h1:loxtEFqvYtpoGh1jOqEt6aDzctYuQsi3vb3dMpvWiWw= @@ -714,6 +763,8 @@ github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0Mw github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pavius/impi v0.0.0-20180302134524-c1cbdcb8df2b h1:yS0+/i6mwRZCdssUd+MkFJkCn/Evh1PlUKCYe3aCtQw= github.com/pavius/impi v0.0.0-20180302134524-c1cbdcb8df2b/go.mod h1:x/hU0bfdWIhuOT1SKwiJg++yvkk6EuOtJk8WtDZqgr8= +github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ= @@ -769,6 +820,8 @@ github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzG github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f h1:UFr9zpz4xgTnIE5yIMtWAMngCdZ9p/+q6lTbgelo80M= @@ -801,6 +854,8 @@ github.com/signalfx/gomemcache v0.0.0-20180823214636-4f7ef64c72a9/go.mod h1:Ytb8 github.com/signalfx/sapm-proto v0.4.0 h1:5lQX++6FeIjUZEIcnSgBqhOpmSjMkRBW3y/4ZiKMo5E= github.com/signalfx/sapm-proto v0.4.0/go.mod h1:x3gtwJ1GRejtkghB4nYpwixh2zqJrLbPU959ZNhM0Fk= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -814,6 +869,8 @@ github.com/smartystreets/goconvey v1.6.4-0.20190306220146-200a235640ff/go.mod h1 github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -871,15 +928,25 @@ github.com/wvanbergen/kafka v0.0.0-20171203153745-e2edea948ddf h1:TOV5PC6fIWwFOF github.com/wvanbergen/kafka v0.0.0-20171203153745-e2edea948ddf/go.mod h1:nxx7XRXbR9ykhnC8lXqQyJS0rfvJGxKyKw/sT1YOttg= github.com/wvanbergen/kazoo-go v0.0.0-20180202103751-f72d8611297a h1:ILoU84rj4AQ3q6cjQvtb9jBjx4xzR/Riq/zYhmDQiOk= github.com/wvanbergen/kazoo-go v0.0.0-20180202103751-f72d8611297a/go.mod h1:vQQATAGxVK20DC1rRubTJbZDDhhpA4QfU02pMdPxGO4= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.0.2 h1:akYIkZ28e6A96dkWNJQu3nmCzH3YfwMPQExUYDaRv7w= +github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= +github.com/xdg-go/stringprep v1.0.2 h1:6iq84/ryjjeRmMJwxutI51F2GIPlP5BfTvXHeYjyhBc= +github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1 h1:ruQGxdhGHe7FWOJPT0mKs5+pD2Xs1Bm/kdGlHO04FmM= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/gopher-lua v0.0.0-20180630135845-46796da1b0b4 h1:f6CCNiTjQZ0uWK4jPwhwYB8QIGGfn0ssD9kVzRUUUpk= github.com/yuin/gopher-lua v0.0.0-20180630135845-46796da1b0b4/go.mod h1:aEV29XrmTYFr3CiRxZeGHpkvbwq+prZduBqMaascyCU= +go.mongodb.org/mongo-driver v1.5.3 h1:wWbFB6zaGHpzguF3f7tW94sVE8sFl3lHx8OZx/4OuFI= +go.mongodb.org/mongo-driver v1.5.3/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -896,6 +963,7 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -905,6 +973,7 @@ golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -997,6 +1066,7 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= @@ -1012,11 +1082,14 @@ golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1060,8 +1133,9 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1077,9 +1151,13 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= diff --git a/plugins/inputs/mongodb/mongodb.go b/plugins/inputs/mongodb/mongodb.go index 654f5855c0a58..0366636200064 100644 --- a/plugins/inputs/mongodb/mongodb.go +++ b/plugins/inputs/mongodb/mongodb.go @@ -1,10 +1,10 @@ package mongodb import ( + "context" "crypto/tls" "crypto/x509" "fmt" - "net" "net/url" "strings" "sync" @@ -13,20 +13,24 @@ import ( "github.com/influxdata/telegraf" tlsint "github.com/influxdata/telegraf/plugins/common/tls" "github.com/influxdata/telegraf/plugins/inputs" - "gopkg.in/mgo.v2" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "go.mongodb.org/mongo-driver/mongo/readpref" ) type MongoDB struct { Servers []string Ssl Ssl - mongos map[string]*Server GatherClusterStatus bool GatherPerdbStats bool GatherColStats bool + GatherTopStat bool ColStatsDbs []string tlsint.ClientConfig - Log telegraf.Logger + Log telegraf.Logger `toml:"-"` + + clients []*Server } type Ssl struct { @@ -53,6 +57,10 @@ var sampleConfig = ` ## When true, collect per collection stats # gather_col_stats = false + ## When true, collect usage statistics for each collection + ## (insert, update, queries, remove, getmore, commands etc...). + # gather_top_stat = false + ## List of db where collections stats are collected ## If empty, all db are concerned # col_stats_dbs = ["local"] @@ -73,126 +81,107 @@ func (*MongoDB) Description() string { return "Read metrics from one or many MongoDB servers" } -var localhost = &url.URL{Host: "mongodb://127.0.0.1:27017"} +func (m *MongoDB) Init() error { + var tlsConfig *tls.Config + if m.Ssl.Enabled { + // Deprecated TLS config + tlsConfig = &tls.Config{ + InsecureSkipVerify: m.ClientConfig.InsecureSkipVerify, + } + if len(m.Ssl.CaCerts) == 0 { + return fmt.Errorf("you must explicitly set insecure_skip_verify to skip cerificate validation") + } + + roots := x509.NewCertPool() + for _, caCert := range m.Ssl.CaCerts { + if ok := roots.AppendCertsFromPEM([]byte(caCert)); !ok { + return fmt.Errorf("failed to parse root certificate") + } + } + tlsConfig.RootCAs = roots + } else { + var err error + tlsConfig, err = m.ClientConfig.TLSConfig() + if err != nil { + return err + } + } -// Reads stats from all configured servers accumulates stats. -// Returns one of the errors encountered while gather stats (if any). -func (m *MongoDB) Gather(acc telegraf.Accumulator) error { if len(m.Servers) == 0 { - m.gatherServer(m.getMongoServer(localhost), acc) - return nil + m.Servers = []string{"mongodb://127.0.0.1:27017"} } - var wg sync.WaitGroup - for i, serv := range m.Servers { - if !strings.HasPrefix(serv, "mongodb://") { + for _, connURL := range m.Servers { + if !strings.HasPrefix(connURL, "mongodb://") && !strings.HasPrefix(connURL, "mongodb+srv://") { // Preserve backwards compatibility for hostnames without a // scheme, broken in go 1.8. Remove in Telegraf 2.0 - serv = "mongodb://" + serv - m.Log.Warnf("Using %q as connection URL; please update your configuration to use an URL", serv) - m.Servers[i] = serv + connURL = "mongodb://" + connURL + m.Log.Warnf("Using %q as connection URL; please update your configuration to use an URL", connURL) } - u, err := url.Parse(serv) + u, err := url.Parse(connURL) if err != nil { - m.Log.Errorf("Unable to parse address %q: %s", serv, err.Error()) - continue + return fmt.Errorf("unable to parse connection URL: %q", err) } - if u.Host == "" { - m.Log.Errorf("Unable to parse address %q", serv) - continue - } - - wg.Add(1) - go func(srv *Server) { - defer wg.Done() - err := m.gatherServer(srv, acc) - if err != nil { - m.Log.Errorf("Error in plugin: %v", err) - } - }(m.getMongoServer(u)) - } - wg.Wait() - return nil -} + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() //nolint:revive -func (m *MongoDB) getMongoServer(url *url.URL) *Server { - if _, ok := m.mongos[url.Host]; !ok { - m.mongos[url.Host] = &Server{ - Log: m.Log, - URL: url, + opts := options.Client().ApplyURI(connURL) + if tlsConfig != nil { + opts.TLSConfig = tlsConfig } - } - return m.mongos[url.Host] -} - -func (m *MongoDB) gatherServer(server *Server, acc telegraf.Accumulator) error { - if server.Session == nil { - var dialAddrs []string - if server.URL.User != nil { - dialAddrs = []string{server.URL.String()} - } else { - dialAddrs = []string{server.URL.Host} + if opts.ReadPreference == nil { + opts.ReadPreference = readpref.Nearest() } - dialInfo, err := mgo.ParseURL(dialAddrs[0]) + + client, err := mongo.Connect(ctx, opts) if err != nil { - return fmt.Errorf("unable to parse URL %q: %s", dialAddrs[0], err.Error()) - } - dialInfo.Direct = true - dialInfo.Timeout = 5 * time.Second - - var tlsConfig *tls.Config - - if m.Ssl.Enabled { - // Deprecated TLS config - tlsConfig = &tls.Config{} - if len(m.Ssl.CaCerts) > 0 { - roots := x509.NewCertPool() - for _, caCert := range m.Ssl.CaCerts { - ok := roots.AppendCertsFromPEM([]byte(caCert)) - if !ok { - return fmt.Errorf("failed to parse root certificate") - } - } - tlsConfig.RootCAs = roots - } else { - tlsConfig.InsecureSkipVerify = true - } - } else { - tlsConfig, err = m.ClientConfig.TLSConfig() - if err != nil { - return err - } + return fmt.Errorf("unable to connect to MongoDB: %q", err) } - // If configured to use TLS, add a dial function - if tlsConfig != nil { - dialInfo.DialServer = func(addr *mgo.ServerAddr) (net.Conn, error) { - conn, err := tls.Dial("tcp", addr.String(), tlsConfig) - if err != nil { - fmt.Printf("error in Dial, %s\n", err.Error()) - } - return conn, err - } + err = client.Ping(ctx, opts.ReadPreference) + if err != nil { + return fmt.Errorf("unable to connect to MongoDB: %s", err) } - sess, err := mgo.DialWithInfo(dialInfo) - if err != nil { - return fmt.Errorf("unable to connect to MongoDB(%s): %s", strings.Join(dialInfo.Addrs, ","), err.Error()) + server := &Server{ + client: client, + hostname: u.Host, + Log: m.Log, } - server.Session = sess + m.clients = append(m.clients, server) } - return server.gatherData(acc, m.GatherClusterStatus, m.GatherPerdbStats, m.GatherColStats, m.ColStatsDbs) + + return nil +} + +// Reads stats from all configured servers accumulates stats. +// Returns one of the errors encountered while gather stats (if any). +func (m *MongoDB) Gather(acc telegraf.Accumulator) error { + var wg sync.WaitGroup + for _, client := range m.clients { + wg.Add(1) + go func(srv *Server) { + defer wg.Done() + err := srv.gatherData(acc, m.GatherClusterStatus, m.GatherPerdbStats, m.GatherColStats, m.GatherTopStat, m.ColStatsDbs) + if err != nil { + m.Log.Errorf("failed to gather data: %q", err) + } + }(client) + } + + wg.Wait() + return nil } func init() { inputs.Add("mongodb", func() telegraf.Input { return &MongoDB{ - mongos: make(map[string]*Server), GatherClusterStatus: true, GatherPerdbStats: false, GatherColStats: false, + GatherTopStat: false, ColStatsDbs: []string{"local"}, } }) diff --git a/plugins/inputs/mongodb/mongodb_data.go b/plugins/inputs/mongodb/mongodb_data.go index 685ee85ee8aaf..e26c0e45231eb 100644 --- a/plugins/inputs/mongodb/mongodb_data.go +++ b/plugins/inputs/mongodb/mongodb_data.go @@ -12,11 +12,10 @@ type MongodbData struct { StatLine *StatLine Fields map[string]interface{} Tags map[string]string - MetaTags map[string]string - MetaFields map[string]interface{} DbData []DbData ColData []ColData ShardHostData []DbData + TopStatsData []DbData } type DbData struct { @@ -30,27 +29,16 @@ type ColData struct { Fields map[string]interface{} } -func copyTagsMap(globalTags map[string]string) map[string]string { - tags := make(map[string]string) - for k, v := range globalTags { - tags[k] = v - } - - return tags -} - func NewMongodbData(statLine *StatLine, tags map[string]string) *MongodbData { return &MongodbData{ StatLine: statLine, Tags: tags, Fields: make(map[string]interface{}), - MetaTags: copyTagsMap(tags), - MetaFields: make(map[string]interface{}), DbData: []DbData{}, } } -var DefaultStats = map[string]string{ +var defaultStats = map[string]string{ "uptime_ns": "UptimeNanos", "inserts": "InsertCnt", "inserts_per_sec": "Insert", @@ -107,7 +95,7 @@ var DefaultStats = map[string]string{ "total_docs_scanned": "TotalObjectsScanned", } -var DefaultAssertsStats = map[string]string{ +var defaultAssertsStats = map[string]string{ "assert_regular": "Regular", "assert_warning": "Warning", "assert_msg": "Msg", @@ -115,7 +103,7 @@ var DefaultAssertsStats = map[string]string{ "assert_rollovers": "Rollovers", } -var DefaultCommandsStats = map[string]string{ +var defaultCommandsStats = map[string]string{ "aggregate_command_total": "AggregateCommandTotal", "aggregate_command_failed": "AggregateCommandFailed", "count_command_total": "CountCommandTotal", @@ -136,7 +124,7 @@ var DefaultCommandsStats = map[string]string{ "update_command_failed": "UpdateCommandFailed", } -var DefaultLatencyStats = map[string]string{ +var defaultLatencyStats = map[string]string{ "latency_writes_count": "WriteOpsCnt", "latency_writes": "WriteLatency", "latency_reads_count": "ReadOpsCnt", @@ -145,7 +133,7 @@ var DefaultLatencyStats = map[string]string{ "latency_commands": "CommandLatency", } -var DefaultReplStats = map[string]string{ +var defaultReplStats = map[string]string{ "repl_inserts": "InsertRCnt", "repl_inserts_per_sec": "InsertR", "repl_queries": "QueryRCnt", @@ -177,37 +165,37 @@ var DefaultReplStats = map[string]string{ "repl_executor_unsignaled_events": "ReplExecutorUnsignaledEvents", } -var DefaultClusterStats = map[string]string{ +var defaultClusterStats = map[string]string{ "jumbo_chunks": "JumboChunksCount", } -var DefaultShardStats = map[string]string{ +var defaultShardStats = map[string]string{ "total_in_use": "TotalInUse", "total_available": "TotalAvailable", "total_created": "TotalCreated", "total_refreshing": "TotalRefreshing", } -var ShardHostStats = map[string]string{ +var shardHostStats = map[string]string{ "in_use": "InUse", "available": "Available", "created": "Created", "refreshing": "Refreshing", } -var MmapStats = map[string]string{ +var mmapStats = map[string]string{ "mapped_megabytes": "Mapped", "non-mapped_megabytes": "NonMapped", "page_faults": "FaultsCnt", "page_faults_per_sec": "Faults", } -var WiredTigerStats = map[string]string{ +var wiredTigerStats = map[string]string{ "percent_cache_dirty": "CacheDirtyPercent", "percent_cache_used": "CacheUsedPercent", } -var WiredTigerExtStats = map[string]string{ +var wiredTigerExtStats = map[string]string{ "wtcache_tracked_dirty_bytes": "TrackedDirtyBytes", "wtcache_current_bytes": "CurrentCachedBytes", "wtcache_max_bytes_configured": "MaxBytesConfigured", @@ -228,7 +216,7 @@ var WiredTigerExtStats = map[string]string{ "wtcache_unmodified_pages_evicted": "UnmodifiedPagesEvicted", } -var DefaultTCMallocStats = map[string]string{ +var defaultTCMallocStats = map[string]string{ "tcmalloc_current_allocated_bytes": "TCMallocCurrentAllocatedBytes", "tcmalloc_heap_size": "TCMallocHeapSize", "tcmalloc_central_cache_free_bytes": "TCMallocCentralCacheFreeBytes", @@ -250,13 +238,13 @@ var DefaultTCMallocStats = map[string]string{ "tcmalloc_pageheap_total_reserve_bytes": "TCMallocPageheapTotalReserveBytes", } -var DefaultStorageStats = map[string]string{ +var defaultStorageStats = map[string]string{ "storage_freelist_search_bucket_exhausted": "StorageFreelistSearchBucketExhausted", "storage_freelist_search_requests": "StorageFreelistSearchRequests", "storage_freelist_search_scanned": "StorageFreelistSearchScanned", } -var DbDataStats = map[string]string{ +var dbDataStats = map[string]string{ "collections": "Collections", "objects": "Objects", "avg_obj_size": "AvgObjSize", @@ -268,7 +256,7 @@ var DbDataStats = map[string]string{ "ok": "Ok", } -var ColDataStats = map[string]string{ +var colDataStats = map[string]string{ "count": "Count", "size": "Size", "avg_obj_size": "AvgObjSize", @@ -277,6 +265,27 @@ var ColDataStats = map[string]string{ "ok": "Ok", } +var topDataStats = map[string]string{ + "total_time": "TotalTime", + "total_count": "TotalCount", + "read_lock_time": "ReadLockTime", + "read_lock_count": "ReadLockCount", + "write_lock_time": "WriteLockTime", + "write_lock_count": "WriteLockCount", + "queries_time": "QueriesTime", + "queries_count": "QueriesCount", + "get_more_time": "GetMoreTime", + "get_more_count": "GetMoreCount", + "insert_time": "InsertTime", + "insert_count": "InsertCount", + "update_time": "UpdateTime", + "update_count": "UpdateCount", + "remove_time": "RemoveTime", + "remove_count": "RemoveCount", + "commands_time": "CommandsTime", + "commands_count": "CommandsCount", +} + func (d *MongodbData) AddDbStats() { for _, dbstat := range d.StatLine.DbStatsLines { dbStatLine := reflect.ValueOf(&dbstat).Elem() @@ -285,7 +294,7 @@ func (d *MongodbData) AddDbStats() { Fields: make(map[string]interface{}), } newDbData.Fields["type"] = "db_stat" - for key, value := range DbDataStats { + for key, value := range dbDataStats { val := dbStatLine.FieldByName(value).Interface() newDbData.Fields[key] = val } @@ -302,7 +311,7 @@ func (d *MongodbData) AddColStats() { Fields: make(map[string]interface{}), } newColData.Fields["type"] = "col_stat" - for key, value := range ColDataStats { + for key, value := range colDataStats { val := colStatLine.FieldByName(value).Interface() newColData.Fields[key] = val } @@ -318,7 +327,7 @@ func (d *MongodbData) AddShardHostStats() { Fields: make(map[string]interface{}), } newDbData.Fields["type"] = "shard_host_stat" - for k, v := range ShardHostStats { + for k, v := range shardHostStats { val := hostStatLine.FieldByName(v).Interface() newDbData.Fields[k] = val } @@ -326,55 +335,63 @@ func (d *MongodbData) AddShardHostStats() { } } -func (d *MongodbData) AddMetaInfo() { - if d.StatLine.ReplSetName != "" { - d.MetaTags["rs_name"] = d.StatLine.ReplSetName - } - - if d.StatLine.Version != "" { - d.MetaTags["version"] = d.StatLine.Version - } - - if d.StatLine.StorageEngine != "" { - d.MetaTags["storage_engine"] = d.StatLine.StorageEngine +func (d *MongodbData) AddTopStats() { + for _, topStat := range d.StatLine.TopStatLines { + topStatLine := reflect.ValueOf(&topStat).Elem() + newTopStatData := &DbData{ + Name: topStat.CollectionName, + Fields: make(map[string]interface{}), + } + newTopStatData.Fields["type"] = "top_stat" + for key, value := range topDataStats { + val := topStatLine.FieldByName(value).Interface() + newTopStatData.Fields[key] = val + } + d.TopStatsData = append(d.TopStatsData, *newTopStatData) } - - d.addMeta("meta_info", 1) } func (d *MongodbData) AddDefaultStats() { statLine := reflect.ValueOf(d.StatLine).Elem() - d.addStat(statLine, DefaultStats) + d.addStat(statLine, defaultStats) if d.StatLine.NodeType != "" { - d.addStat(statLine, DefaultReplStats) - d.add("state_int", d.StatLine.NodeTypeInt) + d.addStat(statLine, defaultReplStats) + d.Tags["node_type"] = d.StatLine.NodeType } if d.StatLine.ReadLatency > 0 { - d.addStat(statLine, DefaultLatencyStats) + d.addStat(statLine, defaultLatencyStats) + } + + if d.StatLine.ReplSetName != "" { + d.Tags["rs_name"] = d.StatLine.ReplSetName } if d.StatLine.OplogStats != nil { d.add("repl_oplog_window_sec", d.StatLine.OplogStats.TimeDiff) } - d.addStat(statLine, DefaultAssertsStats) - d.addStat(statLine, DefaultClusterStats) - d.addStat(statLine, DefaultCommandsStats) - d.addStat(statLine, DefaultShardStats) - d.addStat(statLine, DefaultStorageStats) - d.addStat(statLine, DefaultTCMallocStats) + if d.StatLine.Version != "" { + d.add("version", d.StatLine.Version) + } + + d.addStat(statLine, defaultAssertsStats) + d.addStat(statLine, defaultClusterStats) + d.addStat(statLine, defaultCommandsStats) + d.addStat(statLine, defaultShardStats) + d.addStat(statLine, defaultStorageStats) + d.addStat(statLine, defaultTCMallocStats) if d.StatLine.StorageEngine == "mmapv1" || d.StatLine.StorageEngine == "rocksdb" { - d.addStat(statLine, MmapStats) + d.addStat(statLine, mmapStats) } else if d.StatLine.StorageEngine == "wiredTiger" { - for key, value := range WiredTigerStats { + for key, value := range wiredTigerStats { val := statLine.FieldByName(value).Interface() percentVal := fmt.Sprintf("%.1f", val.(float64)*100) floatVal, _ := strconv.ParseFloat(percentVal, 64) d.add(key, floatVal) } - d.addStat(statLine, WiredTigerExtStats) + d.addStat(statLine, wiredTigerExtStats) d.add("page_faults", d.StatLine.FaultsCnt) } } @@ -390,19 +407,7 @@ func (d *MongodbData) add(key string, val interface{}) { d.Fields[key] = val } -func (d *MongodbData) addMeta(key string, val interface{}) { - d.MetaFields[key] = val -} - func (d *MongodbData) flush(acc telegraf.Accumulator) { - acc.AddFields( - "mongodb_meta", - d.MetaFields, - d.MetaTags, - d.StatLine.Time, - ) - d.MetaFields = make(map[string]interface{}) - acc.AddFields( "mongodb", d.Fields, @@ -411,8 +416,6 @@ func (d *MongodbData) flush(acc telegraf.Accumulator) { ) d.Fields = make(map[string]interface{}) - - for _, db := range d.DbData { d.Tags["db_name"] = db.Name acc.AddFields( @@ -435,7 +438,7 @@ func (d *MongodbData) flush(acc telegraf.Accumulator) { col.Fields = make(map[string]interface{}) } for _, host := range d.ShardHostData { - d.Tags["server"] = host.Name + d.Tags["hostname"] = host.Name acc.AddFields( "mongodb_shard_stats", host.Fields, @@ -444,4 +447,14 @@ func (d *MongodbData) flush(acc telegraf.Accumulator) { ) host.Fields = make(map[string]interface{}) } + for _, col := range d.TopStatsData { + d.Tags["collection"] = col.Name + acc.AddFields( + "mongodb_top_stats", + col.Fields, + d.Tags, + d.StatLine.Time, + ) + col.Fields = make(map[string]interface{}) + } } diff --git a/plugins/inputs/mongodb/mongodb_server.go b/plugins/inputs/mongodb/mongodb_server.go index 9c528cc28a990..9232e20f21ec6 100644 --- a/plugins/inputs/mongodb/mongodb_server.go +++ b/plugins/inputs/mongodb/mongodb_server.go @@ -1,19 +1,23 @@ package mongodb import ( + "context" "fmt" - "net/url" + "go.mongodb.org/mongo-driver/bson/primitive" + "strconv" "strings" "time" "github.com/influxdata/telegraf" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "go.mongodb.org/mongo-driver/x/bsonx" ) type Server struct { - URL *url.URL - Session *mgo.Session + client *mongo.Client + hostname string lastResult *MongoStatus Log telegraf.Logger @@ -21,12 +25,12 @@ type Server struct { func (s *Server) getDefaultTags() map[string]string { tags := make(map[string]string) - tags["server"] = s.URL.Host + tags["hostname"] = s.hostname return tags } type oplogEntry struct { - Timestamp bson.MongoTimestamp `bson:"ts"` + Timestamp primitive.Timestamp `bson:"ts"` } func IsAuthorization(err error) bool { @@ -41,15 +45,23 @@ func (s *Server) authLog(err error) { } } +func (s *Server) runCommand(database string, cmd interface{}, result interface{}) error { + r := s.client.Database(database).RunCommand(context.Background(), cmd) + if r.Err() != nil { + return r.Err() + } + return r.Decode(result) +} + func (s *Server) gatherServerStatus() (*ServerStatus, error) { serverStatus := &ServerStatus{} - err := s.Session.DB("admin").Run(bson.D{ + err := s.runCommand("admin", bson.D{ { - Name: "serverStatus", + Key: "serverStatus", Value: 1, }, { - Name: "recordStats", + Key: "recordStats", Value: 0, }, }, serverStatus) @@ -61,9 +73,9 @@ func (s *Server) gatherServerStatus() (*ServerStatus, error) { func (s *Server) gatherReplSetStatus() (*ReplSetStatus, error) { replSetStatus := &ReplSetStatus{} - err := s.Session.DB("admin").Run(bson.D{ + err := s.runCommand("admin", bson.D{ { - Name: "replSetGetStatus", + Key: "replSetGetStatus", Value: 1, }, }, replSetStatus) @@ -73,22 +85,72 @@ func (s *Server) gatherReplSetStatus() (*ReplSetStatus, error) { return replSetStatus, nil } +func (s *Server) gatherTopStatData() (*TopStats, error) { + dest := &bsonx.Doc{} + err := s.runCommand("admin", bson.D{ + { + Key: "top", + Value: 1, + }, + }, dest) + if err != nil { + return nil, err + } + + // From: https://github.com/mongodb/mongo-tools/blob/master/mongotop/mongotop.go#L49-L70 + // Remove 'note' field that prevents easy decoding, then round-trip + // again to simplify unpacking into the nested data structure + totals, err := dest.LookupErr("totals") + if err != nil { + return nil, err + } + recoded, err := totals.Document().Delete("note").MarshalBSON() + if err != nil { + return nil, err + } + topInfo := make(map[string]TopStatCollection) + if err := bson.Unmarshal(recoded, &topInfo); err != nil { + return nil, err + } + + return &TopStats{Totals: topInfo}, nil +} + func (s *Server) gatherClusterStatus() (*ClusterStatus, error) { - chunkCount, err := s.Session.DB("config").C("chunks").Find(bson.M{"jumbo": true}).Count() + chunkCount, err := s.client.Database("config").Collection("chunks").CountDocuments(context.Background(), bson.M{"jumbo": true}) if err != nil { return nil, err } return &ClusterStatus{ - JumboChunksCount: int64(chunkCount), + JumboChunksCount: chunkCount, }, nil } -func (s *Server) gatherShardConnPoolStats() (*ShardStats, error) { +func poolStatsCommand(version string) (string, error) { + majorPart := string(version[0]) + major, err := strconv.ParseInt(majorPart, 10, 64) + if err != nil { + return "", err + } + + if major >= 5 { + return "connPoolStats", nil + } + return "shardConnPoolStats", nil +} + +func (s *Server) gatherShardConnPoolStats(version string) (*ShardStats, error) { shardStats := &ShardStats{} - err := s.Session.DB("admin").Run(bson.D{ + + command, err := poolStatsCommand(version) + if err != nil { + return nil, err + } + + err = s.runCommand("admin", bson.D{ { - Name: "shardConnPoolStats", + Key: command, Value: 1, }, }, &shardStats) @@ -100,9 +162,9 @@ func (s *Server) gatherShardConnPoolStats() (*ShardStats, error) { func (s *Server) gatherDBStats(name string) (*Db, error) { stats := &DbStatsData{} - err := s.Session.DB(name).Run(bson.D{ + err := s.runCommand(name, bson.D{ { - Name: "dbStats", + Key: "dbStats", Value: 1, }, }, stats) @@ -120,19 +182,25 @@ func (s *Server) getOplogReplLag(collection string) (*OplogStats, error) { query := bson.M{"ts": bson.M{"$exists": true}} var first oplogEntry - err := s.Session.DB("local").C(collection).Find(query).Sort("$natural").Limit(1).One(&first) - if err != nil { + firstResult := s.client.Database("local").Collection(collection).FindOne(context.Background(), query, options.FindOne().SetSort(bson.M{"$natural": 1})) + if firstResult.Err() != nil { + return nil, firstResult.Err() + } + if err := firstResult.Decode(&first); err != nil { return nil, err } var last oplogEntry - err = s.Session.DB("local").C(collection).Find(query).Sort("-$natural").Limit(1).One(&last) - if err != nil { + lastResult := s.client.Database("local").Collection(collection).FindOne(context.Background(), query, options.FindOne().SetSort(bson.M{"$natural": -1})) + if lastResult.Err() != nil { + return nil, lastResult.Err() + } + if err := lastResult.Decode(&last); err != nil { return nil, err } - firstTime := time.Unix(int64(first.Timestamp>>32), 0) - lastTime := time.Unix(int64(last.Timestamp>>32), 0) + firstTime := time.Unix(int64(first.Timestamp.T), 0) + lastTime := time.Unix(int64(last.Timestamp.T), 0) stats := &OplogStats{ TimeDiff: int64(lastTime.Sub(firstTime).Seconds()), } @@ -154,7 +222,7 @@ func (s *Server) gatherOplogStats() (*OplogStats, error) { } func (s *Server) gatherCollectionStats(colStatsDbs []string) (*ColStats, error) { - names, err := s.Session.DatabaseNames() + names, err := s.client.ListDatabaseNames(context.Background(), bson.D{}) if err != nil { return nil, err } @@ -163,16 +231,16 @@ func (s *Server) gatherCollectionStats(colStatsDbs []string) (*ColStats, error) for _, dbName := range names { if stringInSlice(dbName, colStatsDbs) || len(colStatsDbs) == 0 { var colls []string - colls, err = s.Session.DB(dbName).CollectionNames() + colls, err = s.client.Database(dbName).ListCollectionNames(context.Background(), bson.D{}) if err != nil { s.Log.Errorf("Error getting collection names: %s", err.Error()) continue } for _, colName := range colls { colStatLine := &ColStatsData{} - err = s.Session.DB(dbName).Run(bson.D{ + err = s.runCommand(dbName, bson.D{ { - Name: "collStats", + Key: "collStats", Value: colName, }, }, colStatLine) @@ -192,10 +260,7 @@ func (s *Server) gatherCollectionStats(colStatsDbs []string) (*ColStats, error) return results, nil } -func (s *Server) gatherData(acc telegraf.Accumulator, gatherClusterStatus bool, gatherDbStats bool, gatherColStats bool, colStatsDbs []string) error { - s.Session.SetMode(mgo.Eventual, true) - s.Session.SetSocketTimeout(0) - +func (s *Server) gatherData(acc telegraf.Accumulator, gatherClusterStatus bool, gatherDbStats bool, gatherColStats bool, gatherTopStat bool, colStatsDbs []string) error { serverStatus, err := s.gatherServerStatus() if err != nil { return err @@ -227,7 +292,7 @@ func (s *Server) gatherData(acc telegraf.Accumulator, gatherClusterStatus bool, clusterStatus = status } - shardStats, err := s.gatherShardConnPoolStats() + shardStats, err := s.gatherShardConnPoolStats(serverStatus.Version) if err != nil { s.authLog(fmt.Errorf("unable to gather shard connection pool stats: %s", err.Error())) } @@ -243,7 +308,7 @@ func (s *Server) gatherData(acc telegraf.Accumulator, gatherClusterStatus bool, dbStats := &DbStats{} if gatherDbStats { - names, err := s.Session.DatabaseNames() + names, err := s.client.ListDatabaseNames(context.Background(), bson.D{}) if err != nil { return err } @@ -257,6 +322,16 @@ func (s *Server) gatherData(acc telegraf.Accumulator, gatherClusterStatus bool, } } + topStatData := &TopStats{} + if gatherTopStat { + topStats, err := s.gatherTopStatData() + if err != nil { + s.Log.Debugf("Unable to gather top stat data: %s", err.Error()) + return err + } + topStatData = topStats + } + result := &MongoStatus{ ServerStatus: serverStatus, ReplSetStatus: replSetStatus, @@ -265,6 +340,7 @@ func (s *Server) gatherData(acc telegraf.Accumulator, gatherClusterStatus bool, ColStats: collectionStats, ShardStats: shardStats, OplogStats: oplogStats, + TopStats: topStatData, } result.SampleTime = time.Now() @@ -275,14 +351,14 @@ func (s *Server) gatherData(acc telegraf.Accumulator, gatherClusterStatus bool, durationInSeconds = 1 } data := NewMongodbData( - NewStatLine(*s.lastResult, *result, s.URL.Host, true, durationInSeconds), + NewStatLine(*s.lastResult, *result, s.hostname, true, durationInSeconds), s.getDefaultTags(), ) data.AddDefaultStats() - data.AddMetaInfo() data.AddDbStats() data.AddColStats() data.AddShardHostStats() + data.AddTopStats() data.flush(acc) } diff --git a/plugins/inputs/mongodb/mongodb_server_test.go b/plugins/inputs/mongodb/mongodb_server_test.go index b8461ab5b91ca..2cf58689a6eab 100644 --- a/plugins/inputs/mongodb/mongodb_server_test.go +++ b/plugins/inputs/mongodb/mongodb_server_test.go @@ -15,7 +15,7 @@ func TestGetDefaultTags(t *testing.T) { in string out string }{ - {"server", server.Url.Host}, + {"hostname", server.hostname}, } defaultTags := server.getDefaultTags() for _, tt := range tagTests { @@ -28,14 +28,14 @@ func TestGetDefaultTags(t *testing.T) { func TestAddDefaultStats(t *testing.T) { var acc testutil.Accumulator - err := server.gatherData(&acc, false) + err := server.gatherData(&acc, false, true, true, true, []string{"local"}) require.NoError(t, err) // need to call this twice so it can perform the diff - err = server.gatherData(&acc, false) + err = server.gatherData(&acc, false, true, true, true, []string{"local"}) require.NoError(t, err) - for key := range DefaultStats { + for key := range defaultStats { assert.True(t, acc.HasInt64Field("mongodb", key)) } } diff --git a/plugins/inputs/mongodb/mongodb_test.go b/plugins/inputs/mongodb/mongodb_test.go index cd3b741e250d8..9484118dd19ab 100644 --- a/plugins/inputs/mongodb/mongodb_test.go +++ b/plugins/inputs/mongodb/mongodb_test.go @@ -3,60 +3,41 @@ package mongodb import ( + "context" "log" "math/rand" - "net/url" "os" "testing" "time" - "gopkg.in/mgo.v2" + "github.com/influxdata/telegraf/testutil" ) -var connect_url string var server *Server -func init() { - connect_url = os.Getenv("MONGODB_URL") - if connect_url == "" { - connect_url = "127.0.0.1:27017" - server = &Server{URL: &url.URL{Host: connect_url}} - } else { - full_url, err := url.Parse(connect_url) - if err != nil { - log.Fatalf("Unable to parse URL (%s), %s\n", full_url, err.Error()) - } - server = &Server{URL: full_url} +func testSetup(_ *testing.M) { + connectionString := os.Getenv("MONGODB_URL") + if connectionString == "" { + connectionString = "mongodb://127.0.0.1:27017" } -} -func testSetup(m *testing.M) { - var err error - var dialAddrs []string - if server.URL.User != nil { - dialAddrs = []string{server.URL.String()} - } else { - dialAddrs = []string{server.URL.Host} - } - dialInfo, err := mgo.ParseURL(dialAddrs[0]) - if err != nil { - log.Fatalf("Unable to parse URL (%s), %s\n", dialAddrs[0], err.Error()) - } - dialInfo.Direct = true - dialInfo.Timeout = 5 * time.Second - sess, err := mgo.DialWithInfo(dialInfo) - if err != nil { - log.Fatalf("Unable to connect to MongoDB, %s\n", err.Error()) + m := &MongoDB{ + Log: testutil.Logger{}, + Servers: []string{connectionString}, } - server.Session = sess - server.Session, _ = mgo.Dial(server.URL.Host) + err := m.Init() if err != nil { - log.Fatalln(err.Error()) + log.Fatalf("Failed to connect to MongoDB: %v", err) } + + server = m.clients[0] } -func testTeardown(m *testing.M) { - server.Session.Close() +func testTeardown(_ *testing.M) { + err := server.client.Disconnect(context.Background()) + if err != nil { + log.Fatalf("failed to disconnect: %v", err) + } } func TestMain(m *testing.M) { From ed6ca1d9ecd55ed09f76540eea0189c523f0fb37 Mon Sep 17 00:00:00 2001 From: ygsa Date: Thu, 17 Aug 2023 15:01:03 +0800 Subject: [PATCH 2/7] Switch MongoDB libraries. Update test data. https://github.com/influxdata/telegraf/pull/9493 (cherry picked from commit 7deb03a392b1e80096e383dbd43a7b7986de54b5) --- plugins/inputs/mongodb/mongodb_data_test.go | 70 +++++++++++--- plugins/inputs/mongodb/mongostat.go | 100 ++++++++++++++++---- 2 files changed, 137 insertions(+), 33 deletions(-) diff --git a/plugins/inputs/mongodb/mongodb_data_test.go b/plugins/inputs/mongodb/mongodb_data_test.go index e1eba728c32f0..378268916054d 100644 --- a/plugins/inputs/mongodb/mongodb_data_test.go +++ b/plugins/inputs/mongodb/mongodb_data_test.go @@ -64,7 +64,7 @@ func TestAddNonReplStats(t *testing.T) { d.AddDefaultStats() d.flush(&acc) - for key := range DefaultStats { + for key := range defaultStats { assert.True(t, acc.HasFloatField("mongodb", key) || acc.HasInt64Field("mongodb", key), key) } } @@ -85,7 +85,7 @@ func TestAddReplStats(t *testing.T) { d.AddDefaultStats() d.flush(&acc) - for key := range MmapStats { + for key := range mmapStats { assert.True(t, acc.HasInt64Field("mongodb", key), key) } } @@ -119,11 +119,11 @@ func TestAddWiredTigerStats(t *testing.T) { d.AddDefaultStats() d.flush(&acc) - for key := range WiredTigerStats { + for key := range wiredTigerStats { assert.True(t, acc.HasFloatField("mongodb", key), key) } - for key := range WiredTigerExtStats { + for key := range wiredTigerExtStats { assert.True(t, acc.HasFloatField("mongodb", key) || acc.HasInt64Field("mongodb", key), key) } @@ -146,7 +146,7 @@ func TestAddShardStats(t *testing.T) { d.AddDefaultStats() d.flush(&acc) - for key := range DefaultShardStats { + for key := range defaultShardStats { assert.True(t, acc.HasInt64Field("mongodb", key)) } } @@ -169,7 +169,7 @@ func TestAddLatencyStats(t *testing.T) { d.AddDefaultStats() d.flush(&acc) - for key := range DefaultLatencyStats { + for key := range defaultLatencyStats { assert.True(t, acc.HasInt64Field("mongodb", key)) } } @@ -191,7 +191,7 @@ func TestAddAssertsStats(t *testing.T) { d.AddDefaultStats() d.flush(&acc) - for key := range DefaultAssertsStats { + for key := range defaultAssertsStats { assert.True(t, acc.HasInt64Field("mongodb", key)) } } @@ -226,7 +226,7 @@ func TestAddCommandsStats(t *testing.T) { d.AddDefaultStats() d.flush(&acc) - for key := range DefaultCommandsStats { + for key := range defaultCommandsStats { assert.True(t, acc.HasInt64Field("mongodb", key)) } } @@ -262,7 +262,7 @@ func TestAddTCMallocStats(t *testing.T) { d.AddDefaultStats() d.flush(&acc) - for key := range DefaultTCMallocStats { + for key := range defaultTCMallocStats { assert.True(t, acc.HasInt64Field("mongodb", key)) } } @@ -282,7 +282,7 @@ func TestAddStorageStats(t *testing.T) { d.AddDefaultStats() d.flush(&acc) - for key := range DefaultStorageStats { + for key := range defaultStorageStats { assert.True(t, acc.HasInt64Field("mongodb", key)) } } @@ -312,11 +312,11 @@ func TestAddShardHostStats(t *testing.T) { var hostsFound []string for host := range hostStatLines { - for key := range ShardHostStats { + for key := range shardHostStats { assert.True(t, acc.HasInt64Field("mongodb_shard_stats", key)) } - assert.True(t, acc.HasTag("mongodb_shard_stats", "server")) + assert.True(t, acc.HasTag("mongodb_shard_stats", "hostname")) hostsFound = append(hostsFound, host) } sort.Strings(hostsFound) @@ -485,3 +485,49 @@ func TestStateTag(t *testing.T) { } acc.AssertContainsTaggedFields(t, "mongodb", fields, stateTags) } + +func TestAddTopStats(t *testing.T) { + collections := []string{"collectionOne", "collectionTwo"} + var topStatLines []TopStatLine + for _, collection := range collections { + topStatLine := TopStatLine{ + CollectionName: collection, + TotalTime: 0, + TotalCount: 0, + ReadLockTime: 0, + ReadLockCount: 0, + WriteLockTime: 0, + WriteLockCount: 0, + QueriesTime: 0, + QueriesCount: 0, + GetMoreTime: 0, + GetMoreCount: 0, + InsertTime: 0, + InsertCount: 0, + UpdateTime: 0, + UpdateCount: 0, + RemoveTime: 0, + RemoveCount: 0, + CommandsTime: 0, + CommandsCount: 0, + } + topStatLines = append(topStatLines, topStatLine) + } + + d := NewMongodbData( + &StatLine{ + TopStatLines: topStatLines, + }, + tags, + ) + + var acc testutil.Accumulator + d.AddTopStats() + d.flush(&acc) + + for range topStatLines { + for key := range topDataStats { + assert.True(t, acc.HasInt64Field("mongodb_top_stats", key)) + } + } +} diff --git a/plugins/inputs/mongodb/mongostat.go b/plugins/inputs/mongodb/mongostat.go index 995168fccb069..41f735d389c7a 100644 --- a/plugins/inputs/mongodb/mongostat.go +++ b/plugins/inputs/mongodb/mongostat.go @@ -37,9 +37,12 @@ type MongoStatus struct { ColStats *ColStats ShardStats *ShardStats OplogStats *OplogStats + TopStats *TopStats } type ServerStatus struct { + SampleTime time.Time `bson:""` + Flattened map[string]interface{} `bson:""` Host string `bson:"host"` Version string `bson:"version"` Process string `bson:"process"` @@ -63,7 +66,7 @@ type ServerStatus struct { Mem *MemStats `bson:"mem"` Repl *ReplStatus `bson:"repl"` ShardCursorType map[string]interface{} `bson:"shardCursorType"` - StorageEngine map[string]string `bson:"storageEngine"` + StorageEngine *StorageEngine `bson:"storageEngine"` WiredTiger *WiredTiger `bson:"wiredTiger"` Metrics *MetricsStats `bson:"metrics"` TCMallocStats *TCMallocStats `bson:"tcmalloc"` @@ -169,6 +172,27 @@ type ShardHostStatsData struct { Refreshing int64 `bson:"refreshing"` } +type TopStats struct { + Totals map[string]TopStatCollection `bson:"totals"` +} + +type TopStatCollection struct { + Total TopStatCollectionData `bson:"total"` + ReadLock TopStatCollectionData `bson:"readLock"` + WriteLock TopStatCollectionData `bson:"writeLock"` + Queries TopStatCollectionData `bson:"queries"` + GetMore TopStatCollectionData `bson:"getmore"` + Insert TopStatCollectionData `bson:"insert"` + Update TopStatCollectionData `bson:"update"` + Remove TopStatCollectionData `bson:"remove"` + Commands TopStatCollectionData `bson:"commands"` +} + +type TopStatCollectionData struct { + Time int64 `bson:"time"` + Count int64 `bson:"count"` +} + type ConcurrentTransactions struct { Write ConcurrentTransStats `bson:"write"` Read ConcurrentTransStats `bson:"read"` @@ -212,6 +236,10 @@ type CacheStats struct { UnmodifiedPagesEvicted int64 `bson:"unmodified pages evicted"` } +type StorageEngine struct { + Name string `bson:"name"` +} + // TransactionStats stores transaction checkpoints in WiredTiger. type TransactionStats struct { TransCheckpointsTotalTimeMsecs int64 `bson:"transaction checkpoint total time (msecs)"` @@ -220,7 +248,7 @@ type TransactionStats struct { // ReplStatus stores data related to replica sets. type ReplStatus struct { - SetName interface{} `bson:"setName"` + SetName string `bson:"setName"` IsMaster interface{} `bson:"ismaster"` Secondary interface{} `bson:"secondary"` IsReplicaSet interface{} `bson:"isreplicaset"` @@ -735,7 +763,6 @@ type StatLine struct { NumConnections int64 ReplSetName string NodeType string - NodeTypeInt int64 NodeState string NodeStateInt int64 @@ -769,6 +796,8 @@ type StatLine struct { // Shard Hosts stats field ShardHostStatsLines map[string]ShardHostStatLine + TopStatLines []TopStatLine + // TCMalloc stats field TCMallocCurrentAllocatedBytes int64 TCMallocHeapSize int64 @@ -826,6 +855,19 @@ type ShardHostStatLine struct { Refreshing int64 } +type TopStatLine struct { + CollectionName string + TotalTime, TotalCount int64 + ReadLockTime, ReadLockCount int64 + WriteLockTime, WriteLockCount int64 + QueriesTime, QueriesCount int64 + GetMoreTime, GetMoreCount int64 + InsertTime, InsertCount int64 + UpdateTime, UpdateCount int64 + RemoveTime, RemoveCount int64 + CommandsTime, CommandsCount int64 +} + func parseLocks(stat ServerStatus) map[string]LockUsage { returnVal := map[string]LockUsage{} for namespace, lockInfo := range stat.Locks { @@ -892,8 +934,8 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec returnVal.TotalCreatedC = newStat.Connections.TotalCreated // set the storage engine appropriately - if newStat.StorageEngine != nil && newStat.StorageEngine["name"] != "" { - returnVal.StorageEngine = newStat.StorageEngine["name"] + if newStat.StorageEngine != nil && newStat.StorageEngine.Name != "" { + returnVal.StorageEngine = newStat.StorageEngine.Name } else { returnVal.StorageEngine = "mmapv1" } @@ -1102,7 +1144,7 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec returnVal.Time = newMongo.SampleTime returnVal.IsMongos = - (newStat.ShardCursorType != nil || strings.HasPrefix(newStat.Process, MongosProcess)) + newStat.ShardCursorType != nil || strings.HasPrefix(newStat.Process, MongosProcess) // BEGIN code modification if oldStat.Mem.Supported.(bool) { @@ -1119,28 +1161,19 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec } if newStat.Repl != nil { - setName, isReplSet := newStat.Repl.SetName.(string) - if isReplSet { - returnVal.ReplSetName = setName - } + returnVal.ReplSetName = newStat.Repl.SetName // BEGIN code modification if newStat.Repl.IsMaster.(bool) { returnVal.NodeType = "PRI" - returnVal.NodeTypeInt = 1 } else if newStat.Repl.Secondary != nil && newStat.Repl.Secondary.(bool) { returnVal.NodeType = "SEC" - returnVal.NodeTypeInt = 2 } else if newStat.Repl.ArbiterOnly != nil && newStat.Repl.ArbiterOnly.(bool) { returnVal.NodeType = "ARB" - returnVal.NodeTypeInt = 7 } else { returnVal.NodeType = "UNK" - returnVal.NodeTypeInt = 6 - } - // END code modification + } // END code modification } else if returnVal.IsMongos { returnVal.NodeType = "RTR" - returnVal.NodeTypeInt = 99 } if oldStat.ExtraInfo != nil && newStat.ExtraInfo != nil && @@ -1186,9 +1219,7 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec // Get the entry with the highest lock highestLocked := lockdiffs[len(lockdiffs)-1] - var timeDiffMillis int64 - timeDiffMillis = newStat.UptimeMillis - oldStat.UptimeMillis - + timeDiffMillis := newStat.UptimeMillis - oldStat.UptimeMillis lockToReport := highestLocked.Writes // if the highest locked namespace is not '.' @@ -1216,7 +1247,7 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec } if newStat.GlobalLock != nil { - hasWT := (newStat.WiredTiger != nil && oldStat.WiredTiger != nil) + hasWT := newStat.WiredTiger != nil && oldStat.WiredTiger != nil //If we have wiredtiger stats, use those instead if newStat.GlobalLock.CurrentQueue != nil { if hasWT { @@ -1371,5 +1402,32 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec } } + if newMongo.TopStats != nil { + for collection, data := range newMongo.TopStats.Totals { + topStatDataLine := &TopStatLine{ + CollectionName: collection, + TotalTime: data.Total.Time, + TotalCount: data.Total.Count, + ReadLockTime: data.ReadLock.Time, + ReadLockCount: data.ReadLock.Count, + WriteLockTime: data.WriteLock.Time, + WriteLockCount: data.WriteLock.Count, + QueriesTime: data.Queries.Time, + QueriesCount: data.Queries.Count, + GetMoreTime: data.GetMore.Time, + GetMoreCount: data.GetMore.Count, + InsertTime: data.Insert.Time, + InsertCount: data.Insert.Count, + UpdateTime: data.Update.Time, + UpdateCount: data.Update.Count, + RemoveTime: data.Remove.Time, + RemoveCount: data.Remove.Count, + CommandsTime: data.Commands.Time, + CommandsCount: data.Commands.Count, + } + returnVal.TopStatLines = append(returnVal.TopStatLines, *topStatDataLine) + } + } + return returnVal } From ff2771ca3c39598094ae5868c0bf3f643027cdcc Mon Sep 17 00:00:00 2001 From: ygsa Date: Tue, 29 Aug 2023 18:23:19 +0800 Subject: [PATCH 3/7] Reset c.tlsCfg.ServerName between multiple certs/sources to fix [issue](https://github.com/influxdata/telegraf/issues/8914) https://github.com/influxdata/telegraf/pull/9289 --- plugins/inputs/x509_cert/x509_cert.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/plugins/inputs/x509_cert/x509_cert.go b/plugins/inputs/x509_cert/x509_cert.go index 49559ffccb515..22d5534a73826 100644 --- a/plugins/inputs/x509_cert/x509_cert.go +++ b/plugins/inputs/x509_cert/x509_cert.go @@ -87,14 +87,16 @@ func (c *X509Cert) serverName(u *url.URL) (string, error) { } func (c *X509Cert) getCert(u *url.URL, timeout time.Duration) ([]*x509.Certificate, error) { + protocol := u.Scheme switch u.Scheme { case "https": - u.Scheme = "tcp" + protocol = "tcp" + //u.Scheme = "tcp" fallthrough case "udp", "udp4", "udp6": fallthrough case "tcp", "tcp4", "tcp6": - ipConn, err := net.DialTimeout(u.Scheme, u.Host, timeout) + ipConn, err := net.DialTimeout(protocol, u.Host, timeout) if err != nil { return nil, err } @@ -109,7 +111,7 @@ func (c *X509Cert) getCert(u *url.URL, timeout time.Duration) ([]*x509.Certifica c.tlsCfg.InsecureSkipVerify = true conn := tls.Client(ipConn, c.tlsCfg) defer conn.Close() - + defer func() { c.tlsCfg.ServerName = "" }() hsErr := conn.Handshake() if hsErr != nil { return nil, hsErr @@ -268,7 +270,14 @@ func (c *X509Cert) Init() error { if tlsCfg == nil { tlsCfg = &tls.Config{} } - + if tlsCfg.ServerName != "" && c.ServerName == "" { + // Save SNI from tlsCfg.ServerName to c.ServerName and reset tlsCfg.ServerName. + // We need to reset c.tlsCfg.ServerName for each certificate when there's + // no explicit SNI (c.tlsCfg.ServerName or c.ServerName) otherwise we'll always (re)use + // first uri HostName for all certs (see issue 8914) + c.ServerName = tlsCfg.ServerName + tlsCfg.ServerName = "" + } c.tlsCfg = tlsCfg return nil From 29e8c258a27958861b494c8f2966f7e10d4ba312 Mon Sep 17 00:00:00 2001 From: ygsa Date: Tue, 12 Sep 2023 15:18:40 +0800 Subject: [PATCH 4/7] change mongodb to compatible graphite (cherry picked from commit 66e590898c4cc8fc3025d507699f6bf9f236c959) --- plugins/inputs/mongodb/mongodb_data.go | 57 ++++++++++++++----- plugins/inputs/mongodb/mongodb_data_test.go | 8 +-- plugins/inputs/mongodb/mongodb_server.go | 3 +- plugins/inputs/mongodb/mongodb_server_test.go | 2 +- 4 files changed, 50 insertions(+), 20 deletions(-) diff --git a/plugins/inputs/mongodb/mongodb_data.go b/plugins/inputs/mongodb/mongodb_data.go index e26c0e45231eb..89cdc396a21ad 100644 --- a/plugins/inputs/mongodb/mongodb_data.go +++ b/plugins/inputs/mongodb/mongodb_data.go @@ -12,6 +12,8 @@ type MongodbData struct { StatLine *StatLine Fields map[string]interface{} Tags map[string]string + MetaTags map[string]string + MetaFields map[string]interface{} DbData []DbData ColData []ColData ShardHostData []DbData @@ -29,12 +31,22 @@ type ColData struct { Fields map[string]interface{} } +func copyTagsMap(globalTags map[string]string) map[string]string { + tags := make(map[string]string) + for k, v := range globalTags { + tags[k] = v + } + + return tags +} func NewMongodbData(statLine *StatLine, tags map[string]string) *MongodbData { return &MongodbData{ - StatLine: statLine, - Tags: tags, - Fields: make(map[string]interface{}), - DbData: []DbData{}, + StatLine: statLine, + Tags: tags, + Fields: make(map[string]interface{}), + DbData: []DbData{}, + MetaTags: copyTagsMap(tags), + MetaFields: make(map[string]interface{}), } } @@ -301,7 +313,21 @@ func (d *MongodbData) AddDbStats() { d.DbData = append(d.DbData, *newDbData) } } +func (d *MongodbData) AddMetaInfo() { + if d.StatLine.ReplSetName != "" { + d.MetaTags["rs_name"] = d.StatLine.ReplSetName + } + if d.StatLine.Version != "" { + d.MetaTags["version"] = d.StatLine.Version + } + + if d.StatLine.StorageEngine != "" { + d.MetaTags["storage_engine"] = d.StatLine.StorageEngine + } + + d.addMeta("meta_info", 1) +} func (d *MongodbData) AddColStats() { for _, colstat := range d.StatLine.ColStatsLines { colStatLine := reflect.ValueOf(&colstat).Elem() @@ -356,25 +382,17 @@ func (d *MongodbData) AddDefaultStats() { d.addStat(statLine, defaultStats) if d.StatLine.NodeType != "" { d.addStat(statLine, defaultReplStats) - d.Tags["node_type"] = d.StatLine.NodeType + d.add("state_int", d.StatLine.NodeTypeInt) } if d.StatLine.ReadLatency > 0 { d.addStat(statLine, defaultLatencyStats) } - if d.StatLine.ReplSetName != "" { - d.Tags["rs_name"] = d.StatLine.ReplSetName - } - if d.StatLine.OplogStats != nil { d.add("repl_oplog_window_sec", d.StatLine.OplogStats.TimeDiff) } - if d.StatLine.Version != "" { - d.add("version", d.StatLine.Version) - } - d.addStat(statLine, defaultAssertsStats) d.addStat(statLine, defaultClusterStats) d.addStat(statLine, defaultCommandsStats) @@ -407,7 +425,18 @@ func (d *MongodbData) add(key string, val interface{}) { d.Fields[key] = val } +func (d *MongodbData) addMeta(key string, val interface{}) { + d.MetaFields[key] = val +} + func (d *MongodbData) flush(acc telegraf.Accumulator) { + acc.AddFields( + "mongodb_meta", + d.MetaFields, + d.MetaTags, + d.StatLine.Time, + ) + d.MetaFields = make(map[string]interface{}) acc.AddFields( "mongodb", d.Fields, @@ -438,7 +467,7 @@ func (d *MongodbData) flush(acc telegraf.Accumulator) { col.Fields = make(map[string]interface{}) } for _, host := range d.ShardHostData { - d.Tags["hostname"] = host.Name + d.Tags["server"] = host.Name acc.AddFields( "mongodb_shard_stats", host.Fields, diff --git a/plugins/inputs/mongodb/mongodb_data_test.go b/plugins/inputs/mongodb/mongodb_data_test.go index 378268916054d..ccf375afaae19 100644 --- a/plugins/inputs/mongodb/mongodb_data_test.go +++ b/plugins/inputs/mongodb/mongodb_data_test.go @@ -316,7 +316,7 @@ func TestAddShardHostStats(t *testing.T) { assert.True(t, acc.HasInt64Field("mongodb_shard_stats", key)) } - assert.True(t, acc.HasTag("mongodb_shard_stats", "hostname")) + assert.True(t, acc.HasTag("mongodb_shard_stats", "server")) hostsFound = append(hostsFound, host) } sort.Strings(hostsFound) @@ -340,14 +340,15 @@ func TestStateTag(t *testing.T) { ) stateTags := make(map[string]string) - stateTags["node_type"] = "PRI" - stateTags["rs_name"] = "rs1" + //stateTags["version"] = "3.6.17" + //stateTags["rs_name"] = "rs1" var acc testutil.Accumulator d.AddDefaultStats() d.flush(&acc) fields := map[string]interface{}{ + "state_int": int64(0), "active_reads": int64(0), "active_writes": int64(0), "aggregate_command_failed": int64(0), @@ -480,7 +481,6 @@ func TestStateTag(t *testing.T) { "updates": int64(0), "updates_per_sec": int64(0), "uptime_ns": int64(0), - "version": "3.6.17", "vsize_megabytes": int64(0), } acc.AssertContainsTaggedFields(t, "mongodb", fields, stateTags) diff --git a/plugins/inputs/mongodb/mongodb_server.go b/plugins/inputs/mongodb/mongodb_server.go index 9232e20f21ec6..f1023e1504ad0 100644 --- a/plugins/inputs/mongodb/mongodb_server.go +++ b/plugins/inputs/mongodb/mongodb_server.go @@ -25,7 +25,7 @@ type Server struct { func (s *Server) getDefaultTags() map[string]string { tags := make(map[string]string) - tags["hostname"] = s.hostname + tags["server"] = s.hostname return tags } @@ -355,6 +355,7 @@ func (s *Server) gatherData(acc telegraf.Accumulator, gatherClusterStatus bool, s.getDefaultTags(), ) data.AddDefaultStats() + data.AddMetaInfo() data.AddDbStats() data.AddColStats() data.AddShardHostStats() diff --git a/plugins/inputs/mongodb/mongodb_server_test.go b/plugins/inputs/mongodb/mongodb_server_test.go index 2cf58689a6eab..a5a4604ee296b 100644 --- a/plugins/inputs/mongodb/mongodb_server_test.go +++ b/plugins/inputs/mongodb/mongodb_server_test.go @@ -15,7 +15,7 @@ func TestGetDefaultTags(t *testing.T) { in string out string }{ - {"hostname", server.hostname}, + {"server", server.hostname}, } defaultTags := server.getDefaultTags() for _, tt := range tagTests { From 31f9f7a1229878c040d5f79e01cb1e4e1a105f63 Mon Sep 17 00:00:00 2001 From: ygsa Date: Tue, 12 Sep 2023 15:18:06 +0800 Subject: [PATCH 5/7] Support mongodb 5.x [issue](https://github.com/influxdata/telegraf/issues/9968) hhttps://github.com/influxdata/telegraf/pull/9892 (cherry picked from commit bf65b02d37c73f5f8ff82ecc842b2e8f6be5279d) --- plugins/inputs/mongodb/mongostat.go | 32 +++++++++++++++++++---------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/plugins/inputs/mongodb/mongostat.go b/plugins/inputs/mongodb/mongostat.go index 41f735d389c7a..6441a8fda183c 100644 --- a/plugins/inputs/mongodb/mongostat.go +++ b/plugins/inputs/mongodb/mongostat.go @@ -248,14 +248,15 @@ type TransactionStats struct { // ReplStatus stores data related to replica sets. type ReplStatus struct { - SetName string `bson:"setName"` - IsMaster interface{} `bson:"ismaster"` - Secondary interface{} `bson:"secondary"` - IsReplicaSet interface{} `bson:"isreplicaset"` - ArbiterOnly interface{} `bson:"arbiterOnly"` - Hosts []string `bson:"hosts"` - Passives []string `bson:"passives"` - Me string `bson:"me"` + SetName string `bson:"setName"` + IsWritablePrimary interface{} `bson:"isWritablePrimary"` // mongodb 5.x + IsMaster interface{} `bson:"ismaster"` + Secondary interface{} `bson:"secondary"` + IsReplicaSet interface{} `bson:"isreplicaset"` + ArbiterOnly interface{} `bson:"arbiterOnly"` + Hosts []string `bson:"hosts"` + Passives []string `bson:"passives"` + Me string `bson:"me"` } // DBRecordStats stores data related to memory operations across databases. @@ -763,6 +764,7 @@ type StatLine struct { NumConnections int64 ReplSetName string NodeType string + NodeTypeInt int64 NodeState string NodeStateInt int64 @@ -1163,17 +1165,25 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec if newStat.Repl != nil { returnVal.ReplSetName = newStat.Repl.SetName // BEGIN code modification - if newStat.Repl.IsMaster.(bool) { + if val, ok := newStat.Repl.IsMaster.(bool); ok && val { returnVal.NodeType = "PRI" - } else if newStat.Repl.Secondary != nil && newStat.Repl.Secondary.(bool) { + returnVal.NodeTypeInt = 1 + } else if val, ok := newStat.Repl.IsWritablePrimary.(bool); ok && val { + returnVal.NodeType = "PRI" + returnVal.NodeTypeInt = 1 + } else if val, ok := newStat.Repl.Secondary.(bool); ok && val { returnVal.NodeType = "SEC" - } else if newStat.Repl.ArbiterOnly != nil && newStat.Repl.ArbiterOnly.(bool) { + returnVal.NodeTypeInt = 2 + } else if val, ok := newStat.Repl.ArbiterOnly.(bool); ok && val { returnVal.NodeType = "ARB" + returnVal.NodeTypeInt = 7 } else { returnVal.NodeType = "UNK" + returnVal.NodeTypeInt = 6 } // END code modification } else if returnVal.IsMongos { returnVal.NodeType = "RTR" + returnVal.NodeTypeInt = 99 } if oldStat.ExtraInfo != nil && newStat.ExtraInfo != nil && From 97a8e250bd858ee5af3cd7ca8ab9522ba9fa5d2e Mon Sep 17 00:00:00 2001 From: ygsa Date: Thu, 30 Nov 2023 18:02:53 +0800 Subject: [PATCH 6/7] Change netPort metrics tags to support only gather tcp/udp port status and compatible old metrics. --- plugins/inputs/net/netport.go | 77 ++++++++++++++++++++++++++--------- 1 file changed, 57 insertions(+), 20 deletions(-) diff --git a/plugins/inputs/net/netport.go b/plugins/inputs/net/netport.go index 2abbceb8e1654..b53c5cdd12180 100644 --- a/plugins/inputs/net/netport.go +++ b/plugins/inputs/net/netport.go @@ -2,18 +2,24 @@ package net import ( "fmt" - "syscall" "strconv" + "syscall" - "github.com/shirou/gopsutil/net" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/inputs" "github.com/influxdata/telegraf/plugins/inputs/system" + "github.com/shirou/gopsutil/net" +) + +const ( + TcpProtocol = "tcp" + UdpProtocol = "udp" ) type NetPorts struct { - ps system.PS - Ports []uint32 `toml:"ports"` + ps system.PS + Ports []uint32 `toml:"ports"` + Protocol string `toml:"protocol"` } func (ns *NetPorts) Description() string { @@ -23,6 +29,8 @@ func (ns *NetPorts) Description() string { var portSampleConfig = ` ## gather the port connection status if you set ports parameter # ports = [22, 30022] + ## protocol need tcp or udp, default will gather all. + # protocol = "tcp" ` func (ns *NetPorts) SampleConfig() string { @@ -40,12 +48,11 @@ func (ns *NetPorts) Gather(acc telegraf.Accumulator) error { return nil } - func (ns *NetPorts) gatherPorts(ports []uint32, netconns []net.ConnectionStat, acc telegraf.Accumulator) error { portsRes := map[uint32]map[string]int{} for _, port := range ports { portsRes[port] = map[string]int{ - "UDP":0, + "UDP": 0, } } @@ -69,21 +76,51 @@ func (ns *NetPorts) gatherPorts(ports []uint32, netconns []net.ConnectionStat, a } for port, counts := range portsRes { + tags := map[string]string{"port": strconv.Itoa(int(port))} - fields := map[string]interface{}{ - "tcp_established": counts["ESTABLISHED"], - "tcp_syn_sent": counts["SYN_SENT"], - "tcp_syn_recv": counts["SYN_RECV"], - "tcp_fin_wait1": counts["FIN_WAIT1"], - "tcp_fin_wait2": counts["FIN_WAIT2"], - "tcp_time_wait": counts["TIME_WAIT"], - "tcp_close": counts["CLOSE"], - "tcp_close_wait": counts["CLOSE_WAIT"], - "tcp_last_ack": counts["LAST_ACK"], - "tcp_listen": counts["LISTEN"], - "tcp_closing": counts["CLOSING"], - "tcp_none": counts["NONE"], - "udp_socket": counts["UDP"], + fields := map[string]interface{}{} + + switch ns.Protocol { + case TcpProtocol: + tags["protocol"] = TcpProtocol + + fields["tcp_established"] = counts["ESTABLISHED"] + fields["tcp_syn_sent"] = counts["SYN_SENT"] + fields["tcp_established"] = counts["ESTABLISHED"] + fields["tcp_syn_sent"] = counts["SYN_SENT"] + fields["tcp_syn_recv"] = counts["SYN_RECV"] + fields["tcp_fin_wait1"] = counts["FIN_WAIT1"] + fields["tcp_fin_wait2"] = counts["FIN_WAIT2"] + fields["tcp_time_wait"] = counts["TIME_WAIT"] + fields["tcp_close"] = counts["CLOSE"] + fields["tcp_close_wait"] = counts["CLOSE_WAIT"] + fields["tcp_last_ack"] = counts["LAST_ACK"] + fields["tcp_listen"] = counts["LISTEN"] + fields["tcp_closing"] = counts["CLOSING"] + fields["tcp_none"] = counts["NONE"] + + case UdpProtocol: + tags["protocol"] = UdpProtocol + + fields["udp_socket"] = counts["UDP"] + + default: + fields["tcp_established"] = counts["ESTABLISHED"] + fields["tcp_syn_sent"] = counts["SYN_SENT"] + fields["tcp_established"] = counts["ESTABLISHED"] + fields["tcp_syn_sent"] = counts["SYN_SENT"] + fields["tcp_syn_recv"] = counts["SYN_RECV"] + fields["tcp_fin_wait1"] = counts["FIN_WAIT1"] + fields["tcp_fin_wait2"] = counts["FIN_WAIT2"] + fields["tcp_time_wait"] = counts["TIME_WAIT"] + fields["tcp_close"] = counts["CLOSE"] + fields["tcp_close_wait"] = counts["CLOSE_WAIT"] + fields["tcp_last_ack"] = counts["LAST_ACK"] + fields["tcp_listen"] = counts["LISTEN"] + fields["tcp_closing"] = counts["CLOSING"] + fields["tcp_none"] = counts["NONE"] + fields["udp_socket"] = counts["UDP"] + } acc.AddFields("netport", fields, tags) } From d805ee5b1d0de904dfcb2b0a5b27a57c1ea45427 Mon Sep 17 00:00:00 2001 From: ygsa Date: Fri, 22 Dec 2023 16:55:11 +0800 Subject: [PATCH 7/7] Fix docker healthy status matching. --- plugins/inputs/docker/docker.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/plugins/inputs/docker/docker.go b/plugins/inputs/docker/docker.go index ae6e301d00151..b9f5e610469df 100644 --- a/plugins/inputs/docker/docker.go +++ b/plugins/inputs/docker/docker.go @@ -643,12 +643,16 @@ func (d *Docker) gatherContainerInspect( } healthStatusVal := -1 switch info.State.Health.Status { - case "Starting": + case types.NoHealthcheck: + healthStatusVal = 3 + case types.Starting: healthStatusVal = 2 - case "Healthy": + case types.Healthy: healthStatusVal = 1 - case "Unhealthy": + case types.Unhealthy: healthStatusVal = 0 + default: + d.Log.Warn("docker: unknown health status [%s]", info.State.Health.Status) } healthfields["health_status"] = healthStatusVal acc.AddFields("docker_container_health", healthfields, tags, now())