diff --git a/.github/workflows/add-to-core-team-project.yml b/.github/workflows/add-to-core-team-project.yml new file mode 100644 index 0000000000..a913c8589d --- /dev/null +++ b/.github/workflows/add-to-core-team-project.yml @@ -0,0 +1,17 @@ +name: Add New Issues To Core Team Project + +on: + issues: + types: + - opened + +jobs: + add-to-project: + name: Add issue to project + runs-on: ubuntu-latest + steps: + - uses: actions/add-to-project@v0.1.0 + with: + project-url: https://github.com/orgs/golemfactory/projects/30 + github-token: ${{ secrets.CORE_BOARD_ACTIONS_TOKEN }} + diff --git a/.github/workflows/dod-checker.yml b/.github/workflows/dod-checker.yml new file mode 100644 index 0000000000..f2c08d2808 --- /dev/null +++ b/.github/workflows/dod-checker.yml @@ -0,0 +1,16 @@ +name: Definition of Done +on: + pull_request: + types: [opened, edited] + +jobs: + check-dod: + runs-on: ubuntu-20.04 + steps: + - name: Clone Repo + uses: actions/checkout@v3 + - name: Check DoD + uses: platisd/definition-of-done@master + with: + dod_yaml: 'dod.yml' + github_token: ${{ secrets.CORE_BOARD_ACTIONS_TOKEN }} diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index d35efdd463..5c0c135f7e 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -44,6 +44,12 @@ jobs: command: fmt args: --all -- --check + - name: Check clippy lints + uses: actions-rs/cargo@v1 + with: + command: clippy + args: --all-targets --all-features --workspace -- -D warnings + - name: Install openssl ( Windows only ) if: runner.os == 'Windows' run: | @@ -56,4 +62,3 @@ jobs: with: command: test args: --workspace --exclude=["./agent/provider/src/market"] --locked - diff --git a/.gitignore b/.gitignore index 2e8a98a5f1..b2fb991625 100644 --- a/.gitignore +++ b/.gitignore @@ -25,12 +25,14 @@ releases/ # Provider dir **/ya-prov +**/ya-prov_hybrid # Requestor dir **/ya-req +**/ya-req_hybrid # Keys **/*.key # bak files (generated by notepad++) -**/*.bak \ No newline at end of file +**/*.bak diff --git a/Cargo.lock b/Cargo.lock index 3a8fbc85b7..f8ae34330b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,10 +11,10 @@ dependencies = [ "abi_stable_derive", "abi_stable_shared", "core_extensions", - "crossbeam-channel 0.5.4", + "crossbeam-channel 0.5.6", "generational-arena", "libloading", - "lock_api 0.4.7", + "lock_api 0.4.9", "parking_lot 0.11.2", "repr_offset", "rustc_version 0.2.3", @@ -57,8 +57,8 @@ dependencies = [ "actix-rt", "actix_derive", "bitflags", - "bytes 1.1.0", - "crossbeam-channel 0.5.4", + "bytes 1.2.1", + "crossbeam-channel 0.5.6", "futures-core", "futures-sink", "futures-task", @@ -68,8 +68,8 @@ dependencies = [ "parking_lot 0.12.1", "pin-project-lite 0.2.9", "smallvec", - "tokio 1.19.1", - "tokio-util 0.7.3", + "tokio 1.21.2", + "tokio-util 0.7.4", ] [[package]] @@ -79,21 +79,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57a7559404a7f3573127aab53c08ce37a6c6a315c374a31070f3c91cd1b4a7fe" dependencies = [ "bitflags", - "bytes 1.1.0", + "bytes 1.2.1", "futures-core", "futures-sink", "log", "memchr", "pin-project-lite 0.2.9", - "tokio 1.19.1", - "tokio-util 0.7.3", + "tokio 1.21.2", + "tokio-util 0.7.4", ] [[package]] name = "actix-files" -version = "0.6.0" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d81bde9a79336aa51ebed236e91fc1a0528ff67cfdf4f68ca4c61ede9fd26fb5" +checksum = "d832782fac6ca7369a70c9ee9a20554623c5e51c76e190ad151780ebea1cf689" dependencies = [ "actix-http", "actix-service", @@ -101,7 +101,7 @@ dependencies = [ "actix-web", "askama_escape", "bitflags", - "bytes 1.1.0", + "bytes 1.2.1", "derive_more", "futures-core", "http-range", @@ -114,9 +114,9 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.0.4" +version = "3.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5885cb81a0d4d0d322864bea1bb6c2a8144626b4fdc625d4c51eba197e7797a" +checksum = "0c83abf9903e1f0ad9973cc4f7b9767fd5a03a583f51a5b7a339e07987cd2724" dependencies = [ "actix-codec", "actix-rt", @@ -127,26 +127,26 @@ dependencies = [ "base64 0.13.0", "bitflags", "brotli", - "bytes 1.1.0", + "bytes 1.2.1", "bytestring", "derive_more", "encoding_rs", "flate2", "futures-core", - "h2 0.3.13", + "h2 0.3.14", "http", "httparse", "httpdate 1.0.2", - "itoa 1.0.2", + "itoa 1.0.4", "language-tags", "local-channel", - "log", "mime", "percent-encoding", "pin-project-lite 0.2.9", "rand 0.8.5", - "sha-1 0.10.0", + "sha1", "smallvec", + "tracing", "zstd", ] @@ -162,16 +162,15 @@ dependencies = [ [[package]] name = "actix-router" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb60846b52c118f2f04a56cc90880a274271c489b2498623d58176f8ca21fa80" +checksum = "d66ff4d247d2b160861fa2866457e85706833527840e4133f8f49aa423a38799" dependencies = [ "bytestring", - "firestorm", "http", - "log", "regex", "serde", + "tracing", ] [[package]] @@ -182,7 +181,7 @@ checksum = "7ea16c295198e958ef31930a6ef37d0fb64e9ca3b6116e6b93a8bdae96ee1000" dependencies = [ "actix-macros", "futures-core", - "tokio 1.19.1", + "tokio 1.21.2", ] [[package]] @@ -196,10 +195,10 @@ dependencies = [ "actix-utils", "futures-core", "futures-util", - "mio 0.8.3", + "mio 0.8.4", "num_cpus", - "socket2 0.4.4", - "tokio 1.19.1", + "socket2 0.4.7", + "tokio 1.21.2", "tracing", ] @@ -230,7 +229,7 @@ dependencies = [ "openssl", "pin-project-lite 0.2.9", "tokio-openssl", - "tokio-util 0.7.3", + "tokio-util 0.7.4", ] [[package]] @@ -245,9 +244,9 @@ dependencies = [ [[package]] name = "actix-web" -version = "4.0.1" +version = "4.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4e5ebffd51d50df56a3ae0de0e59487340ca456f05dd0b90c0a7a6dd6a74d31" +checksum = "d48f7b6534e06c7bfc72ee91db7917d4af6afe23e7d223b51e68fffbb21e96b9" dependencies = [ "actix-codec", "actix-http", @@ -260,7 +259,7 @@ dependencies = [ "actix-utils", "actix-web-codegen", "ahash 0.7.6", - "bytes 1.1.0", + "bytes 1.2.1", "bytestring", "cfg-if 1.0.0", "cookie", @@ -268,7 +267,8 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "itoa 1.0.2", + "http", + "itoa 1.0.4", "language-tags", "log", "mime", @@ -279,7 +279,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "smallvec", - "socket2 0.4.4", + "socket2 0.4.7", "time 0.3.9", "url", ] @@ -294,18 +294,18 @@ dependencies = [ "actix-codec", "actix-http", "actix-web", - "bytes 1.1.0", + "bytes 1.2.1", "bytestring", "futures-core", "pin-project-lite 0.2.9", - "tokio 1.19.1", + "tokio 1.21.2", ] [[package]] name = "actix-web-codegen" -version = "4.0.0" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7525bedf54704abb1d469e88d7e7e9226df73778798a69cea5022d53b2ae91bc" +checksum = "1fa9362663c8643d67b2d5eafba49e4cb2c8a053a29ed00a0bea121f17c76b13" dependencies = [ "actix-router", "proc-macro2", @@ -409,16 +409,16 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" dependencies = [ - "getrandom 0.2.6", + "getrandom 0.2.7", "once_cell", "version_check", ] [[package]] name = "aho-corasick" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e" dependencies = [ "memchr", ] @@ -442,19 +442,28 @@ checksum = "ca77caf0ca1057c274cda103cda1363d892b7cad5f2e646afde4df0697bea100" [[package]] name = "alloc-no-stdlib" -version = "2.0.3" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35ef4730490ad1c4eae5c4325b2a95f521d023e5c885853ff7aca0a6a1631db3" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" [[package]] name = "alloc-stdlib" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "697ed7edc0f1711de49ce108c541623a0af97c6c60b2f6e2b65229847ac843c2" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "ansi_term" version = "0.12.1" @@ -466,9 +475,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.57" +version = "1.0.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc" +checksum = "98161a4e3e2184da77bb14f02184cdd111e83bbbcc9979dfee3c44b9a85f5602" [[package]] name = "appdirs" @@ -550,6 +559,20 @@ dependencies = [ "nom", ] +[[package]] +name = "assert_cmd" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93ae1ddd39efd67689deb1979d80bad3bf7f2b09c6e6117c8d1f2443b5e2f83e" +dependencies = [ + "bstr", + "doc-comment", + "predicates", + "predicates-core", + "predicates-tree", + "wait-timeout", +] + [[package]] name = "async-compression" version = "0.3.7" @@ -563,7 +586,7 @@ dependencies = [ "futures-io", "memchr", "pin-project-lite 0.2.9", - "tokio 1.19.1", + "tokio 1.21.2", "xz2", ] @@ -581,9 +604,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.56" +version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96cf8829f67d2eab0b2dfa42c5d0ef737e0724e4a82b01b3e292456202b19716" +checksum = "76464446b8bc32758d7e88ee1a804d9914cd9b1cb264c029899680b0be29826f" dependencies = [ "proc-macro2", "quote", @@ -627,9 +650,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "awc" -version = "3.0.0" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65c60c44fbf3c8cee365e86b97d706e513b733c4eeb16437b45b88d2fffe889a" +checksum = "80ca7ff88063086d2e2c70b9f3b29b2fcd999bac68ac21731e66781970d68519" dependencies = [ "actix-codec", "actix-http", @@ -639,15 +662,15 @@ dependencies = [ "actix-utils", "ahash 0.7.6", "base64 0.13.0", - "bytes 1.1.0", + "bytes 1.2.1", "cfg-if 1.0.0", "cookie", "derive_more", "futures-core", "futures-util", - "h2 0.3.13", + "h2 0.3.14", "http", - "itoa 1.0.2", + "itoa 1.0.4", "log", "mime", "openssl", @@ -657,7 +680,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "tokio 1.19.1", + "tokio 1.21.2", ] [[package]] @@ -672,9 +695,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.65" +version = "0.3.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11a17d453482a265fd5f8479f2a3f405566e6ca627837aaddb85af8b1ab8ef61" +checksum = "cab84319d616cfb654d03394f38ab7e6f0919e181b1b57e1fd15e7fb4077d9a7" dependencies = [ "addr2line", "cc", @@ -706,7 +729,7 @@ checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" [[package]] name = "bellman_ce" version = "0.3.2" -source = "git+https://github.com/matter-labs/bellman?branch=beta#5809cc165db0a2e15be34c844fec568d8d6005bc" +source = "git+https://github.com/matter-labs/bellman?branch=beta#416f79d3f93fc855fb96bb61bfe73d2472e95548" dependencies = [ "bit-vec", "blake2s_const", @@ -714,7 +737,7 @@ dependencies = [ "byteorder", "cfg-if 1.0.0", "crossbeam", - "futures 0.3.21", + "futures 0.3.24", "hex", "lazy_static", "num_cpus", @@ -825,7 +848,7 @@ dependencies = [ [[package]] name = "blake2s_const" version = "0.6.0" -source = "git+https://github.com/matter-labs/bellman?branch=beta#5809cc165db0a2e15be34c844fec568d8d6005bc" +source = "git+https://github.com/matter-labs/bellman?branch=beta#416f79d3f93fc855fb96bb61bfe73d2472e95548" dependencies = [ "arrayref", "arrayvec 0.5.2", @@ -862,16 +885,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" dependencies = [ "block-padding 0.2.1", - "generic-array 0.14.5", + "generic-array 0.14.6", ] [[package]] name = "block-buffer" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324" +checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" dependencies = [ - "generic-array 0.14.5", + "generic-array 0.14.6", ] [[package]] @@ -934,9 +957,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.10.0" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3" +checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d" [[package]] name = "byte-slice-cast" @@ -979,9 +1002,9 @@ checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" [[package]] name = "bytes" -version = "1.1.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" +checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" [[package]] name = "bytesize" @@ -991,11 +1014,11 @@ checksum = "6c58ec36aac5066d5ca17df51b3e70279f5670a72102f5752cb7e7c856adfc70" [[package]] name = "bytestring" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90706ba19e97b90786e19dc0d5e2abd80008d99d4c0c5d1ad0b5e72cec7c494d" +checksum = "86b6a75fd3048808ef06af5cd79712be8111960adaf89d90250974b38fc3928a" dependencies = [ - "bytes 1.1.0", + "bytes 1.2.1", ] [[package]] @@ -1052,16 +1075,18 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.19" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1" dependencies = [ - "libc", + "iana-time-zone", + "js-sys", "num-integer", "num-traits", "rustc-serialize", "serde", - "time 0.1.43", + "time 0.1.44", + "wasm-bindgen", "winapi 0.3.9", ] @@ -1071,7 +1096,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12f8e7987cbd042a63249497f41aed09f8e65add917ea6566effbc56578d6801" dependencies = [ - "generic-array 0.14.5", + "generic-array 0.14.6", ] [[package]] @@ -1101,6 +1126,17 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "clipboard-win" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4ab1b92798304eedc095b53942963240037c0516452cb11aeba709d420b2219" +dependencies = [ + "error-code", + "str-buf", + "winapi 0.3.9", +] + [[package]] name = "cloudabi" version = "0.0.3" @@ -1119,6 +1155,16 @@ dependencies = [ "cc", ] +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + [[package]] name = "console" version = "0.10.3" @@ -1126,7 +1172,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2586208b33573b7f76ccfbe5adb076394c88deaf81b84d7213969805b0a952a7" dependencies = [ "clicolors-control", - "encode_unicode", + "encode_unicode 0.3.6", "lazy_static", "libc", "regex", @@ -1138,14 +1184,13 @@ dependencies = [ [[package]] name = "console" -version = "0.15.0" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28b32d32ca44b70c3e4acd7db1babf555fa026e385fb95f18028f88848b3c31" +checksum = "c050367d967ced717c04b65d8c619d863ef9292ce0c5760028655a2fb298718c" dependencies = [ - "encode_unicode", + "encode_unicode 0.3.6", + "lazy_static", "libc", - "once_cell", - "regex", "terminal_size", "unicode-width", "winapi 0.3.9", @@ -1165,9 +1210,9 @@ checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" [[package]] name = "cookie" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94d4706de1b0fa5b132270cddffa8585166037822e260a944fe161acd137ca05" +checksum = "344adc371239ef32293cb1c4fe519592fcf21206c79c02854320afcdf3ab4917" dependencies = [ "percent-encoding", "time 0.3.9", @@ -1201,9 +1246,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.2" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b" +checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" dependencies = [ "libc", ] @@ -1243,12 +1288,12 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.4" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53" +checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.8", + "crossbeam-utils 0.8.12", ] [[package]] @@ -1264,13 +1309,13 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" +checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" dependencies = [ "cfg-if 1.0.0", - "crossbeam-epoch 0.9.8", - "crossbeam-utils 0.8.8", + "crossbeam-epoch 0.9.11", + "crossbeam-utils 0.8.12", ] [[package]] @@ -1290,14 +1335,13 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.8" +version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c" +checksum = "f916dfc5d356b0ed9dae65f1db9fc9770aa2851d2662b988ccf4fe3516e86348" dependencies = [ "autocfg 1.1.0", "cfg-if 1.0.0", - "crossbeam-utils 0.8.8", - "lazy_static", + "crossbeam-utils 0.8.12", "memoffset 0.6.5", "scopeguard", ] @@ -1326,12 +1370,11 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.8" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" +checksum = "edbafec5fa1f196ca66527c1b12c2ec4745ca14b50f1ad8f9f6f720b55d11fac" dependencies = [ "cfg-if 1.0.0", - "lazy_static", ] [[package]] @@ -1343,7 +1386,7 @@ dependencies = [ "bitflags", "crossterm_winapi", "libc", - "mio 0.8.3", + "mio 0.8.4", "parking_lot 0.12.1", "signal-hook", "signal-hook-mio", @@ -1367,11 +1410,11 @@ checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" [[package]] name = "crypto-common" -version = "0.1.3" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ - "generic-array 0.14.5", + "generic-array 0.14.6", "typenum", ] @@ -1381,7 +1424,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" dependencies = [ - "generic-array 0.14.5", + "generic-array 0.14.6", "subtle", ] @@ -1391,7 +1434,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bff07008ec701e8028e2ceb8f83f0e4274ee62bd2dbdc4fefff2e9a91824081a" dependencies = [ - "generic-array 0.14.5", + "generic-array 0.14.6", "subtle", ] @@ -1401,7 +1444,7 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" dependencies = [ - "generic-array 0.14.5", + "generic-array 0.14.6", "subtle", ] @@ -1429,9 +1472,9 @@ dependencies = [ [[package]] name = "ctor" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f877be4f7c9f246b183111634f75baa039715e3f46ce860677d3b19a69fb229c" +checksum = "cdffe87e1d521a10f9696f833fe502293ea446d7f256c06128293a4119bdf4cb" dependencies = [ "quote", "syn", @@ -1446,6 +1489,60 @@ dependencies = [ "cipher", ] +[[package]] +name = "ctrlc" +version = "3.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d91974fbbe88ec1df0c24a4f00f99583667a7e2e6272b2b92d294d81e462173" +dependencies = [ + "nix 0.25.0", + "winapi 0.3.9", +] + +[[package]] +name = "cxx" +version = "1.0.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f39818dcfc97d45b03953c1292efc4e80954e1583c4aa770bac1383e2310a4" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e580d70777c116df50c390d1211993f62d40302881e54d4b79727acb83d0199" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56a46460b88d1cec95112c8c363f0e2c39afdb237f60583b0b36343bf627ea9c" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "747b608fecf06b0d72d440f27acc99288207324b793be2c17991839f3d4995ea" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "darling" version = "0.10.2" @@ -1526,6 +1623,19 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "dashmap" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" +dependencies = [ + "cfg-if 1.0.0", + "hashbrown 0.12.3", + "lock_api 0.4.9", + "once_cell", + "parking_lot_core 0.9.3", +] + [[package]] name = "data-encoding" version = "2.3.2" @@ -1612,6 +1722,12 @@ dependencies = [ "migrations_macros", ] +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + [[package]] name = "digest" version = "0.8.1" @@ -1627,16 +1743,16 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" dependencies = [ - "generic-array 0.14.5", + "generic-array 0.14.6", ] [[package]] name = "digest" -version = "0.10.3" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" +checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c" dependencies = [ - "block-buffer 0.10.2", + "block-buffer 0.10.3", "crypto-common", ] @@ -1703,6 +1819,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + [[package]] name = "dotenv" version = "0.15.0" @@ -1711,9 +1833,9 @@ checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" [[package]] name = "either" -version = "1.6.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" [[package]] name = "encode_unicode" @@ -1721,6 +1843,12 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "encoding_rs" version = "0.8.31" @@ -1730,11 +1858,17 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "endian-type" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" + [[package]] name = "enum-as-inner" -version = "0.3.4" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "570d109b813e904becc80d8d5da38376818a143348413f7149f1340fe04754d4" +checksum = "21cdad81446a7f7dc43f6a77409efeb9733d2fa65553efef6018ef257c959b73" dependencies = [ "heck 0.4.0", "proc-macro2", @@ -1784,6 +1918,37 @@ dependencies = [ "winapi 0.2.8", ] +[[package]] +name = "errno" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +dependencies = [ + "errno-dragonfly", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "error-code" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64f18991e7bf11e7ffee451b5318b5c1a73c52d0d0ada6e5a3017c8c1ced6a21" +dependencies = [ + "libc", + "str-buf", +] + [[package]] name = "ethabi" version = "14.1.0" @@ -1797,7 +1962,7 @@ dependencies = [ "serde_json", "sha3 0.9.1", "thiserror", - "uint 0.9.3", + "uint 0.9.4", ] [[package]] @@ -1852,7 +2017,7 @@ dependencies = [ "impl-rlp", "impl-serde", "primitive-types 0.8.0", - "uint 0.9.3", + "uint 0.9.4", ] [[package]] @@ -1866,7 +2031,7 @@ dependencies = [ "impl-rlp", "impl-serde", "primitive-types 0.9.1", - "uint 0.9.3", + "uint 0.9.4", ] [[package]] @@ -1891,13 +2056,24 @@ checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" [[package]] name = "fastrand" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3fcf0cee53519c866c09b5de1f6c56ff9d647101f81c1964fa632e148896cdf" +checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" dependencies = [ "instant", ] +[[package]] +name = "fd-lock" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11dcc7e4d79a8c89b9ab4c6f5c30b1fc4a83c420792da3542fd31179ed5f517" +dependencies = [ + "cfg-if 1.0.0", + "rustix", + "windows-sys", +] + [[package]] name = "ff_ce" version = "0.12.0" @@ -1936,22 +2112,16 @@ dependencies = [ [[package]] name = "filetime" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0408e2626025178a6a7f7ffc05a25bc47103229f19c113755de7bf63816290c" +checksum = "e94a7bbaa59354bc20dd75b67f23e2797b4490e9d6928203fb105c79e448c86c" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall 0.2.13", - "winapi 0.3.9", + "redox_syscall 0.2.16", + "windows-sys", ] -[[package]] -name = "firestorm" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c5f6c2c942da57e2aaaa84b8a521489486f14e75e7fa91dab70aba913975f98" - [[package]] name = "fixed-hash" version = "0.7.0" @@ -1972,9 +2142,9 @@ checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d" [[package]] name = "fixedbitset" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "279fb028e20b3c4c320317955b77c5e0c9701f05a1d309905d6fc702cdc5053e" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" @@ -1988,9 +2158,9 @@ dependencies = [ [[package]] name = "flexbuffers" -version = "0.2.2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3556c7e0d5b671814709229de0b91aa1e6dff7844df6102cd6b7ec6c3a46a187" +checksum = "15d14128f06405808ce75bfebe11e9b0f9da18719ede6d7bdb1702d6bfe0f7e8" dependencies = [ "bitflags", "byteorder", @@ -2018,9 +2188,9 @@ dependencies = [ [[package]] name = "flexi_logger" -version = "0.22.5" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee9a6796ff68a1014f6665dac55341820f26e63ec706e58bfaee468cf0ac174f" +checksum = "0c76a80dd14a27fc3d8bc696502132cb52b3f227256fd8601166c3a35e45f409" dependencies = [ "ansi_term", "atty", @@ -2033,6 +2203,15 @@ dependencies = [ "time 0.3.9", ] +[[package]] +name = "float-cmp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +dependencies = [ + "num-traits", +] + [[package]] name = "fnv" version = "1.0.7" @@ -2056,11 +2235,10 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" dependencies = [ - "matches", "percent-encoding", ] @@ -2156,9 +2334,9 @@ checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" [[package]] name = "futures" -version = "0.3.21" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f73fe65f54d1e12b726f517d3e2135ca3125a437b6d998caf1962961f7172d9e" +checksum = "7f21eda599937fba36daeb58a22e8f5cee2d14c4a17b5b7739c7c8e5e3b8230c" dependencies = [ "futures-channel", "futures-core", @@ -2171,9 +2349,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.21" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3083ce4b914124575708913bca19bfe887522d6e2e6d0952943f5eac4a74010" +checksum = "30bdd20c28fadd505d0fd6712cdfcb0d4b5648baf45faef7f852afb2399bb050" dependencies = [ "futures-core", "futures-sink", @@ -2181,15 +2359,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.21" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3" +checksum = "4e5aa3de05362c3fb88de6531e6296e85cde7739cccad4b9dfeeb7f6ebce56bf" [[package]] name = "futures-executor" -version = "0.3.21" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9420b90cfa29e327d0429f19be13e7ddb68fa1cccb09d65e5706b8c7a749b8a6" +checksum = "9ff63c23854bee61b6e9cd331d523909f238fc7636290b96826e9cfa5faa00ab" dependencies = [ "futures-core", "futures-task", @@ -2199,15 +2377,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.21" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b" +checksum = "bbf4d2a7a308fd4578637c0b17c7e1c7ba127b8f6ba00b29f717e9655d85eb68" [[package]] name = "futures-macro" -version = "0.3.21" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512" +checksum = "42cd15d1c7456c04dbdf7e88bcd69760d74f3a798d6444e16974b505b0e62f17" dependencies = [ "proc-macro2", "quote", @@ -2216,15 +2394,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.21" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21163e139fa306126e6eedaf49ecdb4588f939600f0b1e770f4205ee4b7fa868" +checksum = "21b20ba5a92e727ba30e72834706623d94ac93a725410b6a6b6fbc1b07f7ba56" [[package]] name = "futures-task" -version = "0.3.21" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a" +checksum = "a6508c467c73851293f390476d4491cf4d227dbabcd4170f3bb6044959b294f1" [[package]] name = "futures-timer" @@ -2234,9 +2412,9 @@ checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" [[package]] name = "futures-util" -version = "0.3.21" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a" +checksum = "44fb6cb1be61cc1d2e43b262516aafcf63b241cffdb1d3fa115f91d9c7b09c90" dependencies = [ "futures 0.1.31", "futures-channel", @@ -2271,9 +2449,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd48d33ec7f05fbfa152300fdad764757cbded343c1aa1cff2fbaf4134851803" +checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" dependencies = [ "typenum", "version_check", @@ -2292,25 +2470,25 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" +checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" dependencies = [ "cfg-if 1.0.0", "libc", - "wasi 0.10.2+wasi-snapshot-preview1", + "wasi 0.11.0+wasi-snapshot-preview1", ] [[package]] name = "gftp" -version = "0.2.5" +version = "0.3.0" dependencies = [ "actix-rt", "anyhow", "digest 0.8.1", "dotenv", "env_logger 0.7.1", - "futures 0.3.21", + "futures 0.3.24", "log", "rand 0.8.5", "serde", @@ -2319,7 +2497,7 @@ dependencies = [ "structopt", "tempdir", "thiserror", - "tokio 1.19.1", + "tokio 1.21.2", "url", "ya-compile-time-utils", "ya-core-model", @@ -2328,9 +2506,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4" +checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" [[package]] name = "git-version" @@ -2362,9 +2540,9 @@ checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" [[package]] name = "globset" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10463d9ff00a2a068db14231982f5132edebad0d7660cd956a1c30292dbcbfbd" +checksum = "0a1e17342619edbc21a964c2afbeb6c820c6a2560032872f397bb97ea127bd0a" dependencies = [ "aho-corasick", "bstr", @@ -2375,7 +2553,7 @@ dependencies = [ [[package]] name = "golemsp" -version = "0.2.0" +version = "0.3.0" dependencies = [ "actix-rt", "ansi_term", @@ -2387,22 +2565,22 @@ dependencies = [ "directories", "dotenv", "env_logger 0.7.1", - "futures 0.3.21", + "futures 0.3.24", "lazy_static", "libc", "log", "names", "nix 0.22.3", - "prettytable-rs", + "prettytable-rs 0.8.0", "promptly", "rustyline 6.3.0", "serde", "serde_json", "strip-ansi-escapes", "structopt", - "strum 0.20.0", - "strum_macros 0.20.1", - "tokio 1.19.1", + "strum 0.24.1", + "strum_macros 0.24.3", + "tokio 1.21.2", "url", "ya-client", "ya-compile-time-utils", @@ -2418,8 +2596,8 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06c5d2f987ee8f6dff3fa1a352058dc59b990e447e4c7846aa7d804971314f7b" dependencies = [ - "dashmap", - "futures 0.3.21", + "dashmap 4.0.2", + "futures 0.3.24", "futures-timer", "no-std-compat", "nonzero_ext", @@ -2431,8 +2609,8 @@ dependencies = [ [[package]] name = "graphene-sgx" -version = "0.4.0" -source = "git+https://github.com/golemfactory/graphene-rust.git?rev=3619f63a0c1ff44962ab604e55cd5dfb62177c56#3619f63a0c1ff44962ab604e55cd5dfb62177c56" +version = "0.3.3" +source = "git+https://github.com/golemfactory/graphene-rust.git?rev=dbd993ebad7f9190410ea390a589348479af6407#dbd993ebad7f9190410ea390a589348479af6407" dependencies = [ "anyhow", "base64 0.11.0", @@ -2441,7 +2619,7 @@ dependencies = [ "field-offset", "hex", "http", - "hyper 0.14.19", + "hyper 0.14.20", "hyper-tls 0.5.0", "openssl", "serde", @@ -2471,11 +2649,11 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57" +checksum = "5ca32592cf21ac7ccab1825cd87f6c9b3d9022c44d086172ed0966bec8af30be" dependencies = [ - "bytes 1.1.0", + "bytes 1.2.1", "fnv", "futures-core", "futures-sink", @@ -2483,8 +2661,8 @@ dependencies = [ "http", "indexmap", "slab", - "tokio 1.19.1", - "tokio-util 0.7.3", + "tokio 1.21.2", + "tokio-util 0.7.4", "tracing", ] @@ -2500,9 +2678,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.11.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hdrhistogram" @@ -2516,18 +2694,18 @@ dependencies = [ [[package]] name = "headers" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cff78e5788be1e0ab65b04d306b2ed5092c815ec97ec70f4ebd5aee158aa55d" +checksum = "f3e372db8e5c0d213e0cd0b9be18be2aca3d44cf2fe30a9d46a65581cd454584" dependencies = [ "base64 0.13.0", "bitflags", - "bytes 1.1.0", + "bytes 1.2.1", "headers-core", "http", "httpdate 1.0.2", "mime", - "sha-1 0.10.0", + "sha1", ] [[package]] @@ -2602,13 +2780,13 @@ dependencies = [ [[package]] name = "http" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff8670570af52249509a86f5e3e18a08c60b177071826898fde8997cf5f6bfbb" +checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" dependencies = [ - "bytes 1.1.0", + "bytes 1.2.1", "fnv", - "itoa 1.0.2", + "itoa 1.0.4", ] [[package]] @@ -2627,7 +2805,7 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ - "bytes 1.1.0", + "bytes 1.2.1", "http", "pin-project-lite 0.2.9", ] @@ -2640,9 +2818,9 @@ checksum = "21dec9db110f5f872ed9699c3ecf50cf16f423502706ba5c72462e28d3157573" [[package]] name = "httparse" -version = "1.7.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "496ce29bb5a52785b44e0f7ca2847ae0bb839c9bd28f69acac9b99d461c0c04c" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" [[package]] name = "httpdate" @@ -2697,7 +2875,7 @@ dependencies = [ "httparse", "httpdate 0.3.2", "itoa 0.4.8", - "pin-project 1.0.10", + "pin-project 1.0.12", "socket2 0.3.19", "tokio 0.2.25", "tower-service", @@ -2707,23 +2885,23 @@ dependencies = [ [[package]] name = "hyper" -version = "0.14.19" +version = "0.14.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42dc3c131584288d375f2d07f822b0cb012d8c6fb899a5b9fdb3cb7eb9b6004f" +checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac" dependencies = [ - "bytes 1.1.0", + "bytes 1.2.1", "futures-channel", "futures-core", "futures-util", - "h2 0.3.13", + "h2 0.3.14", "http", "http-body 0.4.5", "httparse", "httpdate 1.0.2", - "itoa 1.0.2", + "itoa 1.0.4", "pin-project-lite 0.2.9", - "socket2 0.4.4", - "tokio 1.19.1", + "socket2 0.4.7", + "tokio 1.21.2", "tower-service", "tracing", "want", @@ -2736,7 +2914,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f93ec5be69758dfc06b9b29efa9d6e9306e387c85eb362c603912eead2ad98c7" dependencies = [ "bytes 0.5.6", - "futures 0.3.21", + "futures 0.3.24", "http", "hyper 0.13.10", "hyper-tls 0.4.3", @@ -2766,13 +2944,37 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ - "bytes 1.1.0", - "hyper 0.14.19", + "bytes 1.2.1", + "hyper 0.14.20", "native-tls", - "tokio 1.19.1", + "tokio 1.21.2", "tokio-native-tls", ] +[[package]] +name = "iana-time-zone" +version = "0.1.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5a6ef98976b22b3b7f2f3a806f858cb862044cfa66805aa3ad84cb3d3b785ed" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "winapi 0.3.9", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fde6edd6cef363e9359ed3c98ba64590ba9eecba2293eb5a723ab32aee8926aa" +dependencies = [ + "cxx", + "cxx-build", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -2790,6 +2992,16 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "im" version = "15.1.0" @@ -2797,7 +3009,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0acd33ff0285af998aaf9b57342af478078f53492322fafc47450e09397e0e9" dependencies = [ "bitmaps", - "rand_core 0.6.3", + "rand_core 0.6.4", "rand_xoshiro", "sized-chunks", "typenum", @@ -2853,12 +3065,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.8.2" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6012d540c5baa3589337a98ce73408de9b5a25ec9fc2c6fd6be8f0d39e0ca5a" +checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" dependencies = [ "autocfg 1.1.0", - "hashbrown 0.11.2", + "hashbrown 0.12.3", ] [[package]] @@ -2867,7 +3079,7 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4" dependencies = [ - "console 0.15.0", + "console 0.15.2", "lazy_static", "number_prefix", "regex", @@ -2902,6 +3114,12 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "io-lifetimes" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ea37f355c05dde75b84bba2d767906ad522e97cd9e2eef2be7a4ab7fb442c06" + [[package]] name = "iovec" version = "0.1.4" @@ -2913,14 +3131,14 @@ dependencies = [ [[package]] name = "ipconfig" -version = "0.2.2" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7e2f18aece9709094573a9f24f483c4f65caa4298e2f7ae1b71cc65d853fad7" +checksum = "723519edce41262b05d4143ceb95050e4c614f483e78e9fd9e39a8275a84ad98" dependencies = [ - "socket2 0.3.19", + "socket2 0.4.7", "widestring", "winapi 0.3.9", - "winreg 0.6.2", + "winreg 0.7.0", ] [[package]] @@ -2940,9 +3158,9 @@ dependencies = [ [[package]] name = "itertools" -version = "0.10.3" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" dependencies = [ "either", ] @@ -2955,24 +3173,24 @@ checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" [[package]] name = "itoa" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" +checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" [[package]] name = "jobserver" -version = "0.1.24" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa" +checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.57" +version = "0.3.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "671a26f820db17c2a2750743f1dd03bafd15b98c9f30c7c2628c024c05d73397" +checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" dependencies = [ "wasm-bindgen", ] @@ -2996,7 +3214,7 @@ version = "16.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a47c4c3ac843f9a4238943f97620619033dadef4b378cd1e8addd170de396b3" dependencies = [ - "futures 0.3.21", + "futures 0.3.24", "log", "serde", "serde_derive", @@ -3009,7 +3227,7 @@ version = "17.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4467ab6dfa369b69e52bd0692e480c4d117410538526a57a304a0f2250fd95e" dependencies = [ - "futures 0.3.21", + "futures 0.3.24", "futures-executor", "futures-util", "log", @@ -3054,9 +3272,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.126" +version = "0.2.135" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" +checksum = "68783febc7782c6c5cb401fbda4de5a9898be1762314da0bb2c10ced61f18b0c" [[package]] name = "libloading" @@ -3070,9 +3288,9 @@ dependencies = [ [[package]] name = "libm" -version = "0.2.2" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33a33a362ce288760ec6a508b94caaec573ae7d3bbbd91b87aa0bad4456839db" +checksum = "292a948cd991e376cf75541fe5b97a1081d713c618b4f1b9500f8844e49eb565" [[package]] name = "libproc" @@ -3080,7 +3298,7 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba412294cf67e64a10b5d9f72844f109894de3fe288cd519eb429514c9ec63ad" dependencies = [ - "errno", + "errno 0.1.8", "libc", ] @@ -3096,10 +3314,25 @@ dependencies = [ ] [[package]] -name = "linked-hash-map" -version = "0.5.4" +name = "link-cplusplus" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" +checksum = "9272ab7b96c9046fbc5bc56c06c117cb639fe2d509df0c421cad82d2915cf369" +dependencies = [ + "cc", +] + +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + +[[package]] +name = "linux-raw-sys" +version = "0.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4d2456c373231a208ad294c33dc5bff30051eafd954cd4caae83a712b12854d" [[package]] name = "local-channel" @@ -3130,9 +3363,9 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" dependencies = [ "autocfg 1.1.0", "scopeguard", @@ -3170,9 +3403,9 @@ dependencies = [ [[package]] name = "lzma-sys" -version = "0.1.17" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdb4b7c3eddad11d3af9e86c487607d2d2442d185d848575365c4856ba96d619" +checksum = "e06754c4acf47d49c727d5665ca9fb828851cda315ed3bd51edd148ef78a8772" dependencies = [ "cc", "libc", @@ -3234,6 +3467,15 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" +[[package]] +name = "md-5" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" +dependencies = [ + "digest 0.10.5", +] + [[package]] name = "memchr" version = "2.5.0" @@ -3289,6 +3531,16 @@ dependencies = [ "t1ha", ] +[[package]] +name = "metrics" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142c53885123b68d94108295a09d4afe1a1388ed95b54d5dacd9a454753030f2" +dependencies = [ + "ahash 0.7.6", + "metrics-macros 0.5.1", +] + [[package]] name = "metrics-core" version = "0.5.2" @@ -3344,6 +3596,17 @@ dependencies = [ "syn", ] +[[package]] +name = "metrics-macros" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49e30813093f757be5cf21e50389a24dc7dbb22c49f23b7e8f51d69b508a5ffa" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "metrics-observer-json" version = "0.1.1" @@ -3376,7 +3639,7 @@ dependencies = [ "hdrhistogram", "metrics-core", "metrics-util", - "serde_yaml", + "serde_yaml 0.8.26", ] [[package]] @@ -3450,9 +3713,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f5c75688da582b8ffc1f1799e9db273f32133c49e048f614d22ec3256773ccc" +checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34" dependencies = [ "adler", ] @@ -3478,9 +3741,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "713d550d9b44d89174e066b7a6217ae06234c10cb47819a88290d2b353c31799" +checksum = "57ee1c23c7c63b0c9250c339ffdc69255f110b298b901b9f6c82547b7b87caaf" dependencies = [ "libc", "log", @@ -3588,6 +3851,15 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "nibble_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" +dependencies = [ + "smallvec", +] + [[package]] name = "nix" version = "0.18.0" @@ -3625,6 +3897,31 @@ dependencies = [ "memoffset 0.6.5", ] +[[package]] +name = "nix" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f866317acbd3a240710c63f065ffb1e4fd466259045ccb504130b7f668f35c6" +dependencies = [ + "bitflags", + "cc", + "cfg-if 1.0.0", + "libc", + "memoffset 0.6.5", +] + +[[package]] +name = "nix" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e322c04a9e3440c327fca7b6c8a63e6890a32fa2ad689db972425f07e0d22abb" +dependencies = [ + "autocfg 1.1.0", + "bitflags", + "cfg-if 1.0.0", + "libc", +] + [[package]] name = "no-std-compat" version = "0.4.1" @@ -3652,6 +3949,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44a1290799eababa63ea60af0cbc3f03363e328e58f32fb0294798ed3e85f444" +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + [[package]] name = "notify" version = "4.0.17" @@ -3841,9 +4144,9 @@ checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" [[package]] name = "object" -version = "0.28.4" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e42c982f2d955fac81dd7e1d0e1426a7d702acd9c98d19ab01083a6a0328c424" +checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53" dependencies = [ "memchr", ] @@ -3860,9 +4163,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.12.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225" +checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1" [[package]] name = "opaque-debug" @@ -3878,9 +4181,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.40" +version = "0.10.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb81a6430ac911acb25fe5ac8f1d2af1b4ea8a4fdfda0f1ee4292af2e2d8eb0e" +checksum = "12fc0523e3bd51a692c8850d075d74dc062ccf251c0110668cbd921917118a13" dependencies = [ "bitflags", "cfg-if 1.0.0", @@ -3910,18 +4213,18 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "111.20.0+1.1.1o" +version = "111.22.0+1.1.1q" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92892c4f87d56e376e469ace79f1128fdaded07646ddf73aa0be4706ff712dec" +checksum = "8f31f0d509d1c1ae9cada2f9539ff8f37933831fd5098879e482aa687d659853" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.74" +version = "0.9.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "835363342df5fba8354c5b453325b110ffd54044e588c539cf2f20a8014e4cb1" +checksum = "5230151e44c0f05157effb743e8d517472843121cf9243e8b81393edb5acd9ce" dependencies = [ "autocfg 1.1.0", "cc", @@ -4022,7 +4325,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" dependencies = [ "instant", - "lock_api 0.4.7", + "lock_api 0.4.9", "parking_lot_core 0.8.5", ] @@ -4032,7 +4335,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ - "lock_api 0.4.7", + "lock_api 0.4.9", "parking_lot_core 0.9.3", ] @@ -4059,7 +4362,7 @@ dependencies = [ "cfg-if 1.0.0", "instant", "libc", - "redox_syscall 0.2.13", + "redox_syscall 0.2.16", "smallvec", "winapi 0.3.9", ] @@ -4072,16 +4375,16 @@ checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall 0.2.13", + "redox_syscall 0.2.16", "smallvec", "windows-sys", ] [[package]] name = "paste" -version = "1.0.7" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc" +checksum = "b1de2e551fb905ac83f73f7aedf2f0cb4a0da7e35efa24a202a936269f1f18e1" [[package]] name = "path-clean" @@ -4106,16 +4409,17 @@ dependencies = [ [[package]] name = "percent-encoding" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "pest" -version = "2.1.3" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" +checksum = "dbc7bc69c062e492337d74d59b120c274fd3d261b6bf6d3207d499b4b379c41a" dependencies = [ + "thiserror", "ucd-trie", ] @@ -4135,44 +4439,33 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143" dependencies = [ - "fixedbitset 0.4.1", + "fixedbitset 0.4.2", "indexmap", ] -[[package]] -name = "petname" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9474d20c003131d8945af478b25c8e29477a522e408ae4ca8e8822839f5fd2f2" -dependencies = [ - "itertools 0.10.3", - "rand 0.8.5", - "structopt", -] - [[package]] name = "pin-project" -version = "0.4.29" +version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9615c18d31137579e9ff063499264ddc1278e7b1982757ebc111028c4d1dc909" +checksum = "3ef0f924a5ee7ea9cbcea77529dba45f8a9ba9f622419fe3386ca581a3ae9d5a" dependencies = [ - "pin-project-internal 0.4.29", + "pin-project-internal 0.4.30", ] [[package]] name = "pin-project" -version = "1.0.10" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58ad3879ad3baf4e44784bc6a718a8698867bb991f8ce24d1bcbe2cfb4c3a75e" +checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" dependencies = [ - "pin-project-internal 1.0.10", + "pin-project-internal 1.0.12", ] [[package]] name = "pin-project-internal" -version = "0.4.29" +version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "044964427019eed9d49d9d5bbce6047ef18f37100ea400912a9fa4a3523ab12a" +checksum = "851c8d0ce9bebe43790dedfc86614c23494ac9f423dd618d3a61fc693eafe61e" dependencies = [ "proc-macro2", "quote", @@ -4181,9 +4474,9 @@ dependencies = [ [[package]] name = "pin-project-internal" -version = "1.0.10" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "744b6f092ba29c3650faf274db506afd39944f48420f6c86b17cfe0ee1cb36bb" +checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" dependencies = [ "proc-macro2", "quote", @@ -4239,6 +4532,36 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" +[[package]] +name = "predicates" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5aab5be6e4732b473071984b3164dbbfb7a3674d30ea5ff44410b6bcd960c3c" +dependencies = [ + "difflib", + "float-cmp", + "itertools 0.10.5", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da1c2388b1513e1b605fcec39a95e0a9e8ef088f71443ef37099fa9ae6673fcb" + +[[package]] +name = "predicates-tree" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d86de6de25020a36c6d3643a86d9a6a9f552107c0559c60ea03551b5e16c032" +dependencies = [ + "predicates-core", + "termtree", +] + [[package]] name = "prettytable-rs" version = "0.8.0" @@ -4247,9 +4570,23 @@ checksum = "0fd04b170004fa2daccf418a7f8253aaf033c27760b5f225889024cf66d7ac2e" dependencies = [ "atty", "csv", - "encode_unicode", + "encode_unicode 0.3.6", + "lazy_static", + "term 0.5.2", + "unicode-width", +] + +[[package]] +name = "prettytable-rs" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f375cb74c23b51d23937ffdeb48b1fbf5b6409d4b9979c1418c1de58bc8f801" +dependencies = [ + "atty", + "csv", + "encode_unicode 1.0.0", "lazy_static", - "term", + "term 0.7.0", "unicode-width", ] @@ -4263,7 +4600,7 @@ dependencies = [ "impl-codec 0.4.2", "impl-rlp", "impl-serde", - "uint 0.9.3", + "uint 0.9.4", ] [[package]] @@ -4276,15 +4613,16 @@ dependencies = [ "impl-codec 0.5.1", "impl-rlp", "impl-serde", - "uint 0.9.3", + "uint 0.9.4", ] [[package]] name = "proc-macro-crate" -version = "1.1.3" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17d47ce914bf4de440332250b0edd23ce48c005f59fab39d3335866b114f11a" +checksum = "eda0fc3b0fb7c975631757e14d9049da17374063edb6ebbcbc54d880d4fe94e9" dependencies = [ + "once_cell", "thiserror", "toml", ] @@ -4321,20 +4659,20 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" [[package]] name = "proc-macro2" -version = "1.0.39" +version = "1.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f" +checksum = "94e2ef8dbfc347b10c094890f778ee2e36ca9bb4262e86dc99cd217e35f3470b" dependencies = [ "unicode-ident", ] [[package]] name = "promptly" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b99cfb0289110d969dd21637cfbc922584329bc9e5037c5e576325c615658509" +checksum = "9acbc6c5a5b029fe58342f58445acb00ccfe24624e538894bc2f04ce112980ba" dependencies = [ - "rustyline 6.3.0", + "rustyline 9.1.2", ] [[package]] @@ -4343,7 +4681,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e6984d2f1a23009bd270b8bb56d0926810a3d483f59c987d77969e9d8e840b2" dependencies = [ - "bytes 1.1.0", + "bytes 1.2.1", "prost-derive 0.7.0", ] @@ -4353,7 +4691,7 @@ version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71adf41db68aa0daaefc69bb30bcd68ded9b9abaad5d1fbb6304c4fb390e083e" dependencies = [ - "bytes 1.1.0", + "bytes 1.2.1", "prost-derive 0.10.1", ] @@ -4363,7 +4701,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32d3ebd75ac2679c2af3a92246639f9fcc8a442ee420719cc4fe195b98dd5fa3" dependencies = [ - "bytes 1.1.0", + "bytes 1.2.1", "heck 0.3.3", "itertools 0.9.0", "log", @@ -4381,11 +4719,11 @@ version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ae5a4388762d5815a9fc0dea33c56b021cdc8dde0c55e0c9ca57197254b0cab" dependencies = [ - "bytes 1.1.0", + "bytes 1.2.1", "cfg-if 1.0.0", "cmake", "heck 0.4.0", - "itertools 0.10.3", + "itertools 0.10.5", "lazy_static", "log", "multimap", @@ -4417,7 +4755,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b670f45da57fb8542ebdbb6105a925fe571b67f9e7ed9f47a06a84e72b4e7cc" dependencies = [ "anyhow", - "itertools 0.10.3", + "itertools 0.10.5", "proc-macro2", "quote", "syn", @@ -4429,7 +4767,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b518d7cdd93dab1d1122cf07fa9a60771836c668dde9d9e2a139f957f0d9f1bb" dependencies = [ - "bytes 1.1.0", + "bytes 1.2.1", "prost 0.7.0", ] @@ -4439,7 +4777,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d0a014229361011dc8e69c8a1ec6c2e8d0f2af7c91e3ea3f5b2170298461e68" dependencies = [ - "bytes 1.1.0", + "bytes 1.2.1", "prost 0.10.4", ] @@ -4482,21 +4820,21 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.18" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1" +checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" dependencies = [ "proc-macro2", ] [[package]] name = "r2d2" -version = "0.8.9" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f" +checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93" dependencies = [ "log", - "parking_lot 0.11.2", + "parking_lot 0.12.1", "scheduled-thread-pool", ] @@ -4512,6 +4850,16 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "643f8f41a8ebc4c5dc4515c82bb8abd397b527fc20fd681b7c011c2aee5d44fb" +[[package]] +name = "radix_trie" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" +dependencies = [ + "endian-type", + "nibble_vec", +] + [[package]] name = "rand" version = "0.3.23" @@ -4575,7 +4923,7 @@ checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", "rand_chacha 0.3.1", - "rand_core 0.6.3", + "rand_core 0.6.4", ] [[package]] @@ -4605,7 +4953,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.3", + "rand_core 0.6.4", ] [[package]] @@ -4634,11 +4982,11 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.6", + "getrandom 0.2.7", ] [[package]] @@ -4718,7 +5066,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" dependencies = [ - "rand_core 0.6.3", + "rand_core 0.6.4", ] [[package]] @@ -4728,7 +5076,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d" dependencies = [ "autocfg 1.1.0", - "crossbeam-deque 0.8.1", + "crossbeam-deque 0.8.2", "either", "rayon-core", ] @@ -4739,9 +5087,9 @@ version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f" dependencies = [ - "crossbeam-channel 0.5.4", - "crossbeam-deque 0.8.1", - "crossbeam-utils 0.8.8", + "crossbeam-channel 0.5.6", + "crossbeam-deque 0.8.2", + "crossbeam-utils 0.8.12", "num_cpus", ] @@ -4773,9 +5121,9 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] name = "redox_syscall" -version = "0.2.13" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ "bitflags", ] @@ -4797,16 +5145,16 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ - "getrandom 0.2.6", - "redox_syscall 0.2.13", + "getrandom 0.2.7", + "redox_syscall 0.2.16", "thiserror", ] [[package]] name = "regex" -version = "1.5.6" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d83f127d94bdbcda4c8cc2e50f6f84f4b611f69c902699ca385a39c3a75f9ff1" +checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" dependencies = [ "aho-corasick", "memchr", @@ -4824,9 +5172,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.26" +version = "0.6.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64" +checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" [[package]] name = "remove_dir_all" @@ -4884,34 +5232,35 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.10" +version = "0.11.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46a1f7aa4f35e5e8b4160449f51afc758f0ce6454315a9fa7d0d113e958c41eb" +checksum = "431949c384f4e2ae07605ccaa56d1d9d2ecdb5cadd4f9577ccfab29f2e5149fc" dependencies = [ "base64 0.13.0", - "bytes 1.1.0", + "bytes 1.2.1", "encoding_rs", "futures-core", "futures-util", - "h2 0.3.13", + "h2 0.3.14", "http", "http-body 0.4.5", - "hyper 0.14.19", + "hyper 0.14.20", "hyper-tls 0.5.0", "ipnet", "js-sys", - "lazy_static", "log", "mime", "native-tls", + "once_cell", "percent-encoding", "pin-project-lite 0.2.9", "serde", "serde_json", "serde_urlencoded", - "tokio 1.19.1", + "tokio 1.21.2", "tokio-native-tls", - "trust-dns-resolver 0.20.4", + "tower-service", + "trust-dns-resolver", "url", "wasm-bindgen", "wasm-bindgen-futures", @@ -4961,7 +5310,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "999508abb0ae792aabed2460c45b89106d97fe4adac593bdaef433c2605847b5" dependencies = [ - "bytes 1.1.0", + "bytes 1.2.1", "rustc-hex", ] @@ -4985,7 +5334,7 @@ dependencies = [ "base64 0.13.0", "blake2b_simd", "constant_time_eq", - "crossbeam-utils 0.8.8", + "crossbeam-utils 0.8.12", ] [[package]] @@ -5030,14 +5379,28 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.9", + "semver 1.0.14", +] + +[[package]] +name = "rustix" +version = "0.35.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbb2fda4666def1433b1b05431ab402e42a1084285477222b72d6c564c417cef" +dependencies = [ + "bitflags", + "errno 0.2.8", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys", ] [[package]] name = "rustversion" -version = "1.0.6" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f" +checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8" [[package]] name = "rustyline" @@ -5079,11 +5442,35 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "rustyline" +version = "9.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db7826789c0e25614b03e5a54a0717a86f9ff6e6e5247f92b369472869320039" +dependencies = [ + "bitflags", + "cfg-if 1.0.0", + "clipboard-win", + "dirs-next 2.0.0", + "fd-lock", + "libc", + "log", + "memchr", + "nix 0.23.1", + "radix_trie", + "scopeguard", + "smallvec", + "unicode-segmentation", + "unicode-width", + "utf8parse", + "winapi 0.3.9", +] + [[package]] name = "ryu" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" +checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" [[package]] name = "salsa20" @@ -5128,6 +5515,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "scratch" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898" + [[package]] name = "scrypt" version = "0.5.0" @@ -5185,9 +5578,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.6.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dc14f172faf8a0194a3aded622712b0de276821addc574fa54fc0a1167e10dc" +checksum = "2bc1bb97804af6631813c55739f771071e0f2ed33ee20b68c86ec505d906356c" dependencies = [ "bitflags", "core-foundation", @@ -5212,12 +5605,12 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "502f26626811e30581f6cb80ee588ccc709f8e490d14a4363f4b16f5d24cd7e2" dependencies = [ - "hyper 0.14.19", + "hyper 0.14.20", "indicatif", "log", "quick-xml", "regex", - "reqwest 0.11.10", + "reqwest 0.11.12", "semver 0.11.0", "serde_json", "tempfile", @@ -5244,9 +5637,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.9" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd" +checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4" dependencies = [ "serde", ] @@ -5348,27 +5741,27 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.137" +version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1" +checksum = "728eb6351430bccb993660dfffc5a72f91ccc1295abaa8ce19b27ebe4f75568b" dependencies = [ "serde_derive", ] [[package]] name = "serde_bytes" -version = "0.11.6" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "212e73464ebcde48d723aa02eb270ba62eff38a9b732df31f33f1b4e145f3a54" +checksum = "cfc50e8183eeeb6178dcb167ae34a8051d63535023ae38b5d8d12beae193d37b" dependencies = [ "serde", ] [[package]] name = "serde_derive" -version = "1.0.137" +version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be" +checksum = "81fa1584d3d1bcacd84c277a0dfe21f5b0f6accf4a23d04d4c6d61f1af522b4c" dependencies = [ "proc-macro2", "quote", @@ -5377,11 +5770,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.81" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" +checksum = "41feea4228a6f1cd09ec7a3593a682276702cd67b5273544757dae23c096f074" dependencies = [ - "itoa 1.0.2", + "itoa 1.0.4", "ryu", "serde", ] @@ -5404,7 +5797,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa 1.0.2", + "itoa 1.0.4", "ryu", "serde", ] @@ -5433,9 +5826,9 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.8.24" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707d15895415db6628332b737c838b88c598522e4dc70647e59b72312924aebc" +checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" dependencies = [ "indexmap", "ryu", @@ -5443,14 +5836,41 @@ dependencies = [ "yaml-rust", ] +[[package]] +name = "serde_yaml" +version = "0.9.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8613d593412a0deb7bbd8de9d908efff5a0cb9ccd8f62c641e7b2ed2f57291d1" +dependencies = [ + "indexmap", + "itoa 1.0.4", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "serial_test" version = "0.5.1" source = "git+https://github.com/tworec/serial_test.git?branch=actix_rt_test#6a9176a54cab7e3682a05dbac3741812b221a8c5" dependencies = [ "lazy_static", - "parking_lot 0.10.2", - "serial_test_derive", + "parking_lot 0.11.2", + "serial_test_derive 0.5.1", +] + +[[package]] +name = "serial_test" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92761393ee4dc3ff8f4af487bd58f4307c9329bbedea02cac0089ad9c411e153" +dependencies = [ + "dashmap 5.4.0", + "futures 0.3.24", + "lazy_static", + "log", + "parking_lot 0.12.1", + "serial_test_derive 0.9.0", ] [[package]] @@ -5463,6 +5883,18 @@ dependencies = [ "syn", ] +[[package]] +name = "serial_test_derive" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b6f5d1c3087fb119617cff2966fe3808a80e5eb59a8c1601d5994d66f4346a5" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "sha-1" version = "0.9.8" @@ -5477,14 +5909,14 @@ dependencies = [ ] [[package]] -name = "sha-1" -version = "0.10.0" +name = "sha1" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" +checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" dependencies = [ "cfg-if 1.0.0", "cpufeatures", - "digest 0.10.3", + "digest 0.10.5", ] [[package]] @@ -5601,7 +6033,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" dependencies = [ "libc", - "mio 0.8.3", + "mio 0.8.4", "signal-hook", ] @@ -5626,15 +6058,18 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32" +checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +dependencies = [ + "autocfg 1.1.0", +] [[package]] name = "smallvec" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "smoltcp" @@ -5652,8 +6087,7 @@ dependencies = [ [[package]] name = "smoltcp" version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72165c4af59f5f19c7fb774b88b95660591b612380305b5f4503157341a9f7ee" +source = "git+https://github.com/golemfactory/smoltcp?rev=acb743b6990fd92303c24c43ff75711422a5cb94#acb743b6990fd92303c24c43ff75711422a5cb94" dependencies = [ "bitflags", "byteorder", @@ -5685,9 +6119,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.4.4" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0" +checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" dependencies = [ "libc", "winapi 0.3.9", @@ -5701,11 +6135,11 @@ checksum = "b5c71ed3d54db0a699f4948e1bb3e45b450fa31fe602621dee6680361d569c88" dependencies = [ "base64 0.12.3", "bytes 0.5.6", - "futures 0.3.21", + "futures 0.3.24", "httparse", "log", "rand 0.7.3", - "sha-1 0.9.8", + "sha-1", ] [[package]] @@ -5720,6 +6154,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "str-buf" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e08d8363704e6c71fc928674353e6b7c23dcea9d82d7012c8faf2a3a025f8d0" + [[package]] name = "strip-ansi-escapes" version = "0.1.1" @@ -5779,25 +6219,13 @@ checksum = "b89a286a7e3b5720b9a477b23253bc50debac207c8d21505f8e70b36792f11b5" [[package]] name = "strum" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7318c509b5ba57f18533982607f24070a55d353e90d4cae30c467cdb2ad5ac5c" - -[[package]] -name = "strum" -version = "0.22.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7ac893c7d471c8a21f31cfe213ec4f6d9afeed25537c772e08ef3f005f8729e" +checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" dependencies = [ - "strum_macros 0.22.0", + "strum_macros 0.24.3", ] -[[package]] -name = "strum" -version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96acfc1b70604b8b2f1ffa4c57e59176c7dbb05d556c71ecd2f5498a1dee7f8" - [[package]] name = "strum_macros" version = "0.19.4" @@ -5812,33 +6240,9 @@ dependencies = [ [[package]] name = "strum_macros" -version = "0.20.1" +version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee8bc6b87a5112aeeab1f4a9f7ab634fe6cbefc4850006df31267f4cfb9e3149" -dependencies = [ - "heck 0.3.3", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "strum_macros" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "339f799d8b549e3744c7ac7feb216383e4005d94bdb22561b3ab8f3b808ae9fb" -dependencies = [ - "heck 0.3.3", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "strum_macros" -version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6878079b17446e4d3eba6192bb0a2950d5b14f0ed8424b852310e5a94345d0ef" +checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" dependencies = [ "heck 0.4.0", "proc-macro2", @@ -5855,9 +6259,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.96" +version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0748dd251e24453cb8717f0354206b91557e4ec8703673a4b30208f2abaf1ebf" +checksum = "3fcd952facd492f9be3ef0d0b7032a6e442ee9b361d4acc2b1d0c4aaa5f613a1" dependencies = [ "proc-macro2", "quote", @@ -5892,6 +6296,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tar" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b55807c0344e1e6c04d7c965f5289c39a8d94ae23ed5c0b57aabac549f871c6" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "tempdir" version = "0.3.7" @@ -5911,7 +6326,7 @@ dependencies = [ "cfg-if 1.0.0", "fastrand", "libc", - "redox_syscall 0.2.13", + "redox_syscall 0.2.16", "remove_dir_all", "winapi 0.3.9", ] @@ -5928,8 +6343,19 @@ dependencies = [ ] [[package]] -name = "termcolor" -version = "1.1.3" +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next 2.0.0", + "rustversion", + "winapi 0.3.9", +] + +[[package]] +name = "termcolor" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" dependencies = [ @@ -5955,6 +6381,34 @@ dependencies = [ "libc", ] +[[package]] +name = "termtree" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507e9898683b6c43a9aa55b64259b721b52ba226e0f3779137e50ad114a4c90b" + +[[package]] +name = "test-case" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21d6cf5a7dffb3f9dceec8e6b8ca528d9bd71d36c9f074defb548ce161f598c0" +dependencies = [ + "test-case-macros", +] + +[[package]] +name = "test-case-macros" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e45b7bf6e19353ddd832745c8fcf77a17a93171df7151187f26623f2b75b5b26" +dependencies = [ + "cfg-if 1.0.0", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "textwrap" version = "0.11.0" @@ -5966,18 +6420,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.31" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a" +checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.31" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a" +checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" dependencies = [ "proc-macro2", "quote", @@ -5995,11 +6449,12 @@ dependencies = [ [[package]] name = "time" -version = "0.1.43" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" dependencies = [ "libc", + "wasi 0.10.0+wasi-snapshot-preview1", "winapi 0.3.9", ] @@ -6009,7 +6464,7 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2702e08a7a860f005826c6815dcac101b19b5eb330c27fe4a5928fec1d20ddd" dependencies = [ - "itoa 1.0.2", + "itoa 1.0.4", "libc", "num_threads", "time-macros", @@ -6080,20 +6535,20 @@ dependencies = [ [[package]] name = "tokio" -version = "1.19.1" +version = "1.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95eec79ea28c00a365f539f1961e9278fbcaf81c0ff6aaf0e93c181352446948" +checksum = "a9e03c497dc955702ba729190dc4aac6f2a0ce97f913e5b1b5912fc5039d9099" dependencies = [ - "bytes 1.1.0", + "autocfg 1.1.0", + "bytes 1.2.1", "libc", "memchr", - "mio 0.8.3", + "mio 0.8.4", "num_cpus", - "once_cell", "parking_lot 0.12.1", "pin-project-lite 0.2.9", "signal-hook-registry", - "socket2 0.4.4", + "socket2 0.4.7", "tokio-macros 1.8.0", "winapi 0.3.9", ] @@ -6105,7 +6560,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5cf347e8ae1d1ffd16c8aed569172a71bd81098a001d0f4964d476c0097aba4a" dependencies = [ "byteorder", - "tokio 1.19.1", + "tokio 1.21.2", ] [[package]] @@ -6118,7 +6573,7 @@ dependencies = [ "once_cell", "pin-project-lite 0.2.9", "tokio 0.2.25", - "tokio 1.19.1", + "tokio 1.21.2", "tokio-stream", ] @@ -6151,7 +6606,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" dependencies = [ "native-tls", - "tokio 1.19.1", + "tokio 1.21.2", ] [[package]] @@ -6163,28 +6618,28 @@ dependencies = [ "futures-util", "openssl", "openssl-sys", - "tokio 1.19.1", + "tokio 1.21.2", ] [[package]] name = "tokio-process-ns" -version = "0.1.0" +version = "0.2.0" dependencies = [ "libc", "nix 0.22.3", - "tokio 1.19.1", + "tokio 1.21.2", ] [[package]] name = "tokio-stream" -version = "0.1.9" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df54d54117d6fdc4e4fea40fe1e4e566b3505700e148a6827e59b34b0d2600d9" +checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce" dependencies = [ "futures-core", "pin-project-lite 0.2.9", - "tokio 1.19.1", - "tokio-util 0.7.3", + "tokio 1.21.2", + "tokio-util 0.7.4", ] [[package]] @@ -6196,8 +6651,8 @@ dependencies = [ "filetime", "futures-core", "libc", - "redox_syscall 0.2.13", - "tokio 1.19.1", + "redox_syscall 0.2.16", + "tokio 1.21.2", "tokio-stream", "xattr", ] @@ -6232,26 +6687,26 @@ version = "0.6.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" dependencies = [ - "bytes 1.1.0", + "bytes 1.2.1", "futures-core", "futures-io", "futures-sink", "log", "pin-project-lite 0.2.9", - "tokio 1.19.1", + "tokio 1.21.2", ] [[package]] name = "tokio-util" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc463cd8deddc3770d20f9852143d50bf6094e640b485cb2e189a2099085ff45" +checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" dependencies = [ - "bytes 1.1.0", + "bytes 1.2.1", "futures-core", "futures-sink", "pin-project-lite 0.2.9", - "tokio 1.19.1", + "tokio 1.21.2", "tracing", ] @@ -6266,15 +6721,15 @@ dependencies = [ [[package]] name = "tower-service" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.34" +version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if 1.0.0", "log", @@ -6285,9 +6740,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.21" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ "proc-macro2", "quote", @@ -6296,11 +6751,11 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.26" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" dependencies = [ - "lazy_static", + "once_cell", "valuable", ] @@ -6310,7 +6765,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" dependencies = [ - "pin-project 1.0.10", + "pin-project 1.0.12", "tracing", ] @@ -6359,31 +6814,9 @@ dependencies = [ [[package]] name = "trust-dns-proto" -version = "0.19.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cad71a0c0d68ab9941d2fb6e82f8fb2e86d9945b94e1661dd0aaea2b88215a9" -dependencies = [ - "async-trait", - "backtrace", - "cfg-if 1.0.0", - "enum-as-inner", - "futures 0.3.21", - "idna", - "lazy_static", - "log", - "rand 0.7.3", - "smallvec", - "socket2 0.3.19", - "thiserror", - "tokio 0.2.25", - "url", -] - -[[package]] -name = "trust-dns-proto" -version = "0.20.4" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca94d4e9feb6a181c690c4040d7a24ef34018d8313ac5044a61d21222ae24e31" +checksum = "9c31f240f59877c3d4bb3b3ea0ec5a6a0cff07323580ff8c7a605cd7d08b255d" dependencies = [ "async-trait", "cfg-if 1.0.0", @@ -6392,7 +6825,7 @@ dependencies = [ "futures-channel", "futures-io", "futures-util", - "idna", + "idna 0.2.3", "ipnet", "lazy_static", "log", @@ -6400,34 +6833,15 @@ dependencies = [ "smallvec", "thiserror", "tinyvec", - "tokio 1.19.1", + "tokio 1.21.2", "url", ] [[package]] name = "trust-dns-resolver" -version = "0.19.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "710f593b371175db53a26d0b38ed2978fafb9e9e8d3868b1acd753ea18df0ceb" -dependencies = [ - "cfg-if 0.1.10", - "futures 0.3.21", - "ipconfig", - "lazy_static", - "log", - "lru-cache", - "resolv-conf", - "smallvec", - "thiserror", - "tokio 0.2.25", - "trust-dns-proto 0.19.7", -] - -[[package]] -name = "trust-dns-resolver" -version = "0.20.4" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecae383baad9995efaa34ce8e57d12c3f305e545887472a492b838f4b5cfb77a" +checksum = "e4ba72c2ea84515690c9fcef4c6c660bb9df3036ed1051686de84605b74fd558" dependencies = [ "cfg-if 1.0.0", "futures-util", @@ -6435,12 +6849,13 @@ dependencies = [ "lazy_static", "log", "lru-cache", - "parking_lot 0.11.2", + "parking_lot 0.12.1", "resolv-conf", "smallvec", "thiserror", - "tokio 1.19.1", - "trust-dns-proto 0.20.4", + "tokio 1.21.2", + "tokio-openssl", + "trust-dns-proto", ] [[package]] @@ -6476,9 +6891,9 @@ checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" [[package]] name = "ucd-trie" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c" +checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81" [[package]] name = "uint" @@ -6493,9 +6908,9 @@ dependencies = [ [[package]] name = "uint" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12f03af7ccf01dd611cc450a0d10dbc9b745770d096473e2faf0ca6e2d66d1e0" +checksum = "a45526d29728d135c2900b0d30573fe3ee79fceb12ef534c7bb30e810a91b601" dependencies = [ "byteorder", "crunchy", @@ -6529,30 +6944,30 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" [[package]] name = "unicode-ident" -version = "1.0.0" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee" +checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" [[package]] name = "unicode-normalization" -version = "0.1.19" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" dependencies = [ "tinyvec", ] [[package]] name = "unicode-segmentation" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99" +checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a" [[package]] name = "unicode-width" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" [[package]] name = "unicode_categories" @@ -6560,15 +6975,20 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" +[[package]] +name = "unsafe-libyaml" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1e5fa573d8ac5f1a856f8d7be41d390ee973daf97c806b2c1a465e4e1406e68" + [[package]] name = "url" -version = "2.2.2" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" dependencies = [ "form_urlencoded", - "idna", - "matches", + "idna 0.3.0", "percent-encoding", "serde", ] @@ -6591,7 +7011,7 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" dependencies = [ - "getrandom 0.2.6", + "getrandom 0.2.7", "serde", ] @@ -6621,7 +7041,7 @@ checksum = "e7141e445af09c8919f1d5f8a20dae0b20c3b57a45dee0d5823c6ed5d237f15a" dependencies = [ "bitflags", "chrono", - "rustc_version 0.2.3", + "rustc_version 0.4.0", ] [[package]] @@ -6661,6 +7081,15 @@ dependencies = [ "quote", ] +[[package]] +name = "wait-timeout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +dependencies = [ + "libc", +] + [[package]] name = "walkdir" version = "2.3.2" @@ -6690,9 +7119,9 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" [[package]] name = "wasi" -version = "0.10.2+wasi-snapshot-preview1" +version = "0.10.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" [[package]] name = "wasi" @@ -6702,9 +7131,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.80" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad" +checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" dependencies = [ "cfg-if 1.0.0", "serde", @@ -6714,13 +7143,13 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.80" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4" +checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" dependencies = [ "bumpalo", - "lazy_static", "log", + "once_cell", "proc-macro2", "quote", "syn", @@ -6729,9 +7158,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.30" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f741de44b75e14c35df886aff5f1eb73aa114fa5d4d00dcd37b5e01259bf3b2" +checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -6741,9 +7170,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.80" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5" +checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -6751,9 +7180,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.80" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b" +checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" dependencies = [ "proc-macro2", "quote", @@ -6764,15 +7193,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.80" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744" +checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" [[package]] name = "web-sys" -version = "0.3.57" +version = "0.3.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b17e741662c70c8bd24ac5c5b18de314a2c26c32bf8346ee1e6f53de919c283" +checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" dependencies = [ "js-sys", "wasm-bindgen", @@ -6789,7 +7218,7 @@ dependencies = [ "derive_more", "ethabi", "ethereum-types 0.11.0", - "futures 0.3.21", + "futures 0.3.24", "futures-timer", "hex", "hyper 0.13.10", @@ -6799,7 +7228,7 @@ dependencies = [ "log", "native-tls", "parking_lot 0.11.2", - "pin-project 1.0.10", + "pin-project 1.0.12", "rlp", "secp256k1 0.20.3", "serde", @@ -6820,26 +7249,26 @@ checksum = "bc4c18ae15621f764fab919f7e4a83d87163494cbc3460884debef7c6bc1bc6b" dependencies = [ "arrayvec 0.5.2", "base64 0.13.0", - "bytes 1.1.0", + "bytes 1.2.1", "derive_more", "ethabi", "ethereum-types 0.11.0", - "futures 0.3.21", + "futures 0.3.24", "futures-timer", "headers", "hex", "jsonrpc-core 17.1.0", "log", "parking_lot 0.11.2", - "pin-project 1.0.10", - "reqwest 0.11.10", + "pin-project 1.0.12", + "reqwest 0.11.12", "rlp", "secp256k1 0.20.3", "serde", "serde_json", "soketto", "tiny-keccak 2.0.2", - "tokio 1.19.1", + "tokio 1.21.2", "tokio-util 0.6.10", "url", "web3-async-native-tls", @@ -6853,26 +7282,26 @@ checksum = "1f6d8d1636b2627fe63518d5a9b38a569405d9c9bc665c43c9c341de57227ebb" dependencies = [ "native-tls", "thiserror", - "tokio 1.19.1", + "tokio 1.21.2", "url", ] [[package]] name = "which" -version = "4.2.5" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c4fb54e6113b6a8772ee41c3404fb0301ac79604489467e0a9ce1f3e97c24ae" +checksum = "1c831fbbee9e129a8cf93e7747a82da9d95ba8e16621cae60ec2cdc849bacb7b" dependencies = [ "either", - "lazy_static", "libc", + "once_cell", ] [[package]] name = "widestring" -version = "0.4.3" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c168940144dd21fd8046987c16a46a33d5fc84eec29ef9dcddc2ac9e31526b7c" +checksum = "17882f045410753661207383517a6f62ec3dbeb6a4ed2acce01f0728238d1983" [[package]] name = "winapi" @@ -6960,15 +7389,6 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" -[[package]] -name = "winreg" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9" -dependencies = [ - "winapi 0.3.9", -] - [[package]] name = "winreg" version = "0.7.0" @@ -7014,16 +7434,16 @@ dependencies = [ [[package]] name = "xz2" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c179869f34fc7c01830d3ce7ea2086bc3a07e0d35289b667d0a8bf910258926c" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" dependencies = [ "lzma-sys", ] [[package]] name = "ya-activity" -version = "0.3.0" +version = "0.4.0" dependencies = [ "actix-http", "actix-rt", @@ -7033,7 +7453,7 @@ dependencies = [ "diesel", "diesel_migrations", "env_logger 0.7.1", - "futures 0.3.21", + "futures 0.3.24", "hex", "lazy_static", "libsqlite3-sys", @@ -7046,10 +7466,10 @@ dependencies = [ "shlex 0.1.1", "structopt", "thiserror", - "tokio 1.19.1", + "tokio 1.21.2", "tokio-stream", "uuid", - "ya-client-model 0.4.0", + "ya-client-model 0.5.0", "ya-core-model", "ya-net", "ya-persistence", @@ -7068,33 +7488,7 @@ dependencies = [ "chrono", "serde", "serde_json", - "serde_yaml", - "thiserror", - "ya-client-model 0.4.0", -] - -[[package]] -name = "ya-agreement-utils" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "022ffb9c20a34581d5acb059af96323ebde68bf7e476dd8fe433365e3ff631d5" -dependencies = [ - "serde", - "serde_json", - "serde_yaml", - "thiserror", - "ya-client-model 0.3.2", -] - -[[package]] -name = "ya-agreement-utils" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a59f53bc89329d33bc1d143027535ae36fd0a981e6c5bfb2363516ff32ddf8e" -dependencies = [ - "serde", - "serde_json", - "serde_yaml", + "serde_yaml 0.8.26", "thiserror", "ya-client-model 0.4.0", ] @@ -7107,11 +7501,11 @@ dependencies = [ "regex", "serde", "serde_json", - "serde_yaml", + "serde_yaml 0.8.26", "shlex 1.1.0", "tempdir", "thiserror", - "ya-client-model 0.4.0", + "ya-client-model 0.5.0", ] [[package]] @@ -7127,7 +7521,7 @@ dependencies = [ "log-derive", "serde", "serde_json", - "serde_yaml", + "serde_yaml 0.8.26", "ya-agreement-utils 0.2.0", "ya-client-model 0.4.0", "ya-negotiator-component", @@ -7135,15 +7529,16 @@ dependencies = [ [[package]] name = "ya-client" -version = "0.6.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=e1d4e843bba2d7b9e48299c9ec976ce307aa78e1#e1d4e843bba2d7b9e48299c9ec976ce307aa78e1" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cceaf1408473b3846288bf4325d527d9f77762c0d17a55f8a92802e4629da8e" dependencies = [ "actix-codec", "awc", - "bytes 1.1.0", + "bytes 1.2.1", "chrono", "envy", - "futures 0.3.21", + "futures 0.3.24", "heck 0.3.3", "hex", "log", @@ -7155,14 +7550,14 @@ dependencies = [ "structopt", "thiserror", "url", - "ya-client-model 0.4.0", + "ya-client-model 0.5.0", ] [[package]] name = "ya-client-model" -version = "0.3.2" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1466c69ef7b38b5c5a27cd915cad19f2e94f4adb29154e67bfdae8577946d067" +checksum = "ad140f78c6350dd741dd446ccb54c4ea1d521f4000edaa6699003ad8a2486f2a" dependencies = [ "bigdecimal 0.2.2", "chrono", @@ -7177,8 +7572,9 @@ dependencies = [ [[package]] name = "ya-client-model" -version = "0.4.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=e1d4e843bba2d7b9e48299c9ec976ce307aa78e1#e1d4e843bba2d7b9e48299c9ec976ce307aa78e1" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70a4a2f46bc12b1c747495c7911a844e7dc2c879826993426d568b9b594c26a1" dependencies = [ "bigdecimal 0.2.2", "chrono", @@ -7207,7 +7603,7 @@ dependencies = [ [[package]] name = "ya-core-model" -version = "0.6.0" +version = "0.8.0" dependencies = [ "bigdecimal 0.2.2", "bitflags", @@ -7218,10 +7614,10 @@ dependencies = [ "serde", "serde_bytes", "structopt", - "strum 0.20.0", - "strum_macros 0.20.1", + "strum 0.24.1", + "strum_macros 0.24.3", "thiserror", - "ya-client-model 0.4.0", + "ya-client-model 0.5.0", "ya-service-bus", ] @@ -7238,18 +7634,18 @@ dependencies = [ [[package]] name = "ya-dummy-driver" -version = "0.2.0" +version = "0.3.0" dependencies = [ "anyhow", "bigdecimal 0.2.2", "chrono", - "futures 0.3.21", + "futures 0.3.24", "log", "maplit", "serde_json", - "tokio 1.19.1", + "tokio 1.21.2", "uuid", - "ya-client-model 0.4.0", + "ya-client-model 0.5.0", "ya-core-model", "ya-payment-driver", "ya-persistence", @@ -7259,7 +7655,7 @@ dependencies = [ [[package]] name = "ya-erc20-driver" -version = "0.3.0" +version = "0.4.0" dependencies = [ "actix-rt", "anyhow", @@ -7273,7 +7669,7 @@ dependencies = [ "ethabi", "ethereum-tx-sign", "ethereum-types 0.11.0", - "futures 0.3.21", + "futures 0.3.24", "hex", "lazy_static", "log", @@ -7287,10 +7683,10 @@ dependencies = [ "structopt", "thiserror", "tiny-keccak 2.0.2", - "tokio 1.19.1", + "tokio 1.21.2", "uuid", "web3 0.16.0", - "ya-client-model 0.4.0", + "ya-client-model 0.5.0", "ya-payment-driver", "ya-service-api-interfaces", "ya-utils-futures", @@ -7299,7 +7695,7 @@ dependencies = [ [[package]] name = "ya-exe-unit" -version = "0.2.0" +version = "0.3.0" dependencies = [ "actix", "actix-files", @@ -7307,14 +7703,14 @@ dependencies = [ "actix-web", "anyhow", "async-trait", - "bytes 1.1.0", + "bytes 1.2.1", "chrono", "derivative", "derive_more", "dotenv", "env_logger 0.7.1", - "flexi_logger 0.22.5", - "futures 0.3.21", + "flexi_logger 0.22.6", + "futures 0.3.24", "graphene-sgx", "hex", "ipnet", @@ -7325,26 +7721,26 @@ dependencies = [ "openssl", "rand 0.6.5", "regex", - "reqwest 0.11.10", + "reqwest 0.11.12", "rustyline 7.1.0", "secp256k1 0.19.0", "serde", "serde_json", - "serde_yaml", + "serde_yaml 0.8.26", "sha3 0.8.2", "shell-words", "signal-hook", - "socket2 0.4.4", + "socket2 0.4.7", "structopt", "tempdir", "thiserror", - "tokio 1.19.1", + "tokio 1.21.2", "tokio-stream", - "tokio-util 0.7.3", + "tokio-util 0.7.4", "url", "winapi 0.3.9", "ya-agreement-utils 0.4.0", - "ya-client-model 0.4.0", + "ya-client-model 0.5.0", "ya-compile-time-utils", "ya-core-model", "ya-manifest-utils", @@ -7370,7 +7766,7 @@ dependencies = [ [[package]] name = "ya-identity" -version = "0.2.1" +version = "0.3.0" dependencies = [ "actix-rt", "actix-service", @@ -7380,12 +7776,13 @@ dependencies = [ "awc", "base64 0.12.3", "chrono", + "ctrlc", "diesel", "diesel_migrations", "dotenv", "env_logger 0.7.1", "ethsign", - "futures 0.3.21", + "futures 0.3.24", "log", "promptly", "r2d2", @@ -7396,9 +7793,9 @@ dependencies = [ "sha2 0.9.9", "structopt", "thiserror", - "tokio 1.19.1", + "tokio 1.21.2", "uuid", - "ya-client-model 0.4.0", + "ya-client-model 0.5.0", "ya-core-model", "ya-persistence", "ya-sb-router", @@ -7406,38 +7803,56 @@ dependencies = [ "ya-service-api-derive", "ya-service-api-interfaces", "ya-service-bus", + "yansi", ] [[package]] -name = "ya-manifest-utils" +name = "ya-manifest-test-utils" version = "0.1.0" +dependencies = [ + "base64 0.13.0", + "openssl", + "tar", + "ya-agreement-utils 0.4.0", + "ya-manifest-utils", +] + +[[package]] +name = "ya-manifest-utils" +version = "0.2.0" dependencies = [ "anyhow", "base64 0.13.0", "chrono", - "ethsign", "hex", - "petname", - "semver 1.0.9", + "log", + "md-5", + "openssl", + "regex", + "semver 1.0.14", "serde", "serde_json", "serde_with", - "serde_yaml", + "serde_yaml 0.9.13", + "serial_test 0.9.0", "sha2 0.9.9", "shlex 1.1.0", "snailquote", "structopt", - "strum 0.22.0", - "strum_macros 0.22.0", - "tempdir", + "strum 0.24.1", + "tar", + "tempfile", + "test-case", "thiserror", "url", "ya-agreement-utils 0.4.0", + "ya-manifest-test-utils", + "ya-utils-path", ] [[package]] name = "ya-market" -version = "0.3.0" +version = "0.4.0" dependencies = [ "actix", "actix-http", @@ -7454,7 +7869,7 @@ dependencies = [ "diesel_migrations", "digest 0.8.1", "env_logger 0.7.1", - "futures 0.3.21", + "futures 0.3.24", "humantime 2.1.0", "lazy_static", "libsqlite3-sys", @@ -7467,15 +7882,15 @@ dependencies = [ "regex", "serde", "serde_json", - "serial_test", + "serial_test 0.5.1", "sha3 0.8.2", "structopt", - "strum 0.24.0", - "strum_macros 0.24.0", + "strum 0.24.1", + "strum_macros 0.24.3", "thiserror", - "tokio 1.19.1", + "tokio 1.21.2", "uuid", - "ya-agreement-utils 0.3.0", + "ya-agreement-utils 0.4.0", "ya-client", "ya-core-model", "ya-diesel-utils", @@ -7504,18 +7919,18 @@ dependencies = [ "semver 0.11.0", "serde_json", "thiserror", - "ya-agreement-utils 0.2.1", + "ya-agreement-utils 0.4.0", ] [[package]] name = "ya-metrics" -version = "0.1.0" +version = "0.2.0" dependencies = [ "actix-web", "anyhow", "awc", "bigdecimal 0.2.2", - "futures 0.3.21", + "futures 0.3.24", "lazy_static", "log", "metrics 0.16.0", @@ -7523,7 +7938,7 @@ dependencies = [ "metrics-runtime", "percent-encoding", "structopt", - "tokio 1.19.1", + "tokio 1.21.2", "url", "ya-core-model", "ya-service-api", @@ -7543,7 +7958,7 @@ dependencies = [ "log-derive", "serde", "serde_json", - "serde_yaml", + "serde_yaml 0.8.26", "ya-agreement-utils 0.2.0", "ya-client-model 0.4.0", ] @@ -7558,7 +7973,7 @@ dependencies = [ "lazy_static", "serde", "serde_json", - "serde_yaml", + "serde_yaml 0.8.26", "thiserror", "ya-agreement-utils 0.2.0", "ya-client-model 0.4.0", @@ -7574,14 +7989,14 @@ dependencies = [ "actix_derive", "anyhow", "derive_more", - "futures 0.3.21", + "futures 0.3.24", "humantime-serde", "log", "serde", "serde_json", - "serde_yaml", + "serde_yaml 0.8.26", "thiserror", - "tokio 1.19.1", + "tokio 1.21.2", "tokio-stream", "ya-agreement-utils 0.2.0", "ya-builtin-negotiators", @@ -7592,14 +8007,16 @@ dependencies = [ [[package]] name = "ya-net" -version = "0.2.0" +version = "0.3.0" dependencies = [ "actix", + "actix-web", "anyhow", - "bytes 1.1.0", + "bytes 1.2.1", + "chrono", "env_logger 0.7.1", "ethsign", - "futures 0.3.21", + "futures 0.3.24", "humantime 2.1.0", "lazy_static", "log", @@ -7609,16 +8026,19 @@ dependencies = [ "serde", "serde_json", "structopt", - "strum 0.22.0", + "strum 0.24.1", + "test-case", "thiserror", - "tokio 1.19.1", + "tokio 1.21.2", "tokio-stream", - "tokio-util 0.7.3", + "tokio-util 0.7.4", "url", + "ya-client-model 0.5.0", "ya-core-model", "ya-relay-client", "ya-sb-proto", "ya-sb-router", + "ya-sb-util", "ya-service-api", "ya-service-api-interfaces", "ya-service-bus", @@ -7627,7 +8047,7 @@ dependencies = [ [[package]] name = "ya-payment" -version = "0.2.0" +version = "0.3.0" dependencies = [ "actix-rt", "actix-web", @@ -7640,7 +8060,7 @@ dependencies = [ "dotenv", "env_logger 0.7.1", "ethsign", - "futures 0.3.21", + "futures 0.3.24", "hex", "humantime 2.1.0", "lazy_static", @@ -7654,12 +8074,12 @@ dependencies = [ "serde_json", "structopt", "thiserror", - "tokio 1.19.1", + "tokio 1.21.2", "uint 0.7.1", "uuid", - "ya-agreement-utils 0.3.0", + "ya-agreement-utils 0.4.0", "ya-client", - "ya-client-model 0.4.0", + "ya-client-model 0.5.0", "ya-core-model", "ya-dummy-driver", "ya-erc20-driver", @@ -7676,7 +8096,7 @@ dependencies = [ [[package]] name = "ya-payment-driver" -version = "0.2.0" +version = "0.3.0" dependencies = [ "actix", "anyhow", @@ -7687,7 +8107,7 @@ dependencies = [ "diesel_migrations", "ethereum-types 0.11.0", "ethsign", - "futures 0.3.21", + "futures 0.3.24", "hex", "log", "num-bigint 0.3.3", @@ -7696,8 +8116,8 @@ dependencies = [ "r2d2", "sha3 0.9.1", "thiserror", - "tokio 1.19.1", - "ya-client-model 0.4.0", + "tokio 1.21.2", + "ya-client-model 0.5.0", "ya-core-model", "ya-persistence", "ya-service-bus", @@ -7705,7 +8125,7 @@ dependencies = [ [[package]] name = "ya-persistence" -version = "0.2.0" +version = "0.3.0" dependencies = [ "anyhow", "bigdecimal 0.2.2", @@ -7718,9 +8138,10 @@ dependencies = [ "serde_json", "structopt", "tempdir", + "test-case", "thiserror", - "tokio 1.19.1", - "ya-client-model 0.4.0", + "tokio 1.21.2", + "ya-client-model 0.5.0", "ya-core-model", "ya-service-api", "ya-service-api-interfaces", @@ -7729,13 +8150,15 @@ dependencies = [ [[package]] name = "ya-provider" -version = "0.2.0" +version = "0.3.0" dependencies = [ "actix", "actix-rt", "actix_derive", "anyhow", + "assert_cmd", "backoff", + "base64 0.13.0", "bigdecimal 0.2.2", "bytesize", "chrono", @@ -7743,7 +8166,7 @@ dependencies = [ "dialoguer", "directories", "dotenv", - "futures 0.3.21", + "futures 0.3.24", "futures-util", "hex", "humantime 2.1.0", @@ -7757,34 +8180,39 @@ dependencies = [ "num-traits", "num_cpus", "path-clean", + "predicates", "regex", "semver 0.11.0", "serde", "serde_json", - "serde_yaml", + "serde_yaml 0.8.26", + "serial_test 0.9.0", "shared_child", "shlex 1.1.0", "signal-hook", "structopt", - "strum 0.20.0", - "strum_macros 0.20.1", + "strum 0.24.1", + "strum_macros 0.24.3", "sys-info", + "test-case", "thiserror", - "tokio 1.19.1", + "tokio 1.21.2", "tokio-stream", "url", "walkdir", "winapi 0.3.9", "ya-agreement-utils 0.2.0", "ya-client", - "ya-client-model 0.4.0", + "ya-client-model 0.5.0", "ya-compile-time-utils", "ya-core-model", "ya-file-logging", + "ya-manifest-test-utils", "ya-manifest-utils", "ya-negotiators", "ya-std-utils", "ya-utils-actix", + "ya-utils-cli", "ya-utils-path", "ya-utils-process", "yansi", @@ -7792,17 +8220,18 @@ dependencies = [ [[package]] name = "ya-relay-client" -version = "0.3.0" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=0e6863c24767a246531d038455921f12c9e75e94#0e6863c24767a246531d038455921f12c9e75e94" +version = "0.4.0" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=ec174a250fc014fba084088a90f9b1c95d613ed3#ec174a250fc014fba084088a90f9b1c95d613ed3" dependencies = [ "anyhow", "chrono", "derive_more", "env_logger 0.8.4", - "futures 0.3.21", + "futures 0.3.24", + "humantime 2.1.0", "log", "thiserror", - "tokio 1.19.1", + "tokio 1.21.2", "tokio-stream", "url", "ya-relay-core", @@ -7812,56 +8241,60 @@ dependencies = [ [[package]] name = "ya-relay-core" -version = "0.3.0" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=0e6863c24767a246531d038455921f12c9e75e94#0e6863c24767a246531d038455921f12c9e75e94" +version = "0.3.1" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=ec174a250fc014fba084088a90f9b1c95d613ed3#ec174a250fc014fba084088a90f9b1c95d613ed3" dependencies = [ "anyhow", "chrono", + "derive_more", "digest 0.9.0", "ethsign", - "futures 0.3.21", + "futures 0.3.24", "governor", "hex", "lazy_static", "log", + "metrics 0.19.0", "rand 0.8.5", "serde_json", "sha2 0.9.9", "sha3 0.9.1", "thiserror", - "tokio 1.19.1", - "tokio-util 0.7.3", + "tokio 1.21.2", + "tokio-util 0.7.4", "url", "uuid", - "ya-client-model 0.4.0", + "ya-client-model 0.5.0", "ya-relay-proto", "ya-relay-stack", ] [[package]] name = "ya-relay-proto" -version = "0.3.0" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=0e6863c24767a246531d038455921f12c9e75e94#0e6863c24767a246531d038455921f12c9e75e94" +version = "0.4.1" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=ec174a250fc014fba084088a90f9b1c95d613ed3#ec174a250fc014fba084088a90f9b1c95d613ed3" dependencies = [ "anyhow", - "bytes 1.1.0", + "bytes 1.2.1", "derive_more", - "futures 0.3.21", + "futures 0.3.24", "prost 0.10.4", "prost-build 0.10.4", "rand 0.8.5", "thiserror", - "tokio 1.19.1", - "tokio-util 0.7.3", + "tokio 1.21.2", + "tokio-util 0.7.4", + "ya-relay-util", ] [[package]] name = "ya-relay-stack" -version = "0.4.0" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=0e6863c24767a246531d038455921f12c9e75e94#0e6863c24767a246531d038455921f12c9e75e94" +version = "0.4.1" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=ec174a250fc014fba084088a90f9b1c95d613ed3#ec174a250fc014fba084088a90f9b1c95d613ed3" dependencies = [ "derive_more", - "futures 0.3.21", + "futures 0.3.24", + "lazy_static", "log", "managed 0.8.0", "num-derive", @@ -7869,45 +8302,55 @@ dependencies = [ "rand 0.8.5", "smoltcp 0.8.1", "thiserror", - "tokio 1.19.1", + "tokio 1.21.2", "tokio-stream", + "ya-relay-util", +] + +[[package]] +name = "ya-relay-util" +version = "0.1.0" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=ec174a250fc014fba084088a90f9b1c95d613ed3#ec174a250fc014fba084088a90f9b1c95d613ed3" +dependencies = [ + "bytes 1.2.1", + "derive_more", ] [[package]] name = "ya-runtime-api" -version = "0.4.1" +version = "0.5.0" dependencies = [ "anyhow", - "bytes 1.1.0", + "bytes 1.2.1", "env_logger 0.7.1", - "futures 0.3.21", + "futures 0.3.24", "log", "prost 0.10.4", "prost-build 0.10.4", "serde", "serde_json", - "tokio 1.19.1", - "tokio-util 0.7.3", + "tokio 1.21.2", + "tokio-util 0.7.4", ] [[package]] name = "ya-sb-proto" -version = "0.4.0" -source = "git+https://github.com/golemfactory/ya-service-bus.git?rev=a60f7476c39a18fd3a369b3045b878544ae25598#a60f7476c39a18fd3a369b3045b878544ae25598" +version = "0.6.1" +source = "git+https://github.com/golemfactory/ya-service-bus.git?rev=4a932a62d48c72dabf59020e85c523d3ff0be7d4#4a932a62d48c72dabf59020e85c523d3ff0be7d4" dependencies = [ - "bytes 1.1.0", + "bytes 1.2.1", "prost 0.10.4", "prost-build 0.7.0", "thiserror", - "tokio 1.19.1", - "tokio-util 0.7.3", + "tokio 1.21.2", + "tokio-util 0.7.4", "url", ] [[package]] name = "ya-sb-router" -version = "0.4.5" -source = "git+https://github.com/golemfactory/ya-service-bus.git?rev=a60f7476c39a18fd3a369b3045b878544ae25598#a60f7476c39a18fd3a369b3045b878544ae25598" +version = "0.6.1" +source = "git+https://github.com/golemfactory/ya-service-bus.git?rev=4a932a62d48c72dabf59020e85c523d3ff0be7d4#4a932a62d48c72dabf59020e85c523d3ff0be7d4" dependencies = [ "actix", "actix-rt", @@ -7916,16 +8359,16 @@ dependencies = [ "anyhow", "bitflags", "chrono", - "futures 0.3.21", + "futures 0.3.24", "lazy_static", "log", "parking_lot 0.11.2", - "pin-project 1.0.10", + "pin-project 1.0.12", "prost 0.10.4", "structopt", - "tokio 1.19.1", + "tokio 1.21.2", "tokio-stream", - "tokio-util 0.7.3", + "tokio-util 0.7.4", "url", "uuid", "ya-sb-proto", @@ -7934,13 +8377,13 @@ dependencies = [ [[package]] name = "ya-sb-util" -version = "0.2.0" -source = "git+https://github.com/golemfactory/ya-service-bus.git?rev=a60f7476c39a18fd3a369b3045b878544ae25598#a60f7476c39a18fd3a369b3045b878544ae25598" +version = "0.4.1" +source = "git+https://github.com/golemfactory/ya-service-bus.git?rev=4a932a62d48c72dabf59020e85c523d3ff0be7d4#4a932a62d48c72dabf59020e85c523d3ff0be7d4" dependencies = [ "actix", "bitflags", - "futures 0.3.21", - "pin-project 0.4.29", + "futures 0.3.24", + "pin-project 0.4.30", ] [[package]] @@ -7949,25 +8392,22 @@ version = "0.1.1" dependencies = [ "anyhow", "lazy_static", - "log", - "prettytable-rs", "serde", - "serde_json", - "serde_yaml", "url", + "ya-utils-cli", ] [[package]] name = "ya-service-api-cache" version = "0.1.0" dependencies = [ - "futures 0.3.21", + "futures 0.3.24", "log", ] [[package]] name = "ya-service-api-derive" -version = "0.1.0" +version = "0.2.0" dependencies = [ "actix-rt", "actix-service", @@ -7979,24 +8419,24 @@ dependencies = [ "proc-macro2", "quote", "structopt", - "strum 0.19.5", - "strum_macros 0.19.4", + "strum 0.24.1", + "strum_macros 0.24.3", "syn", "ya-service-api-interfaces", ] [[package]] name = "ya-service-api-interfaces" -version = "0.1.0" +version = "0.2.0" dependencies = [ "actix-web", "anyhow", - "futures 0.3.21", + "futures 0.3.24", ] [[package]] name = "ya-service-api-web" -version = "0.1.1" +version = "0.2.0" dependencies = [ "actix-rt", "actix-service", @@ -8005,7 +8445,7 @@ dependencies = [ "anyhow", "awc", "env_logger 0.7.1", - "futures 0.3.21", + "futures 0.3.24", "log", "serde", "structopt", @@ -8024,20 +8464,21 @@ dependencies = [ [[package]] name = "ya-service-bus" -version = "0.4.10" -source = "git+https://github.com/golemfactory/ya-service-bus.git?rev=a60f7476c39a18fd3a369b3045b878544ae25598#a60f7476c39a18fd3a369b3045b878544ae25598" +version = "0.6.1" +source = "git+https://github.com/golemfactory/ya-service-bus.git?rev=4a932a62d48c72dabf59020e85c523d3ff0be7d4#4a932a62d48c72dabf59020e85c523d3ff0be7d4" dependencies = [ "actix", "flexbuffers", - "futures 0.3.21", + "futures 0.3.24", "lazy_static", "log", + "miniz_oxide", "rand 0.8.5", "semver 0.11.0", "serde", "thiserror", - "tokio 1.19.1", - "tokio-util 0.7.3", + "tokio 1.21.2", + "tokio-util 0.7.4", "url", "uuid", "ya-sb-proto", @@ -8046,11 +8487,11 @@ dependencies = [ [[package]] name = "ya-sgx" -version = "0.1.0" +version = "0.2.0" dependencies = [ "anyhow", "graphene-sgx", - "ya-client-model 0.4.0", + "ya-client-model 0.5.0", "ya-core-model", "ya-service-bus", ] @@ -8065,7 +8506,7 @@ dependencies = [ [[package]] name = "ya-transfer" -version = "0.1.0" +version = "0.3.0" dependencies = [ "actix-http", "actix-rt", @@ -8073,12 +8514,12 @@ dependencies = [ "anyhow", "async-compression", "awc", - "bytes 1.1.0", + "bytes 1.2.1", "env_logger 0.7.1", - "futures 0.3.21", + "futures 0.3.24", "gftp", "globset", - "h2 0.3.13", + "h2 0.3.14", "hex", "lazy_static", "log", @@ -8091,12 +8532,12 @@ dependencies = [ "structopt", "tempdir", "thiserror", - "tokio 1.19.1", + "tokio 1.21.2", "tokio-tar", - "tokio-util 0.7.3", + "tokio-util 0.7.4", "url", "walkdir", - "ya-client-model 0.4.0", + "ya-client-model 0.5.0", "ya-core-model", "ya-service-bus", "ya-utils-path", @@ -8105,38 +8546,49 @@ dependencies = [ [[package]] name = "ya-utils-actix" -version = "0.1.1" +version = "0.2.0" dependencies = [ "actix", "actix-rt", "anyhow", "chrono", - "futures 0.3.21", + "futures 0.3.24", "log", - "tokio 1.19.1", + "tokio 1.21.2", ] [[package]] -name = "ya-utils-futures" +name = "ya-utils-cli" version = "0.1.0" dependencies = [ - "futures 0.3.21", - "tokio 1.19.1", + "anyhow", + "lazy_static", + "prettytable-rs 0.9.0", + "serde", + "serde_json", + "serde_yaml 0.9.13", +] + +[[package]] +name = "ya-utils-futures" +version = "0.2.0" +dependencies = [ + "futures 0.3.24", + "tokio 1.21.2", ] [[package]] name = "ya-utils-networking" -version = "0.1.1" +version = "0.2.0" dependencies = [ "anyhow", - "futures 0.3.21", + "futures 0.3.24", "ipnet", "lazy_static", "log", "regex", "thiserror", - "tokio-compat-02", - "trust-dns-resolver 0.19.7", + "trust-dns-resolver", "url", "ya-relay-stack", ] @@ -8152,18 +8604,18 @@ dependencies = [ [[package]] name = "ya-utils-process" -version = "0.1.0" +version = "0.2.0" dependencies = [ "actix", "anyhow", "derive_more", "fs2", - "futures 0.3.21", + "futures 0.3.24", "futures-util", "libc", "nix 0.22.3", "shared_child", - "tokio 1.19.1", + "tokio 1.21.2", ] [[package]] @@ -8175,7 +8627,7 @@ dependencies = [ [[package]] name = "ya-version" -version = "0.1.0" +version = "0.2.0" dependencies = [ "actix-web", "anyhow", @@ -8189,7 +8641,7 @@ dependencies = [ "serde_json", "structopt", "thiserror", - "tokio 1.19.1", + "tokio 1.21.2", "ya-client", "ya-compile-time-utils", "ya-core-model", @@ -8201,16 +8653,16 @@ dependencies = [ [[package]] name = "ya-vpn" -version = "0.1.0" +version = "0.2.0" dependencies = [ "actix", "actix-rt", "actix-web", "actix-web-actors", "anyhow", - "bytes 1.1.0", + "bytes 1.2.1", "env_logger 0.7.1", - "futures 0.3.21", + "futures 0.3.24", "hex", "ipnet", "lazy_static", @@ -8224,11 +8676,11 @@ dependencies = [ "smoltcp 0.7.5", "structopt", "thiserror", - "tokio 1.19.1", + "tokio 1.21.2", "url", "uuid", "ya-client", - "ya-client-model 0.4.0", + "ya-client-model 0.5.0", "ya-core-model", "ya-net", "ya-persistence", @@ -8241,7 +8693,7 @@ dependencies = [ [[package]] name = "ya-zksync-driver" -version = "0.2.0" +version = "0.3.0" dependencies = [ "actix-rt", "anyhow", @@ -8252,7 +8704,7 @@ dependencies = [ "dotenv", "env_logger 0.7.1", "ethereum-types 0.10.0", - "futures 0.3.21", + "futures 0.3.24", "hex", "lazy_static", "log", @@ -8264,10 +8716,10 @@ dependencies = [ "serde_json", "structopt", "tiny-keccak 1.5.0", - "tokio 1.19.1", + "tokio 1.21.2", "tokio-compat-02", "uuid", - "ya-client-model 0.4.0", + "ya-client-model 0.5.0", "ya-payment-driver", "ya-service-api-interfaces", "ya-utils-futures", @@ -8278,7 +8730,7 @@ dependencies = [ [[package]] name = "yagna" -version = "0.10.0" +version = "0.11.0" dependencies = [ "actix-rt", "actix-service", @@ -8287,7 +8739,7 @@ dependencies = [ "chrono", "directories", "dotenv", - "futures 0.3.21", + "futures 0.3.24", "gftp", "lazy_static", "log", @@ -8296,13 +8748,13 @@ dependencies = [ "serde", "serde_json", "structopt", - "tokio 1.19.1", + "tokio 1.21.2", "tokio-stream", - "tokio-util 0.7.3", + "tokio-util 0.7.4", "url", "ya-activity", "ya-client", - "ya-client-model 0.4.0", + "ya-client-model 0.5.0", "ya-compile-time-utils", "ya-core-model", "ya-dummy-driver", @@ -8349,9 +8801,9 @@ checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" [[package]] name = "zeroize" -version = "1.5.5" +version = "1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94693807d016b2f2d2e14420eb3bfcca689311ff775dcf113d74ea624b7cdf07" +checksum = "c394b5bd0c6f669e7275d9c20aa90ae064cb22e75a1cad54e1b34088034b149f" [[package]] name = "zip" @@ -8364,8 +8816,8 @@ dependencies = [ "crc32fast", "flate2", "thiserror", - "time 0.1.43", - "tokio 1.19.1", + "time 0.1.44", + "tokio 1.21.2", "tokio-byteorder", ] @@ -8520,7 +8972,7 @@ source = "git+https://github.com/matter-labs/zksync?rev=0e28e238f71b3be128e4a760 dependencies = [ "anyhow", "bigdecimal 0.2.2", - "futures 0.3.21", + "futures 0.3.24", "hex", "num", "serde", @@ -8530,18 +8982,18 @@ dependencies = [ [[package]] name = "zstd" -version = "0.10.2+zstd.1.5.2" +version = "0.11.2+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4a6bd64f22b5e3e94b4e238669ff9f10815c27a5180108b849d24174a83847" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "4.1.6+zstd.1.5.2" +version = "5.0.2+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94b61c51bb270702d6167b8ce67340d2754b088d0c091b06e593aa772c3ee9bb" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" dependencies = [ "libc", "zstd-sys", @@ -8549,9 +9001,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "1.6.3+zstd.1.5.2" +version = "2.0.1+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc49afa5c8d634e75761feda8c592051e7eeb4683ba827211eb0d731d3402ea8" +checksum = "9fd07cbbc53846d9145dbffdf6dd09a7a0aa52be46741825f5c97bdd4f73f12b" dependencies = [ "cc", "libc", diff --git a/Cargo.toml b/Cargo.toml index 6d94a04cee..9fcc222940 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "yagna" -version = "0.10.0" +version = "0.11.0" description = "Open platform and marketplace for distributed computations" readme = "README.md" authors = ["Golem Factory "] @@ -24,38 +24,38 @@ name = "yagna" path = "core/serv/src/main.rs" [dependencies] -ya-activity = "0.3" +ya-activity = "0.4" ya-compile-time-utils = "0.2" -ya-core-model = { version = "^0.6" } -ya-dummy-driver = { version = "0.2", optional = true } +ya-core-model = { version = "^0.8" } +ya-dummy-driver = { version = "0.3", optional = true } ya-file-logging = "0.1" -ya-erc20-driver = { version = "0.3", optional = true } -ya-zksync-driver = { version = "0.2", optional = true } -ya-identity = "0.2" -ya-market = "0.3" -ya-metrics = "0.1" -ya-net = { version = "0.2", features = ["service"] } -ya-payment = "0.2" -ya-persistence = { version = "0.2", features = ["service"] } -ya-sb-proto = "0.4" -ya-sb-router = "0.4" +ya-erc20-driver = { version = "0.4", optional = true } +ya-zksync-driver = { version = "0.3", optional = true } +ya-identity = "0.3" +ya-market = "0.4" +ya-metrics = "0.2" +ya-net = { version = "0.3", features = ["service"] } +ya-payment = "0.3" +ya-persistence = { version = "0.3", features = ["service"] } +ya-sb-proto = "0.6" +ya-sb-router = "0.6" ya-service-api = "0.1" -ya-service-api-derive = "0.1" -ya-service-api-interfaces = "0.1" -ya-service-api-web = "0.1" -ya-service-bus = "0.4" -ya-sgx = "0.1" +ya-service-api-derive = "0.2" +ya-service-api-interfaces = "0.2" +ya-service-api-web = "0.2" +ya-service-bus = "0.6" +ya-sgx = "0.2" ya-utils-path = "0.1" -ya-utils-futures = "0.1" -ya-utils-process = { version = "0.1", features = ["lock"] } -ya-utils-networking = "0.1" -ya-version = "0.1" -ya-vpn = "0.1" -ya-client = "0.6" -ya-client-model = "0.4" +ya-utils-futures = "0.2" +ya-utils-process = { version = "0.2", features = ["lock"] } +ya-utils-networking = "0.2" +ya-version = "0.2" +ya-vpn = "0.2" +ya-client = "0.7" +ya-client-model = "0.5" -gftp = { version = "^0.2", optional = true } # just to enable gftp build for cargo-deb -ya-provider = { version = "0.2", optional = true } # just to enable conditionally running some tests +gftp = { version = "0.3", optional = true } # just to enable gftp build for cargo-deb +ya-provider = { version = "0.3", optional = true } # just to enable conditionally running some tests actix-rt = "2.7" actix-service = "2" @@ -142,9 +142,12 @@ members = [ "golem_cli", "utils/actix_utils", "utils/agreement-utils", + "utils/cli", "utils/compile-time-utils", + "utils/file-logging", "utils/futures", "utils/manifest-utils", + "utils/manifest-utils/test-utils", "utils/networking", "utils/path", "utils/process", @@ -181,10 +184,19 @@ ya-service-api-interfaces = { path = "core/serv-api/interfaces" } ya-service-api-web = { path = "core/serv-api/web" } ## SERVICE BUS -ya-service-bus = { git = "https://github.com/golemfactory/ya-service-bus.git", rev = "a60f7476c39a18fd3a369b3045b878544ae25598"} -ya-sb-proto = { git = "https://github.com/golemfactory/ya-service-bus.git", rev = "a60f7476c39a18fd3a369b3045b878544ae25598"} -ya-sb-router = { git = "https://github.com/golemfactory/ya-service-bus.git", rev = "a60f7476c39a18fd3a369b3045b878544ae25598"} -ya-sb-util = { git = "https://github.com/golemfactory/ya-service-bus.git", rev = "a60f7476c39a18fd3a369b3045b878544ae25598"} +ya-service-bus = { git = "https://github.com/golemfactory/ya-service-bus.git", rev = "4a932a62d48c72dabf59020e85c523d3ff0be7d4"} +ya-sb-proto = { git = "https://github.com/golemfactory/ya-service-bus.git", rev = "4a932a62d48c72dabf59020e85c523d3ff0be7d4"} +ya-sb-router = { git = "https://github.com/golemfactory/ya-service-bus.git", rev = "4a932a62d48c72dabf59020e85c523d3ff0be7d4"} +ya-sb-util = { git = "https://github.com/golemfactory/ya-service-bus.git", rev = "4a932a62d48c72dabf59020e85c523d3ff0be7d4"} + +#ya-service-bus = { path = "../ya-service-bus" } +#ya-sb-proto = { path = "../ya-service-bus/crates/proto" } +#ya-sb-router = { path = "../ya-service-bus/crates/router" } +#ya-sb-util = { path = "../ya-service-bus/crates/util" } + +## CLIENT +#ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "ada12de80045f67670cc0d0fc7d9f2ee3d42ae11" } +#ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "ada12de80045f67670cc0d0fc7d9f2ee3d42ae11" } ## OTHERS gftp = { path = "core/gftp" } @@ -197,6 +209,7 @@ ya-file-logging = { path = "utils/file-logging" } ya-manifest-utils = { path = "utils/manifest-utils" } ya-transfer = { path = "utils/transfer" } ya-utils-actix = { path = "utils/actix_utils"} +ya-utils-cli = { path = "utils/cli"} ya-utils-futures = { path = "utils/futures" } ya-utils-networking = { path = "utils/networking" } ya-utils-path = { path = "utils/path" } @@ -205,12 +218,11 @@ ya-diesel-utils = { path = "utils/diesel-utils"} ya-metrics = { path = "core/metrics" } ya-provider = { path = "agent/provider"} - -ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "e1d4e843bba2d7b9e48299c9ec976ce307aa78e1" } -ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "e1d4e843bba2d7b9e48299c9ec976ce307aa78e1" } +## TEST UTILS +ya-manifest-test-utils = { path = "utils/manifest-utils/test-utils" } ethereum-tx-sign = { git = "https://github.com/mfranciszkiewicz/ethereum-tx-sign.git", rev = "1164c74187a9e2947faeaea7dde104c3cdec4195" } -graphene-sgx = { git = " https://github.com/golemfactory/graphene-rust.git", rev = "3619f63a0c1ff44962ab604e55cd5dfb62177c56" } +graphene-sgx = { git = " https://github.com/golemfactory/graphene-rust.git", rev = "dbd993ebad7f9190410ea390a589348479af6407" } web3 = { git = "https://github.com/golemfactory/rust-web3", branch = "update_ethabi" } # Speed up builds on macOS (will be default in next rust version probably) diff --git a/agent/provider/Cargo.toml b/agent/provider/Cargo.toml index 76929db21c..6f30836e77 100644 --- a/agent/provider/Cargo.toml +++ b/agent/provider/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ya-provider" description = "Yagna Provider Agent reference implementation." -version = "0.2.0" +version = "0.3.0" authors = ["Golem Factory "] edition = "2018" @@ -13,15 +13,16 @@ name = "ya-provider" path = "src/main.rs" [dependencies] -ya-manifest-utils = { version = "^0.1" } -ya-client = { version = "0.6", features = ['cli'] } -ya-client-model = "0.4" +ya-manifest-utils = { version = "0.2" } +ya-client = { version = "0.7", features = ['cli'] } +ya-client-model = "0.5" ya-compile-time-utils = "0.2" -ya-core-model = { version = "^0.6", features = ['activity', 'payment'] } +ya-core-model = { version = "^0.8", features = ['activity', 'payment'] } ya-file-logging = "0.1" -ya-utils-actix = "0.1" +ya-utils-actix = "0.2" +ya-utils-cli = "0.1" ya-utils-path = "0.1" -ya-utils-process = { version = "0.1", features = ['lock'] } +ya-utils-process = { version = "0.2", features = ['lock'] } ya-std-utils = "0.1" ya-negotiators = { git = "https://github.com/nieznanysprawiciel/ya-negotiator", rev = "9c32969bdcb14ccd18260fb570c68ad44bb95b76" } ya-agreement-utils = { git = 'https://github.com/nieznanysprawiciel/ya-negotiator', rev = "9c32969bdcb14ccd18260fb570c68ad44bb95b76" } @@ -59,8 +60,8 @@ serde_yaml = "0.8" shared_child = "0.3.4" signal-hook = "0.3" structopt = "0.3.20" -strum = "0.20" -strum_macros = "0.20" +strum ="0.24" +strum_macros = "0.24" sys-info = "0.7.0" thiserror = "1.0.14" tokio = { version = "1", features = ["process", "signal"] } @@ -76,5 +77,12 @@ nix = "0.22.0" winapi = { version = "0.3.8", features = ["fileapi"] } [dev-dependencies] +assert_cmd = "2.0" +base64 = "0.13" chrono = "0.4" -shlex = "1.1.0" +test-case = "2.1" +predicates = "2.1" +serial_test = "0.9" +shlex = "1.1" + +ya-manifest-test-utils = "0.1" diff --git a/agent/provider/examples/mock_requestor.rs b/agent/provider/examples/mock_requestor.rs index 890d017bc7..5dc228855d 100644 --- a/agent/provider/examples/mock_requestor.rs +++ b/agent/provider/examples/mock_requestor.rs @@ -1,4 +1,3 @@ -use serde_json; use std::{thread, time::Duration}; use ya_client::model::market::{AgreementProposal, NewDemand, RequestorEvent}; @@ -11,14 +10,14 @@ async fn query_events( let mut requestor_events = vec![]; while requestor_events.is_empty() { - requestor_events = client.collect(&subscription_id, Some(1.0), Some(2)).await?; + requestor_events = client.collect(subscription_id, Some(1.0), Some(2)).await?; println!("Waiting for events"); thread::sleep(Duration::from_millis(3000)); } println!("{} events found.", requestor_events.len()); - return Ok(requestor_events); + Ok(requestor_events) } async fn wait_for_approval(client: &MarketRequestorApi, proposal_id: &str) { @@ -61,7 +60,7 @@ async fn simulate_requestor(client: MarketRequestorApi) -> Result<()> { let _res = client.create_agreement(&agreement_proposal).await?; println!("Confirm agreement {}.", &agreement_proposal.proposal_id); - let _res = client + client .confirm_agreement(&agreement_proposal.proposal_id, None) .await?; diff --git a/agent/provider/src/cli.rs b/agent/provider/src/cli.rs index aa9b96d13a..bdb28cf95d 100644 --- a/agent/provider/src/cli.rs +++ b/agent/provider/src/cli.rs @@ -5,3 +5,13 @@ pub mod exe_unit; pub mod keystore; pub mod preset; pub mod profile; +pub mod whitelist; + +use crate::startup_config::ProviderConfig; + +/// Prints line if `json` output disabled. +fn println_conditional(config: &ProviderConfig, txt: &str) { + if !config.json { + println!("{txt}"); + } +} diff --git a/agent/provider/src/cli/keystore.rs b/agent/provider/src/cli/keystore.rs index 4fcbb446b6..9c384e7ddf 100644 --- a/agent/provider/src/cli/keystore.rs +++ b/agent/provider/src/cli/keystore.rs @@ -1,127 +1,160 @@ -use std::fs::OpenOptions; -use std::path::{Path, PathBuf}; +use std::collections::HashSet; +use std::path::PathBuf; -use anyhow::Context; -use serde::{Deserialize, Serialize}; use structopt::StructOpt; -use ya_manifest_utils::{KeyMeta, Keystore}; +use ya_manifest_utils::util::{self, CertBasicData, CertBasicDataVisitor}; +use ya_manifest_utils::KeystoreLoadResult; +use ya_utils_cli::{CommandOutput, ResponseTable}; +use crate::cli::println_conditional; use crate::startup_config::ProviderConfig; #[derive(StructOpt, Clone, Debug)] -#[structopt(rename_all = "kebab-case")] +#[structopt( + rename_all = "kebab-case", + help = "Keystore stores X.509 certificates. +They allow to accept Demands with Computation Payload Manifests which arrive with signature and app author's public certificate. +Certificate gets validated against certificates loaded into the keystore. +Certificates are stored in a file format in directory. Its location can be configured using '--cert-dir' param." +)] pub enum KeystoreConfig { - /// List trusted keys + /// List trusted X.509 certificates List, - /// Add a new trusted key + /// Add new trusted X.509 certificates Add(Add), - /// Remove a trusted key + /// Remove trusted X.509 certificates Remove(Remove), } #[derive(StructOpt, Clone, Debug)] -#[structopt(rename_all = "kebab-case")] pub struct Add { - key: String, - #[structopt(long, short)] - name: Option, - #[structopt(long, short)] - scheme: Option, + /// Paths to X.509 certificates or certificates chains + #[structopt( + parse(from_os_str), + help = "Space separated list of X.509 certificate files (PEM or DER) or PEM certificates chains to be added to the Keystore." + )] + certs: Vec, } #[derive(StructOpt, Clone, Debug)] #[structopt(rename_all = "kebab-case")] pub struct Remove { - name: String, + /// Certificate ids + #[structopt(help = "Space separated list of X.509 certificates' ids. +To find certificate id use `keystore list` command.")] + ids: Vec, } impl KeystoreConfig { pub fn run(self, config: ProviderConfig) -> anyhow::Result<()> { match self { KeystoreConfig::List => list(config), - KeystoreConfig::Add(add_) => add(config, add_), - KeystoreConfig::Remove(remove_) => remove(config, remove_), + KeystoreConfig::Add(cmd) => add(config, cmd), + KeystoreConfig::Remove(cmd) => remove(config, cmd), } } } fn list(config: ProviderConfig) -> anyhow::Result<()> { - let path = keystore_path(&config)?; - let keystore = Keystore::load(path)?; - let keys: Vec<_> = keystore - .keys() - .into_iter() - .map(FormattedKey::from) - .collect(); - - if keys.is_empty() { - return Ok(()); - } - - if config.json { - println!("{}", serde_json::to_string_pretty(&keys)?); - } else { - println!("Name\tKey\tScheme"); - for key in keys { - println!("\n{}\t{}\t{}", key.scheme, key.key, key.name); - } - } - + let cert_dir = config.cert_dir_path()?; + let table = CertTable::new(); + let table = util::visit_certificates(&cert_dir, table)?; + table.print(&config)?; Ok(()) } fn add(config: ProviderConfig, add: Add) -> anyhow::Result<()> { - let path = keystore_path(&config)?; - log::error!("key: {}", add.key); - let key = hex::decode(add.key).context("key is not a hex string")?; - log::error!("decoded: {:?}", key); - let keystore = Keystore::load(&path)?; - keystore.insert(key, add.scheme, add.name)?; - keystore.save(path)?; + let cert_dir = config.cert_dir_path()?; + let keystore_manager = util::KeystoreManager::try_new(&cert_dir)?; + match keystore_manager.load_certs(&add.certs)? { + KeystoreLoadResult::Loaded { loaded, skipped } => { + println_conditional(&config, "Added certificates:"); + let certs_data = util::to_cert_data(&loaded)?; + print_cert_list(&config, certs_data)?; + if !skipped.is_empty() && !config.json { + println!("Certificates already loaded to keystore:"); + let certs_data = util::to_cert_data(&skipped)?; + print_cert_list(&config, certs_data)?; + } + } + KeystoreLoadResult::NothingNewToLoad { skipped } => { + let certs_data = util::to_cert_data(&skipped)?; + if !config.json { + println!("No new certificate to add."); + println!("Dropped duplicated certificates:"); + print_cert_list(&config, certs_data)?; + } else { + // no new certificate added, so empty list for json output + print_cert_list(&config, Vec::new())?; + } + } + } Ok(()) } fn remove(config: ProviderConfig, remove: Remove) -> anyhow::Result<()> { - let path = keystore_path(&config)?; - let keystore = Keystore::load(&path)?; - keystore - .remove_by_name(remove.name) - .ok_or_else(|| anyhow::anyhow!("key does not exist")) - .map(|_| ())?; - keystore.save(path) + let cert_dir = config.cert_dir_path()?; + let keystore_manager = util::KeystoreManager::try_new(&cert_dir)?; + let ids: HashSet = remove.ids.into_iter().collect(); + match keystore_manager.remove_certs(&ids)? { + util::KeystoreRemoveResult::NothingToRemove => { + println_conditional(&config, "No matching certificates to remove."); + if config.json { + print_cert_list(&config, Vec::new())?; + } + } + util::KeystoreRemoveResult::Removed { removed } => { + println!("Removed certificates:"); + let certs_data = util::to_cert_data(&removed)?; + print_cert_list(&config, certs_data)?; + } + }; + Ok(()) } -fn touch(path: impl AsRef) -> anyhow::Result<()> { - let path = path.as_ref(); - OpenOptions::new() - .create(true) - .write(true) - .open(path) - .map(|_| ()) - .context(format!("unable to create file '{}'", path.display())) +fn print_cert_list( + config: &ProviderConfig, + certs_data: Vec, +) -> anyhow::Result<()> { + let mut table = CertTable::new(); + for data in certs_data { + table.add(data); + } + table.print(config)?; + Ok(()) } -fn keystore_path(config: &ProviderConfig) -> anyhow::Result { - let data_dir = config.data_dir.get_or_create()?; - let path = data_dir.join(config.trusted_keys_file.as_path()); - touch(path.as_path())?; - Ok(path) +struct CertTable { + table: ResponseTable, } -#[derive(Clone, Debug, Serialize, Deserialize)] -struct FormattedKey { - name: String, - key: String, - scheme: String, +impl CertTable { + pub fn new() -> Self { + let columns = vec![ + "ID".to_string(), + "Not After".to_string(), + "Subject".to_string(), + ]; + let values = vec![]; + let table = ResponseTable { columns, values }; + Self { table } + } + + pub fn print(self, config: &ProviderConfig) -> anyhow::Result<()> { + let output = CommandOutput::from(self.table); + output.print(config.json)?; + Ok(()) + } + + pub fn add(&mut self, data: CertBasicData) { + let row = serde_json::json! {[ data.id, data.not_after, data.subject ]}; + self.table.values.push(row) + } } -impl From<(Box<[u8]>, KeyMeta)> for FormattedKey { - fn from(tup: (Box<[u8]>, KeyMeta)) -> Self { - FormattedKey { - name: tup.1.name, - key: hex::encode(tup.0), - scheme: tup.1.scheme, - } +impl CertBasicDataVisitor for CertTable { + fn accept(&mut self, data: CertBasicData) { + self.add(data) } } diff --git a/agent/provider/src/cli/preset.rs b/agent/provider/src/cli/preset.rs index 0fe04e52d5..ea2b9262f3 100644 --- a/agent/provider/src/cli/preset.rs +++ b/agent/provider/src/cli/preset.rs @@ -209,12 +209,16 @@ pub fn create(config: ProviderConfig, params: PresetNoInteractive) -> anyhow::Re let mut presets = PresetManager::load_or_create(&config.presets_file)?; - let mut preset = Preset::default(); - preset.name = params - .preset_name - .ok_or(anyhow!("Preset name is required."))?; - preset.exeunit_name = params.exe_unit.ok_or(anyhow!("ExeUnit is required."))?; - preset.pricing_model = params.pricing.unwrap_or("linear".to_string()); + let mut preset = Preset { + name: params + .preset_name + .ok_or_else(|| anyhow!("Preset name is required."))?, + exeunit_name: params + .exe_unit + .ok_or_else(|| anyhow!("ExeUnit is required."))?, + pricing_model: params.pricing.unwrap_or_else(|| "linear".to_string()), + ..Default::default() + }; let registry = config.registry()?; @@ -224,7 +228,7 @@ pub fn create(config: ProviderConfig, params: PresetNoInteractive) -> anyhow::Re if is_initial_coefficient_name(name) { preset.initial_price = *price; } else { - let usage_coefficient = exe_unit_desc.resolve_coefficient(&name)?; + let usage_coefficient = exe_unit_desc.resolve_coefficient(name)?; preset.usage_coeffs.insert(usage_coefficient, *price); } @@ -319,11 +323,11 @@ fn update_presets( } else { preset .usage_coeffs - .insert(exe_unit_desc.resolve_coefficient(&name)?, *price); + .insert(exe_unit_desc.resolve_coefficient(name)?, *price); } } - validate_preset(&config, &preset)?; + validate_preset(config, preset)?; Ok(()) })?; @@ -347,7 +351,7 @@ fn validate_preset(config: &ProviderConfig, preset: &Preset) -> anyhow::Result<( let registry = config.registry()?; registry.find_exeunit(&preset.exeunit_name)?; - if !(preset.pricing_model == "linear") { + if preset.pricing_model != "linear" { bail!("Not supported pricing model.") } diff --git a/agent/provider/src/cli/profile.rs b/agent/provider/src/cli/profile.rs index 5fbf9c8eb3..b912f89b50 100644 --- a/agent/provider/src/cli/profile.rs +++ b/agent/provider/src/cli/profile.rs @@ -40,7 +40,7 @@ impl ProfileConfig { } ProfileConfig::Create { name, resources } => { let mut profiles = Profiles::load_or_create(&config)?; - if let Some(_) = profiles.get(&name) { + if profiles.get(&name).is_some() { return Err(ProfileError::AlreadyExists(name).into()); } profiles.add(name, resources)?; diff --git a/agent/provider/src/cli/whitelist.rs b/agent/provider/src/cli/whitelist.rs new file mode 100644 index 0000000000..7a8a4e1530 --- /dev/null +++ b/agent/provider/src/cli/whitelist.rs @@ -0,0 +1,245 @@ +use std::collections::HashMap; + +use structopt::StructOpt; + +use ya_manifest_utils::{ + matching::domain::{pattern_to_id, DomainPattern, DomainPatterns}, + ArgMatch, +}; +use ya_utils_cli::{CommandOutput, ResponseTable}; + +use crate::cli::println_conditional; +use crate::startup_config::ProviderConfig; + +#[derive(StructOpt, Clone, Debug)] +#[structopt( + rename_all = "kebab-case", + help = "Domain Whitelist allows to accept Demands with Computation Payload Manifests +which declare usage of Outbound Network but arrive with no signature." +)] +pub enum WhitelistConfig { + /// List domain whitelist patterns + List, + /// Add new domain whitelist patterns + Add(Add), + /// Remove domain whitelist patterns + Remove(Remove), +} + +#[derive(StructOpt, Clone, Debug)] +#[structopt(rename_all = "kebab-case")] +pub struct Add { + /// Domain whitelist patterns + #[structopt( + long, + short, + help = "Space separated domain Whitelist patterns. +Adding URL as a pattern ('regex' or 'strict') will not work." + )] + patterns: Vec, + + /// Domain whitelist patterns type + #[structopt( + long = "type", + short = "t", + default_value = "strict", + help = "Domain Whitelist pattern type takes value 'strict' or 'regex'. +Regex patterns are by default wrapped with '.*' patterns." + )] + pattern_type: ArgMatch, +} + +#[derive(StructOpt, Clone, Debug)] +#[structopt(rename_all = "kebab-case")] +pub struct Remove { + /// Domain whitelist pattern ids. + #[structopt( + help = "Space separated list of domain Whitelist patterns' ids to be removed. +To find pattern's id use 'whitelist list' command." + )] + ids: Vec, +} + +impl WhitelistConfig { + pub fn run(self, config: ProviderConfig) -> anyhow::Result<()> { + match self { + WhitelistConfig::List => list(config), + WhitelistConfig::Add(cmd) => add(config, cmd), + WhitelistConfig::Remove(cmd) => remove(config, cmd), + } + } +} + +fn list(config: ProviderConfig) -> anyhow::Result<()> { + let domain_patterns = DomainPatterns::load_or_create(&config.domain_whitelist_file)?; + let table = WhitelistTable::from(domain_patterns); + table.print(&config) +} + +fn add(config: ProviderConfig, add: Add) -> anyhow::Result<()> { + let domain_patterns = DomainPatterns::load_or_create(&config.domain_whitelist_file)?; + let mut domain_patterns = DomainPatternIds::from(domain_patterns); + let added = domain_patterns.add(add); + let domain_patterns: DomainPatterns = domain_patterns.into(); + domain_patterns.save(&config.domain_whitelist_file)?; + if !added.processed.is_empty() { + println_conditional(&config, "Added patterns:"); + WhitelistTable::from(DomainPatterns { + patterns: added.processed, + }) + .print(&config)? + } else { + println_conditional(&config, "No new patterns to add."); + if config.json { + // no new pattern to add, so empty list for json output + WhitelistTable::from(DomainPatterns { + patterns: Vec::new(), + }) + .print(&config)? + } + } + if !added.skipped.is_empty() && !config.json { + println!("Dropped duplicated patterns:"); + WhitelistTable::from(DomainPatterns { + patterns: added.skipped, + }) + .print(&config)?; + } + Ok(()) +} + +fn remove(config: ProviderConfig, remove: Remove) -> anyhow::Result<()> { + let domain_patterns = DomainPatterns::load_or_create(&config.domain_whitelist_file)?; + let mut domain_patterns = DomainPatternIds::from(domain_patterns); + let removed = domain_patterns.remove(remove.ids); + let domain_patterns: DomainPatterns = domain_patterns.into(); + domain_patterns.save(&config.domain_whitelist_file)?; + if !removed.processed.is_empty() { + let table = WhitelistTable::from(DomainPatterns { + patterns: removed.processed, + }); + println_conditional(&config, "Removed patterns:"); + table.print(&config)?; + } else { + println_conditional(&config, "No matching pattern to remove."); + // no new pattern added, so empty list for json output + if config.json { + WhitelistTable::from(DomainPatterns { + patterns: Vec::new(), + }) + .print(&config)? + } + }; + Ok(()) +} + +struct WhitelistTable { + table: ResponseTable, +} + +impl WhitelistTable { + pub fn new() -> Self { + let columns = vec!["ID".to_string(), "Pattern".to_string(), "Type".to_string()]; + let values = vec![]; + let table = ResponseTable { columns, values }; + Self { table } + } + + fn add(&mut self, pattern: DomainPattern) { + let id = pattern_to_id(&pattern); + let row = serde_json::json! {[ id, pattern.domain, pattern.domain_match ]}; + self.table.values.push(row); + } + + pub fn print(self, config: &ProviderConfig) -> anyhow::Result<()> { + let output = CommandOutput::from(self.table); + output.print(config.json)?; + Ok(()) + } +} + +impl From for WhitelistTable { + fn from(domain_patterns: DomainPatterns) -> Self { + let mut table = WhitelistTable::new(); + for pattern in domain_patterns.patterns { + table.add(pattern); + } + table + } +} + +/// Collection of `DomainPattern`s mapped to ids generated from their contents. +/// # See also +/// ya_manifest_utils::matching::domain::pattern_to_id +struct DomainPatternIds { + pattern_ids: HashMap, +} + +impl From for DomainPatternIds { + fn from(patterns: DomainPatterns) -> Self { + let mut pattern_ids = HashMap::new(); + let patterns = patterns.patterns; + for pattern in patterns { + let id = pattern_to_id(&pattern); + pattern_ids.insert(id, pattern); + } + Self { pattern_ids } + } +} + +impl From for DomainPatterns { + fn from(val: DomainPatternIds) -> Self { + let patterns = val.pattern_ids.into_values().collect(); + DomainPatterns { patterns } + } +} + +impl DomainPatternIds { + fn remove(&mut self, ids: Vec) -> DomainPatternsRemoved { + let mut removed = Vec::new(); + let mut skipped = Vec::new(); + for id in ids { + if let Some(pattern) = self.pattern_ids.remove(&id) { + removed.push(pattern); + } else { + skipped.push(id); + } + } + DomainPatternsRemoved { + processed: removed, + skipped, + } + } + + fn add(&mut self, add: Add) -> DomainPatternsAdded { + let mut added = Vec::new(); + let mut skipped = Vec::new(); + let domain_match = add.pattern_type; + for domain in add.patterns.into_iter() { + let domain = domain.to_lowercase(); + let pattern = DomainPattern { + domain, + domain_match, + }; + let id = pattern_to_id(&pattern); + if let Some(duplicate) = self.pattern_ids.insert(id, pattern.clone()) { + skipped.push(duplicate); + } else { + added.push(pattern) + } + } + DomainPatternsAdded { + processed: added, + skipped, + } + } +} + +struct DomainPatternsChange { + processed: Vec, + skipped: Vec, +} + +type DomainPatternsAdded = DomainPatternsChange; +/// Removed `DomainPattern`s with `id`s of skipped patterns. +type DomainPatternsRemoved = DomainPatternsChange; diff --git a/agent/provider/src/config/globals.rs b/agent/provider/src/config/globals.rs index c3dc4c7d50..a39f0712ae 100644 --- a/agent/provider/src/config/globals.rs +++ b/agent/provider/src/config/globals.rs @@ -6,8 +6,8 @@ use serde::{Deserialize, Deserializer, Serialize}; use std::{fs, io}; use ya_utils_path::SwapSave; -pub(crate) const GLOBALS_JSON: &'static str = "globals.json"; -pub(crate) const DEFAULT_SUBNET: &'static str = "public-beta"; +pub(crate) const GLOBALS_JSON: &str = "globals.json"; +pub(crate) const DEFAULT_SUBNET: &str = "public-beta"; fn default_subnet() -> Option { Some(DEFAULT_SUBNET.into()) @@ -16,9 +16,9 @@ fn default_subnet() -> Option { #[derive(Clone, Debug, Default, Serialize, derive_more::Display)] #[display( fmt = "{}{}{}", - "node_name.as_ref().map(|nn| format!(\"Node name: {}\", nn)).unwrap_or(\"\".into())", - "subnet.as_ref().map(|s| format!(\"\nSubnet: {}\", s)).unwrap_or(\"\".into())", - "account.as_ref().map(|a| format!(\"\nAccount: {}\", a)).unwrap_or(\"\".into())" + "node_name.as_ref().map(|nn| format!(\"Node name: {}\", nn)).unwrap_or_else(|| \"\".into())", + "subnet.as_ref().map(|s| format!(\"\nSubnet: {}\", s)).unwrap_or_else(|| \"\".into())", + "account.as_ref().map(|a| format!(\"\nAccount: {}\", a)).unwrap_or_else(|| \"\".into())" )] pub struct GlobalsState { pub node_name: Option, diff --git a/agent/provider/src/config/presets.rs b/agent/provider/src/config/presets.rs index b6f2184371..68aafc2315 100644 --- a/agent/provider/src/config/presets.rs +++ b/agent/provider/src/config/presets.rs @@ -17,7 +17,7 @@ pub struct PresetV0 { pub usage_coeffs: HashMap, } -#[derive(Serialize, Deserialize, Clone, Debug)] +#[derive(Serialize, Deserialize, Clone, Debug, Default)] #[serde(rename_all = "kebab-case")] pub struct Presets { pub active: Vec, @@ -57,7 +57,7 @@ impl Presets { presets .presets .get(name) - .ok_or(anyhow!("Invalid active preset: {:?}", name)) + .ok_or_else(|| anyhow!("Invalid active preset: {:?}", name)) .map(|_| ()) })?; @@ -103,15 +103,6 @@ impl Presets { } } -impl Default for Presets { - fn default() -> Self { - Presets { - active: Vec::new(), - presets: HashMap::new(), - } - } -} - impl From for Presets { fn from(presets_file: PresetsFile) -> Self { match presets_file { diff --git a/agent/provider/src/dir.rs b/agent/provider/src/dir.rs index cedfee0fff..ddd9cc4c4a 100644 --- a/agent/provider/src/dir.rs +++ b/agent/provider/src/dir.rs @@ -1,4 +1,4 @@ -use crate::startup_config::{GLOBALS_JSON, HARDWARE_JSON, PRESETS_JSON}; +use crate::startup_config::{CERT_DIR, GLOBALS_JSON, HARDWARE_JSON, PRESETS_JSON}; use anyhow::{bail, Result}; use std::path::Path; use std::time::{Duration, SystemTime}; @@ -22,6 +22,7 @@ fn is_provider_dir>(dir: P) -> Result { (HARDWARE_JSON, false), (PRESETS_JSON, false), (GLOBALS_JSON, false), + (CERT_DIR, false), ]; dir.as_ref() @@ -64,10 +65,8 @@ fn clean_dir>(dir: P, min_depth: usize, lifetime: Duration, dry_r } }) .fold(0, |acc, path_meta| { - if !dry_run { - if let Err(_) = std::fs::remove_file(&path_meta.0) { - return acc; - } + if !dry_run && std::fs::remove_file(&path_meta.0).is_err() { + return acc; } acc + path_meta.1.len() }); diff --git a/agent/provider/src/execution/registry.rs b/agent/provider/src/execution/registry.rs index b0c7e9f9a5..4a17812efc 100644 --- a/agent/provider/src/execution/registry.rs +++ b/agent/provider/src/execution/registry.rs @@ -109,7 +109,7 @@ impl ExeUnitDesc { return Some(coefficient_name.to_string()); } config.counters.iter().find_map(|(prop_name, definition)| { - if definition.name.eq_ignore_ascii_case(&coefficient_name) { + if definition.name.eq_ignore_ascii_case(coefficient_name) { Some(prop_name.into()) } else { None @@ -142,20 +142,15 @@ impl ExeUnitDesc { /// Responsible for creating ExeUnits. /// Stores registry of ExeUnits that can be created. +#[derive(Default)] pub struct ExeUnitsRegistry { descriptors: HashMap, } impl ExeUnitsRegistry { - pub fn new() -> ExeUnitsRegistry { - ExeUnitsRegistry { - descriptors: HashMap::new(), - } - } - pub fn from_file(path: &Path) -> Result { - let mut registry = ExeUnitsRegistry::new(); - registry.register_exeunits_from_file(&path)?; + let mut registry = ExeUnitsRegistry::default(); + registry.register_exeunits_from_file(path)?; Ok(registry) } @@ -171,7 +166,7 @@ impl ExeUnitsRegistry { ExeUnitInstance::new( name, &exeunit_desc.supervisor_path, - &working_dir, + working_dir, &extended_args, ) } @@ -200,10 +195,12 @@ impl ExeUnitsRegistry { // by supervisor as subprocess. let mut extended_args = Vec::new(); if let Some(runtime_path) = &exeunit_desc.runtime_path { - let runtime_path = runtime_path.to_str().ok_or(anyhow!( - "ExeUnit runtime path [{}] contains invalid characters.", - runtime_path.display() - ))?; + let runtime_path = runtime_path.to_str().ok_or_else(|| { + anyhow!( + "ExeUnit runtime path [{}] contains invalid characters.", + runtime_path.display() + ) + })?; extended_args.push("-b".to_owned()); extended_args.push(runtime_path.to_owned()); } @@ -243,7 +240,7 @@ impl ExeUnitsRegistry { pub fn register_exeunits_from_file(&mut self, path: &Path) -> Result<()> { let current_dir = std::env::current_dir()?; - let base_path = path.parent().unwrap_or_else(|| ¤t_dir); + let base_path = path.parent().unwrap_or(¤t_dir); let file = File::open(path).map_err(|error| { anyhow!( "Can't load ExeUnits to registry from file {}, error: {}.", @@ -301,7 +298,7 @@ impl ExeUnitsRegistry { if errors.is_empty() { return Ok(()); } - return Err(RegistryError(errors)); + Err(RegistryError(errors)) } pub fn test_runtimes(&self) -> anyhow::Result<()> { @@ -329,7 +326,7 @@ pub struct RegistryError(Vec); impl fmt::Display for RegistryError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { for v in &self.0 { - write!(f, "{}\n", v)?; + writeln!(f, "{}", v)?; } Ok(()) } @@ -366,7 +363,7 @@ impl OfferBuilder for ExeUnitDesc { let mut offer_part = self.properties.clone(); offer_part.append(common.as_object_mut().unwrap()); - return serde_json::Value::Object(offer_part); + serde_json::Value::Object(offer_part) } } @@ -383,7 +380,7 @@ fn test_runtime(path: &Path) -> anyhow::Result<()> { if message.is_empty() { message = String::from_utf8_lossy(&output.stdout).to_string(); } - if message.find("--help").is_none() { + if !message.contains("--help") { anyhow::bail!(message); } } @@ -443,7 +440,7 @@ impl fmt::Display for ExeUnitDesc { fn expand_filename(pattern: &Path) -> Result> { use std::fs::read_dir; - let path: &Path = pattern.as_ref(); + let path: &Path = pattern; let (base_dir, file_name) = match (path.parent(), path.file_name()) { (Some(base_dir), Some(file_name)) => (base_dir, file_name), _ => return Ok(vec![PathBuf::from(pattern)]), @@ -453,7 +450,7 @@ fn expand_filename(pattern: &Path) -> Result> None => anyhow::bail!("Not utf-8 filename: {:?}", file_name), }; - if let Some(pos) = file_name.find("*") { + if let Some(pos) = file_name.find('*') { let (prefix, suffix) = file_name.split_at(pos); let suffix = &suffix[1..]; @@ -490,73 +487,61 @@ mod tests { #[test] fn test_fill_registry_from_file() { - let mut registry = ExeUnitsRegistry::new(); + let mut registry = ExeUnitsRegistry::default(); registry .register_exeunits_from_file(&resources_directory().join("example-exeunits.json")) .unwrap(); let dummy_desc = registry.find_exeunit("dummy").unwrap(); assert_eq!(dummy_desc.name.as_str(), "dummy"); - assert_eq!( - dummy_desc - .supervisor_path - .to_str() - .unwrap() - .contains("dummy.exe"), - true - ); + assert!(dummy_desc + .supervisor_path + .to_str() + .unwrap() + .contains("dummy.exe")); let dummy_desc = registry.find_exeunit("wasm").unwrap(); assert_eq!(dummy_desc.name.as_str(), "wasm"); - assert_eq!( - dummy_desc - .supervisor_path - .to_str() - .unwrap() - .contains("wasm.exe"), - true - ); + assert!(dummy_desc + .supervisor_path + .to_str() + .unwrap() + .contains("wasm.exe")); } #[test] fn test_fill_registry_from_local_exe_unit_descriptor() { let exe_units_descriptor = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("../../exe-unit/resources/local-exeunits-descriptor.json"); - let mut registry = ExeUnitsRegistry::new(); + let mut registry = ExeUnitsRegistry::default(); registry .register_exeunits_from_file(&exe_units_descriptor) .unwrap(); let dummy_desc = registry.find_exeunit("wasmtime").unwrap(); assert_eq!(dummy_desc.name.as_str(), "wasmtime"); - assert_eq!( - dummy_desc - .supervisor_path - .to_str() - .unwrap() - .contains("exe-unit"), - true - ); + assert!(dummy_desc + .supervisor_path + .to_str() + .unwrap() + .contains("exe-unit")); } #[test] fn test_fill_registry_from_deb_exe_unit_descriptor() { let exe_units_descriptor = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("../../exe-unit/resources/exeunits-descriptor.json"); - let mut registry = ExeUnitsRegistry::new(); + let mut registry = ExeUnitsRegistry::default(); registry .register_exeunits_from_file(&exe_units_descriptor) .unwrap(); let dummy_desc = registry.find_exeunit("wasmtime").unwrap(); assert_eq!(dummy_desc.name.as_str(), "wasmtime"); - assert_eq!( - dummy_desc - .supervisor_path - .to_str() - .unwrap() - .contains("/usr/lib/yagna/plugins/exe-unit"), - true - ); + assert!(dummy_desc + .supervisor_path + .to_str() + .unwrap() + .contains("/usr/lib/yagna/plugins/exe-unit")); } } diff --git a/agent/provider/src/execution/task_runner.rs b/agent/provider/src/execution/task_runner.rs index 80ea85b96c..32c003cd23 100644 --- a/agent/provider/src/execution/task_runner.rs +++ b/agent/provider/src/execution/task_runner.rs @@ -184,18 +184,20 @@ impl TaskRunner { // Try convert to str to check if won't fail. If not we can than // unwrap() all paths that we created relative to current_dir. - data_dir.to_str().ok_or(anyhow!( - "Current dir [{}] contains invalid characters.", - data_dir.display() - ))?; + data_dir.to_str().ok_or_else(|| { + anyhow!( + "Current dir [{}] contains invalid characters.", + data_dir.display() + ) + })?; Ok(TaskRunner { api: Arc::new(client), registry, tasks: vec![], active_agreements: HashMap::new(), - activity_created: SignalSlot::::new(), - activity_destroyed: SignalSlot::::new(), + activity_created: SignalSlot::::default(), + activity_destroyed: SignalSlot::::default(), config: Arc::new(config), event_ts: Utc::now(), tasks_dir, @@ -216,7 +218,7 @@ impl TaskRunner { // =========================================== // async fn dispatch_events(events: Vec, myself: &Addr) { - if events.len() == 0 { + if events.is_empty() { return; }; @@ -264,13 +266,13 @@ impl TaskRunner { Some(agreement) => agreement, }; - let exeunit_name = exe_unit_name_from(&agreement)?; + let exeunit_name = exe_unit_name_from(agreement)?; let task = match self.create_task( &exeunit_name, &msg.activity_id, &msg.agreement_id, - msg.requestor_pub_key.as_ref().map(|s| s.as_str()), + msg.requestor_pub_key.as_deref(), ) { Ok(task) => task, Err(error) => bail!("Error creating activity: {:?}: {}", msg, error), @@ -288,18 +290,15 @@ impl TaskRunner { let mut finished = Box::pin(proc.wait_until_finished()); let mut monitor = StateMonitor::default(); - loop { - match select(Box::pin(api.get_activity_state(&activity_id)), finished).await { - Either::Left((result, fut)) => { - finished = fut; + while let Either::Left((result, fut)) = + select(Box::pin(api.get_activity_state(&activity_id)), finished).await + { + finished = fut; - if let Ok(state) = result { - monitor.update(state.state); - } - monitor.sleep().await; - } - Either::Right(_) => break, + if let Ok(state) = result { + monitor.update(state.state); } + monitor.sleep().await; } }); @@ -368,9 +367,9 @@ impl TaskRunner { let destroy_msg = ActivityDestroyed { agreement_id: msg.agreement_id.to_string(), - activity_id: msg.activity_id.clone(), + activity_id: msg.activity_id, }; - let _ = self.activity_destroyed.send_signal(destroy_msg.clone()); + let _ = self.activity_destroyed.send_signal(destroy_msg); Ok(()) } @@ -392,7 +391,7 @@ impl TaskRunner { let args = vec![String::from("offer-template")]; self.registry .run_exeunit_with_output(exeunit_name, args, &working_dir) - .map_err(|error| error.context(format!("ExeUnit offer-template command failed"))) + .map_err(|error| error.context("ExeUnit offer-template command failed".to_string())) } fn exeunit_coeffs(&self, exeunit_name: &str) -> Result> { @@ -428,22 +427,34 @@ impl TaskRunner { let agreement_path = working_dir .parent() - .ok_or(anyhow!("None"))? // Parent must exist, since we built this path. + .ok_or_else(|| anyhow!("None"))? // Parent must exist, since we built this path. .join("agreement.json"); - self.save_agreement(&agreement_path, &agreement_id)?; + self.save_agreement(&agreement_path, agreement_id)?; let mut args = vec![ "service-bus", activity_id, ya_core_model::activity::local::BUS_ID, ]; - args.extend(["-c", self.cache_dir.to_str().ok_or(anyhow!("None"))?].iter()); - args.extend(["-w", working_dir.to_str().ok_or(anyhow!("None"))?].iter()); - args.extend(["-a", agreement_path.to_str().ok_or(anyhow!("None"))?].iter()); + args.extend( + [ + "-c", + self.cache_dir.to_str().ok_or_else(|| anyhow!("None"))?, + ] + .iter(), + ); + args.extend(["-w", working_dir.to_str().ok_or_else(|| anyhow!("None"))?].iter()); + args.extend( + [ + "-a", + agreement_path.to_str().ok_or_else(|| anyhow!("None"))?, + ] + .iter(), + ); if let Some(req_pub_key) = requestor_pub_key { - args.extend(["--requestor-pub-key", req_pub_key.as_ref()].iter()); + args.extend(["--requestor-pub-key", req_pub_key].iter()); } let args = args.iter().map(ToString::to_string).collect(); @@ -473,7 +484,7 @@ impl TaskRunner { let agreement = self .active_agreements .get(agreement_id) - .ok_or(anyhow!("Can't find agreement [{}].", agreement_id))?; + .ok_or_else(|| anyhow!("Can't find agreement [{}].", agreement_id))?; let agreement_file = File::create(&agreement_path).map_err(|error| { anyhow!( @@ -575,7 +586,7 @@ forward_actix_handler!(TaskRunner, GetExeUnit, get_exeunit); actix_signal_handler!(TaskRunner, CreateActivity, activity_created); actix_signal_handler!(TaskRunner, ActivityDestroyed, activity_destroyed); -const PROPERTY_USAGE_VECTOR: &'static str = "golem.com.usage.vector"; +const PROPERTY_USAGE_VECTOR: &str = "golem.com.usage.vector"; impl Handler for TaskRunner { type Result = ResponseFuture>>; @@ -609,11 +620,7 @@ impl Handler for TaskRunner { { serde_json::Value::Array(vec) => { let mut usage_vector = vec.clone(); - usage_vector.extend( - coeffs - .into_iter() - .map(|prop| serde_json::Value::String(prop)), - ); + usage_vector.extend(coeffs.into_iter().map(serde_json::Value::String)); template.set_property( PROPERTY_USAGE_VECTOR, serde_json::Value::Array(usage_vector), @@ -638,14 +645,14 @@ impl Handler for TaskRunner { let addr = ctx.address(); let client = self.api.clone(); - let mut event_ts = self.event_ts.clone(); + let mut event_ts = self.event_ts; let app_session_id = self.config.session_id.clone(); let poll_timeout = Duration::from_secs(3); let fut = async move { let result = client .get_activity_events( - Some(event_ts.clone()), + Some(event_ts), Some(app_session_id), Some(poll_timeout), None, @@ -654,10 +661,9 @@ impl Handler for TaskRunner { match result { Ok(events) => { - events - .iter() - .max_by_key(|e| e.event_date) - .map(|e| event_ts = event_ts.max(e.event_date)); + if let Some(e) = events.iter().max_by_key(|e| e.event_date) { + event_ts = event_ts.max(e.event_date); + } Self::dispatch_events(events, &addr).await; } Err(error) => log::error!("Can't query activity events: {:?}", error), @@ -679,7 +685,7 @@ impl Handler for TaskRunner { fn handle(&mut self, msg: TerminateActivity, _ctx: &mut Context) -> Self::Result { let api = self.api.clone(); - let state_retry_interval = self.config.exeunit_state_retry_interval.clone(); + let state_retry_interval = self.config.exeunit_state_retry_interval; async move { set_activity_terminated( @@ -701,7 +707,7 @@ impl Handler for TaskRunner { fn handle(&mut self, msg: CreateActivity, ctx: &mut Context) -> Self::Result { let api = self.api.clone(); let activity_id = msg.activity_id.clone(); - let state_retry_interval = self.config.exeunit_state_retry_interval.clone(); + let state_retry_interval = self.config.exeunit_state_retry_interval; let result = self.on_create_activity(msg, ctx); match result { @@ -768,8 +774,8 @@ impl Handler for TaskRunner { type Result = ActorResponse>; fn handle(&mut self, msg: AgreementClosed, ctx: &mut Context) -> Self::Result { - let agreement_id = msg.agreement_id.to_string(); - let myself = ctx.address().clone(); + let agreement_id = msg.agreement_id; + let myself = ctx.address(); let activities = self.list_activities(&agreement_id); self.active_agreements.remove(&agreement_id); @@ -788,8 +794,8 @@ impl Handler for TaskRunner { type Result = ActorResponse>; fn handle(&mut self, msg: AgreementBroken, ctx: &mut Context) -> Self::Result { - let agreement_id = msg.agreement_id.to_string(); - let myself = ctx.address().clone(); + let agreement_id = msg.agreement_id; + let myself = ctx.address(); let activities = self.list_activities(&agreement_id); self.active_agreements.remove(&agreement_id); @@ -869,10 +875,8 @@ impl StateMonitor { _ => {} } - if self.state.0 == State::Unresponsive { - if state.0 != State::Unresponsive { - log::warn!("ExeUnit is now responsive"); - } + if self.state.0 == State::Unresponsive && state.0 != State::Unresponsive { + log::warn!("ExeUnit is now responsive"); } self.state = state; diff --git a/agent/provider/src/hardware.rs b/agent/provider/src/hardware.rs index e6ee655ace..cd3eb6f6f0 100644 --- a/agent/provider/src/hardware.rs +++ b/agent/provider/src/hardware.rs @@ -93,7 +93,7 @@ impl Resources { ) -> Result { let max_caps = Self::max_caps(path)?; if config.rt_cores.is_some() || config.rt_mem.is_some() || config.rt_storage.is_some() { - let mut user_caps = max_caps.clone(); + let mut user_caps = max_caps; if let Some(cores) = config.rt_cores { user_caps.cpu_threads = cores as i32; @@ -232,7 +232,7 @@ impl Profiles { } std::fs::File::create(&path)?; - let mut profiles = Self::try_with_config(&path, &config)?; + let mut profiles = Self::try_with_config(&path, config)?; let default_caps = Resources::default_caps(&path)?; for profile in profiles.profiles.values_mut() { *profile = profile.cap(&default_caps); @@ -260,7 +260,7 @@ impl Profiles { } fn try_with_config>(path: P, config: &ProviderConfig) -> Result { - let resources = Resources::try_with_config(path.as_ref(), &config)?; + let resources = Resources::try_with_config(path.as_ref(), config)?; let active = DEFAULT_PROFILE_NAME.to_string(); let profiles = vec![(active.clone(), resources)].into_iter().collect(); Ok(Profiles { active, profiles }) @@ -298,7 +298,7 @@ impl Profiles { if name == self.active { return Err(ProfileError::Active(name).into()); } - if let None = self.profiles.remove(&name) { + if self.profiles.remove(&name).is_none() { return Err(ProfileError::Unknown(name).into()); } Ok(()) @@ -351,7 +351,7 @@ impl ManagerState { .profiles .get(&self.profiles.active) .cloned() - .ok_or_else(|| ProfileError::Unknown(name))? + .ok_or(ProfileError::Unknown(name))? .cap(&self.res_available); if res == self.res_cap { @@ -369,11 +369,11 @@ impl ManagerState { impl Manager { pub fn try_new(conf: &ProviderConfig) -> Result { - let profiles = Profiles::load_or_create(&conf)?; + let profiles = Profiles::load_or_create(conf)?; let mut state = ManagerState { profiles, - res_available: Resources::try_with_config(conf.hardware_file.as_path(), &conf)?, + res_available: Resources::try_with_config(conf.hardware_file.as_path(), conf)?, res_cap: Resources::new_empty(), res_remaining: Resources::new_empty(), res_alloc: HashMap::new(), @@ -421,7 +421,7 @@ impl Manager { #[inline] pub fn capped(&self) -> Resources { let state = self.state.lock().unwrap(); - state.res_cap.clone() + state.res_cap } #[allow(dead_code)] @@ -580,9 +580,9 @@ mod tests { storage_gib: 200., }; let state = ManagerState { - res_available: res.clone(), - res_cap: res.clone(), - res_remaining: res.clone(), + res_available: res, + res_cap: res, + res_remaining: res, res_alloc: HashMap::new(), profiles: profiles(), }; @@ -599,9 +599,9 @@ mod tests { storage_gib: 12.37, }; - man.allocate("1".into(), alloc.clone()).unwrap(); - man.allocate("2".into(), alloc.clone()).unwrap(); - man.allocate("3".into(), alloc.clone()).unwrap(); + man.allocate("1".into(), alloc).unwrap(); + man.allocate("2".into(), alloc).unwrap(); + man.allocate("3".into(), alloc).unwrap(); man.release("1".into()).unwrap(); man.release("2".into()).unwrap(); man.release("3".into()).unwrap(); @@ -620,9 +620,9 @@ mod tests { storage_gib: 200., }; let state = ManagerState { - res_available: res.clone(), - res_cap: res.clone(), - res_remaining: res.clone(), + res_available: res, + res_cap: res, + res_remaining: res, res_alloc: HashMap::new(), profiles: profiles(), }; @@ -639,7 +639,7 @@ mod tests { storage_gib: 12.37, }; - man.allocate("1".into(), alloc.clone()).unwrap(); + man.allocate("1".into(), alloc).unwrap(); assert!(man.allocate("1".into(), alloc).is_err()); assert!(man.release("2".into()).is_err()); assert!(man diff --git a/agent/provider/src/main.rs b/agent/provider/src/main.rs index 2c6e61ec55..1e61e35a60 100644 --- a/agent/provider/src/main.rs +++ b/agent/provider/src/main.rs @@ -22,6 +22,7 @@ async fn main() -> anyhow::Result<()> { let mut config = cli_args.config; let data_dir = config.data_dir.get_or_create()?; + config.domain_whitelist_file = data_dir.join(config.domain_whitelist_file); config.globals_file = data_dir.join(config.globals_file); config.presets_file = data_dir.join(config.presets_file); config.hardware_file = data_dir.join(config.hardware_file); @@ -43,6 +44,7 @@ async fn main() -> anyhow::Result<()> { Commands::Profile(profile_cmd) => profile_cmd.run(config), Commands::ExeUnit(exe_unit_cmd) => exe_unit_cmd.run(config), Commands::Keystore(keystore_cmd) => keystore_cmd.run(config), + Commands::Whitelist(whitelist_cmd) => whitelist_cmd.run(config), Commands::Clean(clean_cmd) => clean_cmd.run(config), } } diff --git a/agent/provider/src/market/negotiator/accept_all.rs b/agent/provider/src/market/negotiator/accept_all.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/agent/provider/src/market/negotiator/accept_all.rs @@ -0,0 +1 @@ + diff --git a/agent/provider/src/market/negotiator/builtin/expiration.rs b/agent/provider/src/market/negotiator/builtin/expiration.rs index 22a31a8d59..128e38d7b4 100644 --- a/agent/provider/src/market/negotiator/builtin/expiration.rs +++ b/agent/provider/src/market/negotiator/builtin/expiration.rs @@ -27,17 +27,17 @@ pub struct LimitExpiration { min_deadline: i64, } -pub static DEBIT_NOTE_ACCEPT_TIMEOUT_PROPERTY: &'static str = +pub static DEBIT_NOTE_ACCEPT_TIMEOUT_PROPERTY: &str = "/golem/com/payment/debit-notes/accept-timeout?"; -pub static AGREEMENT_EXPIRATION_PROPERTY: &'static str = "/golem/srv/comp/expiration"; +pub static AGREEMENT_EXPIRATION_PROPERTY: &str = "/golem/srv/comp/expiration"; // TODO: We should unify properties access in agreement-utils, because it is annoying to use both forms. -pub static DEBIT_NOTE_ACCEPT_TIMEOUT_PROPERTY_FLAT: &'static str = +pub static DEBIT_NOTE_ACCEPT_TIMEOUT_PROPERTY_FLAT: &str = "golem.com.payment.debit-notes.accept-timeout?"; // Note: Tests are using this. #[allow(dead_code)] -pub static AGREEMENT_EXPIRATION_PROPERTY_FLAT: &'static str = "golem.srv.comp.expiration"; +pub static AGREEMENT_EXPIRATION_PROPERTY_FLAT: &str = "golem.srv.comp.expiration"; /// Configuration for LimitAgreements Negotiator. #[derive(StructOpt, Clone, Debug, Serialize, Deserialize)] @@ -146,43 +146,46 @@ impl NegotiatorComponent for LimitExpiration { // Both Provider and Requestor support DebitNotes acceptance. We must // negotiate until we will agree to the same value. (Some(req_deadline), Some(our_deadline)) => { - if req_deadline > our_deadline { - NegotiationResult::Reject { + match req_deadline.cmp(&our_deadline) { + std::cmp::Ordering::Greater => NegotiationResult::Reject { reason: RejectReason::new(format!( "DebitNote acceptance deadline should be less than {}.", self.accept_timeout.display() )), is_final: true, + }, + std::cmp::Ordering::Equal => { + // We agree with Requestor to the same deadline. + NegotiationResult::Ready { + proposal: ours, + score, + } } - } else if req_deadline == our_deadline { - // We agree with Requestor to the same deadline. - NegotiationResult::Ready { - proposal: ours, - score, - } - } else { - // Below certain timeout it is impossible for Requestor to accept DebitNotes. - if req_deadline.num_seconds() < self.min_deadline { - return Ok(NegotiationResult::Reject { - reason: RejectReason::new(format!( - "To low DebitNotes timeout: {}", - req_deadline.display() - )), - is_final: true, - }); - } - - // Requestor proposed better deadline, than we required. - // We are expected to set property to the same value if we agree. - let deadline_prop = ours - .pointer_mut(DEBIT_NOTE_ACCEPT_TIMEOUT_PROPERTY) - .unwrap(); - *deadline_prop = serde_json::Value::Number(req_deadline.num_seconds().into()); - - // Since we changed our proposal, we can't return `Ready`. - NegotiationResult::Negotiating { - proposal: ours, - score, + std::cmp::Ordering::Less => { + // Below certain timeout it is impossible for Requestor to accept DebitNotes. + if req_deadline.num_seconds() < self.min_deadline { + return Ok(NegotiationResult::Reject { + reason: RejectReason::new(format!( + "To low DebitNotes timeout: {}", + req_deadline.display() + )), + is_final: true, + }); + } + + // Requestor proposed better deadline, than we required. + // We are expected to set property to the same value if we agree. + let deadline_prop = ours + .pointer_mut(DEBIT_NOTE_ACCEPT_TIMEOUT_PROPERTY) + .unwrap(); + *deadline_prop = + serde_json::Value::Number(req_deadline.num_seconds().into()); + + // Since we changed our proposal, we can't return `Ready`. + NegotiationResult::Negotiating { + proposal: ours, + score, + } } } } diff --git a/agent/provider/src/market/negotiator/builtin/manifest.rs b/agent/provider/src/market/negotiator/builtin/manifest.rs index c5eb0c8322..7810f8e46b 100644 --- a/agent/provider/src/market/negotiator/builtin/manifest.rs +++ b/agent/provider/src/market/negotiator/builtin/manifest.rs @@ -7,20 +7,28 @@ use std::ops::Not; use std::path::PathBuf; use structopt::StructOpt; +use url::Url; use ya_agreement_utils::{Error, ProposalView}; -use ya_manifest_utils::manifest::{ - decode_manifest, Feature, Signature, CAPABILITIES_PROPERTY, DEMAND_MANIFEST_PROPERTY, - DEMAND_MANIFEST_SIG_PROPERTY, +use ya_manifest_utils::matching::domain::{ + DomainWhitelistState, DomainsMatcher, SharedDomainMatchers, }; +use ya_manifest_utils::matching::Matcher; use ya_manifest_utils::policy::{Keystore, Match, Policy, PolicyConfig}; +use ya_manifest_utils::{ + decode_manifest, AppManifest, Feature, CAPABILITIES_PROPERTY, DEMAND_MANIFEST_CERT_PROPERTY, + DEMAND_MANIFEST_PROPERTY, DEMAND_MANIFEST_SIG_ALGORITHM_PROPERTY, DEMAND_MANIFEST_SIG_PROPERTY, +}; use ya_negotiators::component::{RejectReason, Score}; use ya_negotiators::factory::{LoadMode, NegotiatorConfig}; use ya_negotiators::{NegotiationResult, NegotiatorComponent}; +use crate::market::negotiator::*; + #[derive(Default)] pub struct ManifestSignature { enabled: bool, - trusted_keys: Keystore, + keystore: Keystore, + whitelist_matcher: DomainsMatcher, } impl NegotiatorComponent for ManifestSignature { @@ -35,50 +43,44 @@ impl NegotiatorComponent for ManifestSignature { proposal: ours, score, }); + log::trace!("Manifest signature verification disabled."); + return acceptance(ours, score); } - let manifest = match their.get_property::(DEMAND_MANIFEST_PROPERTY) { - Err(Error::NoKey(_)) => { - return Ok(NegotiationResult::Ready { - proposal: ours, - score, - }) - } - Err(e) => return rejection(format!("invalid manifest type: {:?}", e)), - Ok(s) => match decode_manifest(s) { - Ok(manifest) => manifest, + let demand = match their.get_property::(DEMAND_MANIFEST_PROPERTY) { + Ok(manifest_encoded) => match decode_manifest(&manifest_encoded) { + Ok(manifest) => DemandWithManifest { + demand: their, + manifest_encoded, + manifest, + }, Err(e) => return rejection(format!("invalid manifest: {:?}", e)), }, + Err(Error::NoKey(_)) => return acceptance(ours, score), + Err(e) => return rejection(format!("invalid manifest type: {:?}", e)), }; - if manifest.features().is_empty() { - return Ok(NegotiationResult::Ready { - proposal: ours, - score, - }); - } - - let pub_key = match verify_signature(their) { - Ok(pub_key) => pub_key, - Err(e) => return rejection(format!("invalid manifest signature: {:?}", e)), - }; - - if self.trusted_keys.contains(pub_key.as_slice()) { - Ok(NegotiationResult::Ready { - proposal: ours, - score, - }) + if demand.has_signature() { + match demand.verify_signature(&self.keystore) { + Ok(()) => acceptance(ours, score), + Err(err) => rejection(format!("failed to verify manifest signature: {}", err)), + } + } else if demand.requires_signature(&self.whitelist_matcher) { + rejection("manifest requires signature but it has none".to_string()) } else { - rejection("manifest not signed by a trusted authority".to_string()) + log::trace!("No signature required. No signature provided."); + acceptance(ours, score) } } fn control_event(&mut self, _component: &str, params: Value) -> anyhow::Result { - let event: UpdateKeystore = - serde_json::from_value(params).map_err(|e| anyhow!("Unrecognized event: {e}"))?; - - self.trusted_keys = Keystore::load(event.keystore_path) - .map_err(|e| anyhow!("Failed to load keystore file: {e}"))?; + if let Some(event) = serde_json::from_value::(params.clone()) { + self.trusted_keys = Keystore::load(event.keystore_path) + .map_err(|e| anyhow!("Failed to load keystore file: {e}"))?; + } else if let Some(event) = serde_json::from_value::(params) { + self.whitelist_matcher = DomainsMatcher::load_or_create(&event.whitelist_path) + .map_err(|e| anyhow!("Failed to load keystore file: {e}"))?; + } // No return value. Ok(Value::Null) @@ -92,6 +94,13 @@ pub struct UpdateKeystore { pub keystore_path: PathBuf, } +// Control event sent to negotiator, that should cause whitelist update. +#[derive(Message, Clone, Debug, Serialize, Deserialize)] +#[rtype(result = "anyhow::Result")] +pub struct UpdateWhitelist { + pub whitelist_path: PathBuf, +} + impl ManifestSignature { pub fn new(config: serde_yaml::Value) -> anyhow::Result { let config: PolicyConfig = serde_yaml::from_value(config)?; @@ -130,13 +139,82 @@ impl From for ManifestSignature { } }; + let whitelist_matcher = config.domain_patterns.matchers.clone(); + let keystore = config.trusted_keys.unwrap_or_default(); ManifestSignature { enabled, - trusted_keys: config.trusted_keys.unwrap_or_default(), + keystore, + whitelist_matcher, } } } +struct DemandWithManifest<'demand> { + demand: &'demand ProposalView, + manifest_encoded: String, + manifest: AppManifest, +} + +impl<'demand> DemandWithManifest<'demand> { + fn has_signature(&self) -> bool { + self.demand + .get_property::(DEMAND_MANIFEST_SIG_PROPERTY) + .is_ok() + } + + fn requires_signature(&self, whitelist_matcher: &DomainsMatcher) -> bool { + let features = self.manifest.features(); + if features.is_empty() { + log::debug!("No features in demand. Signature not required."); + return false; + // Inet is the only feature + } else if features.contains(&Feature::Inet) && features.len() == 1 { + if let Some(urls) = self + .manifest + .comp_manifest + .as_ref() + .and_then(|comp| comp.net.as_ref()) + .and_then(|net| net.inet.as_ref()) + .and_then(|inet| inet.out.as_ref()) + .and_then(|out| out.urls.as_ref()) + { + let matcher = whitelist_matcher; + let non_whitelisted_urls: Vec<&str> = urls + .iter() + .flat_map(Url::host_str) + .filter(|domain| matcher.matches(domain).not()) + .collect(); + if non_whitelisted_urls.is_empty() { + log::debug!("Demand does not require signature. Every URL on whitelist"); + return false; + } + log::debug!( + "Demand requires signature. Non whitelisted URLs: {:?}", + non_whitelisted_urls + ); + return true; + } + } + log::debug!("Demand requires signature."); + true + } + + fn verify_signature(&self, keystore: &Keystore) -> anyhow::Result<()> { + let sig = self + .demand + .get_property::(DEMAND_MANIFEST_SIG_PROPERTY)?; + log::trace!("sig_hex: {}", sig); + let sig_alg: String = self + .demand + .get_property(DEMAND_MANIFEST_SIG_ALGORITHM_PROPERTY)?; + log::trace!("sig_alg: {}", sig_alg); + let cert: String = self.demand.get_property(DEMAND_MANIFEST_CERT_PROPERTY)?; + log::trace!("cert: {}", cert); + log::trace!("manifest: {}", &self.manifest_encoded); + keystore.verify_signature(cert, sig, sig_alg, &self.manifest_encoded) + } +} + fn rejection(message: String) -> anyhow::Result { Ok(NegotiationResult::Reject { reason: RejectReason::new(message), @@ -144,13 +222,11 @@ fn rejection(message: String) -> anyhow::Result { }) } -fn verify_signature(demand: &ProposalView) -> anyhow::Result> { - let manifest: String = demand.get_property(DEMAND_MANIFEST_PROPERTY)?; - log::debug!("manifest: {}", manifest); - let sig_hex: String = demand.get_property(DEMAND_MANIFEST_SIG_PROPERTY)?; - log::debug!("sig_hex: {}", sig_hex); - let sig = Signature::Secp256k1Hex(sig_hex); - Ok(sig.verify_str(manifest)?) +fn acceptance(offer: ProposalView, score: Score) -> anyhow::Result { + Ok(NegotiationResult::Ready { + proposal: offer, + score, + }) } #[cfg(test)] @@ -192,7 +268,7 @@ mod tests { --policy-disable-component manifest_signature_validation \ --policy-trust-property {}={}", CAPABILITIES_PROPERTY, - Feature::Inet.to_string() + Feature::Inet )); assert!(!policy.enabled); @@ -200,7 +276,7 @@ mod tests { "TEST \ --policy-trust-property {}={}", CAPABILITIES_PROPERTY, - Feature::Inet.to_string() + Feature::Inet )); assert!(!policy.enabled); diff --git a/agent/provider/src/market/negotiator/builtin/note_interval.rs b/agent/provider/src/market/negotiator/builtin/note_interval.rs index 1b27982926..6db9206af1 100644 --- a/agent/provider/src/market/negotiator/builtin/note_interval.rs +++ b/agent/provider/src/market/negotiator/builtin/note_interval.rs @@ -10,10 +10,8 @@ use ya_negotiators::{NegotiationResult, NegotiatorComponent}; use crate::display::EnableDisplay; pub const DEFAULT_DEBIT_NOTE_INTERVAL_SEC: u32 = 120; -pub const DEBIT_NOTE_INTERVAL_PROPERTY: &'static str = - "/golem/com/scheme/payu/debit-note/interval-sec?"; -const DEBIT_NOTE_INTERVAL_PROPERTY_FLAT: &'static str = - "golem.com.scheme.payu.debit-note.interval-sec?"; +pub const DEBIT_NOTE_INTERVAL_PROPERTY: &str = "/golem/com/scheme/payu/debit-note/interval-sec?"; +const DEBIT_NOTE_INTERVAL_PROPERTY_FLAT: &str = "golem.com.scheme.payu.debit-note.interval-sec?"; /// DebitNoteInterval negotiator pub struct DebitNoteInterval { diff --git a/agent/provider/src/market/negotiator/builtin/payment_timeout.rs b/agent/provider/src/market/negotiator/builtin/payment_timeout.rs index d5d41a0171..fad73f979a 100644 --- a/agent/provider/src/market/negotiator/builtin/payment_timeout.rs +++ b/agent/provider/src/market/negotiator/builtin/payment_timeout.rs @@ -9,9 +9,9 @@ use ya_negotiators::{NegotiationResult, NegotiatorComponent}; use crate::display::EnableDisplay; -const PAYMENT_TIMEOUT_PROPERTY_FLAT: &'static str = "golem.com.scheme.payu.payment-timeout-sec?"; -pub const PAYMENT_TIMEOUT_PROPERTY: &'static str = "/golem/com/scheme/payu/payment-timeout-sec?"; -const EXPIRATION_PROPERTY: &'static str = "/golem/srv/comp/expiration"; +const PAYMENT_TIMEOUT_PROPERTY_FLAT: &str = "golem.com.scheme.payu.payment-timeout-sec?"; +pub const PAYMENT_TIMEOUT_PROPERTY: &str = "/golem/com/scheme/payu/payment-timeout-sec?"; +const EXPIRATION_PROPERTY: &str = "/golem/srv/comp/expiration"; /// PaymentTimeout negotiator pub struct PaymentTimeout { diff --git a/agent/provider/src/market/negotiator/component.rs b/agent/provider/src/market/negotiator/component.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/agent/provider/src/market/negotiator/component.rs @@ -0,0 +1 @@ + diff --git a/agent/provider/src/market/negotiator/composite.rs b/agent/provider/src/market/negotiator/composite.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/agent/provider/src/market/negotiator/composite.rs @@ -0,0 +1 @@ + diff --git a/agent/provider/src/market/negotiator/factory.rs b/agent/provider/src/market/negotiator/factory.rs index d5edb1a395..85dd1a0d39 100644 --- a/agent/provider/src/market/negotiator/factory.rs +++ b/agent/provider/src/market/negotiator/factory.rs @@ -83,7 +83,7 @@ pub fn create_negotiator( "Composite" => NegotiatorAddr::from( CompositeNegotiator::new(market, &config.negotiator_config.composite_config).unwrap(), ), - "AcceptAll" => NegotiatorAddr::from(AcceptAllNegotiator::new()), + "AcceptAll" => NegotiatorAddr::from(AcceptAllNegotiator::default()), _ => Default::default(), }; Arc::new(negotiator) @@ -91,6 +91,6 @@ pub fn create_negotiator( impl Default for NegotiatorAddr { fn default() -> Self { - NegotiatorAddr::from(AcceptAllNegotiator::new()) + NegotiatorAddr::from(AcceptAllNegotiator::default()) } } diff --git a/agent/provider/src/market/presets.rs b/agent/provider/src/market/presets.rs index e20b2e54e7..66260c5643 100644 --- a/agent/provider/src/market/presets.rs +++ b/agent/provider/src/market/presets.rs @@ -132,7 +132,7 @@ impl PresetManager { state .presets .remove(name) - .ok_or(anyhow!("Preset [{}] doesn't exists.", name))?; + .ok_or_else(|| anyhow!("Preset [{}] doesn't exists.", name))?; Ok(()) } @@ -164,7 +164,7 @@ impl PresetManager { state.presets.values().cloned().collect() } - pub fn list_matching(&self, names: &Vec) -> Result> { + pub fn list_matching(&self, names: &[String]) -> Result> { let state = self.state.lock().unwrap(); names .iter() @@ -257,33 +257,33 @@ fn display_preset( let align = 20; let align_coeff = align - 4; // Minus indent. - write!(f, "{:width$}{}\n", "Name:", preset.name, width = align)?; - write!( + writeln!(f, "{:width$}{}", "Name:", preset.name, width = align)?; + writeln!( f, - "{:width$}{}\n", + "{:width$}{}", "ExeUnit:", preset.exeunit_name, width = align )?; - write!( + writeln!( f, - "{:width$}{}\n", + "{:width$}{}", "Pricing model:", preset.pricing_model, width = align )?; - write!(f, "{}\n", "Coefficients:")?; + writeln!(f, "Coefficients:")?; let exe_unit = registry.find_exeunit(&preset.exeunit_name).ok(); for (name, coeff) in preset.usage_coeffs.iter() { let price_desc = exe_unit .as_ref() - .and_then(|e| e.coefficient_name(&name)) + .and_then(|e| e.coefficient_name(name)) .unwrap_or_else(|| name.to_string()); - write!( + writeln!( f, - " {:width$}{} GLM\n", + " {:width$}{} GLM", price_desc, coeff, width = align_coeff diff --git a/agent/provider/src/market/provider_market.rs b/agent/provider/src/market/provider_market.rs index 55bd650d75..a312db14b5 100644 --- a/agent/provider/src/market/provider_market.rs +++ b/agent/provider/src/market/provider_market.rs @@ -224,8 +224,8 @@ impl ProviderMarket { config: Arc::new(config), subscriptions: HashMap::new(), postponed_demands: Vec::new(), - agreement_signed_signal: SignalSlot::::new(), - agreement_terminated_signal: SignalSlot::::new(), + agreement_signed_signal: SignalSlot::::default(), + agreement_terminated_signal: SignalSlot::::default(), handles: HashMap::new(), callbacks: Some(callbacks), }) @@ -282,13 +282,13 @@ async fn subscribe( async fn unsubscribe_all(api: Arc, subscriptions: Vec) -> Result<()> { for subscription in subscriptions.iter() { log::info!("Unsubscribing: {}", subscription); - api.unsubscribe(&subscription).await?; + api.unsubscribe(subscription).await?; } Ok(()) } async fn dispatch_events(ctx: AsyncCtx, events: Vec, subscription: &Subscription) { - if events.len() == 0 { + if events.is_empty() { return; }; @@ -677,12 +677,12 @@ impl Handler for ProviderMarket { let to_resubscribe = self .subscriptions .values() - .filter(|sub| &sub.id == &msg.0) + .filter(|sub| sub.id == msg.0) .cloned() .map(|sub| (sub.id.clone(), sub)) .collect::>(); - if to_resubscribe.len() > 0 { + if !to_resubscribe.is_empty() { return ActorResponse::r#async( resubscribe_offers(ctx.address(), self.api.clone(), to_resubscribe) .into_actor(self) @@ -746,18 +746,19 @@ impl Handler for ProviderMarket { type Result = ResponseFuture>; fn handle(&mut self, _msg: Shutdown, ctx: &mut Context) -> Self::Result { - for (_, handle) in self.handles.drain().into_iter() { + for (_, handle) in self.handles.drain() { ctx.cancel_future(handle); } let market = ctx.address(); async move { - Ok(market + market .send(Unsubscribe(OfferKind::Any)) .await? .map_err(|e| log::warn!("Failed to unsubscribe Offers. {}", e)) .ok() - .unwrap_or(())) + .unwrap_or(()); + Ok(()) } .boxed_local() } @@ -771,7 +772,7 @@ impl Handler for ProviderMarket { let reason = msg .reason .map(|msg| msg.message) - .unwrap_or("NotSpecified".to_string()); + .unwrap_or_else(|| "NotSpecified".to_string()); log::info!( "Agreement [{}] terminated by Requestor. Reason: {}", @@ -840,10 +841,10 @@ async fn terminate_agreement(api: Arc, msg: AgreementFinalize ProviderAgreementResult::ClosedByUs => GolemReason::success(), ProviderAgreementResult::BrokenByUs { reason } => GolemReason::new(reason), // No need to terminate, because Requestor already did it. - ProviderAgreementResult::ClosedByRequestor => return (), + ProviderAgreementResult::ClosedByRequestor => return, ProviderAgreementResult::BrokenByRequestor { .. } => return (), // No need to terminate since we didn't have Agreement with Requestor. - ProviderAgreementResult::ApprovalFailed => return (), + ProviderAgreementResult::ApprovalFailed => return, }; log::info!( @@ -864,7 +865,7 @@ async fn terminate_agreement(api: Arc, msg: AgreementFinalize e, repeats.max_elapsed_time, ); - return (); + return; } }; @@ -981,7 +982,7 @@ impl Handler for ProviderMarket { log::info!("Re-negotiating all demands"); - let demands = std::mem::replace(&mut myself.postponed_demands, Vec::new()); + let demands = std::mem::take(&mut myself.postponed_demands); ctx.spawn( renegotiate_demands(async_ctx, myself.subscriptions.clone(), demands) .into_actor(myself), @@ -999,7 +1000,7 @@ impl Handler for ProviderMarket { fn handle(&mut self, msg: AgreementClosed, ctx: &mut Context) -> Self::Result { let msg = AgreementFinalized::from(msg); - let myself = ctx.address().clone(); + let myself = ctx.address(); async move { myself.send(msg).await? }.boxed_local() } @@ -1012,7 +1013,7 @@ impl Handler for ProviderMarket { fn handle(&mut self, msg: AgreementBroken, ctx: &mut Context) -> Self::Result { let msg = AgreementFinalized::from(msg); - let myself = ctx.address().clone(); + let myself = ctx.address(); async move { myself.send(msg).await? }.boxed_local() } @@ -1025,7 +1026,7 @@ impl Handler for ProviderMarket { let subscriptions = match msg.0 { OfferKind::Any => { log::info!("Unsubscribing all active offers"); - std::mem::replace(&mut self.subscriptions, HashMap::new()) + std::mem::take(&mut self.subscriptions) .into_iter() .map(|(k, _)| k) .collect::>() @@ -1107,13 +1108,14 @@ actix_signal_handler!(ProviderMarket, NewAgreement, agreement_signed_signal); fn get_backoff() -> backoff::ExponentialBackoff { // TODO: We could have config for Market actor to be able to set at least initial interval. - let mut backoff = backoff::ExponentialBackoff::default(); - backoff.current_interval = std::time::Duration::from_secs(5); - backoff.initial_interval = std::time::Duration::from_secs(5); - backoff.multiplier = 1.5f64; - backoff.max_interval = std::time::Duration::from_secs(60 * 60); - backoff.max_elapsed_time = Some(std::time::Duration::from_secs(u64::max_value())); - backoff + backoff::ExponentialBackoff { + current_interval: std::time::Duration::from_secs(5), + initial_interval: std::time::Duration::from_secs(5), + multiplier: 1.5f64, + max_interval: std::time::Duration::from_secs(60 * 60), + max_elapsed_time: Some(std::time::Duration::from_secs(u64::max_value())), + ..Default::default() + } } // =========================================== // diff --git a/agent/provider/src/market/termination_reason.rs b/agent/provider/src/market/termination_reason.rs index 7faee7b44a..d92cda0814 100644 --- a/agent/provider/src/market/termination_reason.rs +++ b/agent/provider/src/market/termination_reason.rs @@ -7,7 +7,6 @@ use derive_more::Display; use serde::{Deserialize, Serialize}; use serde_json::Value; use strum::EnumMessage; -use strum_macros::*; use crate::display::EnableDisplay; @@ -78,7 +77,7 @@ impl TryFrom for BreakReason { } } -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] pub struct GolemReason { #[serde(rename = "message")] pub message: String, diff --git a/agent/provider/src/payments/agreement.rs b/agent/provider/src/payments/agreement.rs index 4097074f26..45aad07216 100644 --- a/agent/provider/src/payments/agreement.rs +++ b/agent/provider/src/payments/agreement.rs @@ -175,10 +175,7 @@ impl AgreementPayment { pub fn count_active_activities(&self) -> usize { self.activities .iter() - .filter(|(_, activity)| match activity { - ActivityPayment::Finalized { .. } => false, - _ => true, - }) + .filter(|(_, activity)| !matches!(activity, ActivityPayment::Finalized { .. })) .count() } @@ -268,12 +265,11 @@ impl ActivitiesWaiter { log::debug!("Waiting for all activities to finish."); - while let Some(value) = self + while let Ok(value) = self .watch_receiver .changed() .await .map(|_| *self.watch_receiver.borrow()) - .ok() { log::debug!("Num active activities left: {}.", value); if value == 0 { diff --git a/agent/provider/src/payments/mod.rs b/agent/provider/src/payments/mod.rs index f3d1bb7117..435c88ff8a 100644 --- a/agent/provider/src/payments/mod.rs +++ b/agent/provider/src/payments/mod.rs @@ -1,6 +1,7 @@ mod agreement; mod factory; mod model; +#[allow(clippy::module_inception)] mod payments; mod pricing; diff --git a/agent/provider/src/payments/model.rs b/agent/provider/src/payments/model.rs index d165ff42bc..7cf54827ac 100644 --- a/agent/provider/src/payments/model.rs +++ b/agent/provider/src/payments/model.rs @@ -13,7 +13,7 @@ use crate::market::negotiator::builtin::payment_timeout::PAYMENT_TIMEOUT_PROPERT /// Implementation of payment model which knows, how to compute amount /// of money, that requestor should pay for computations. pub trait PaymentModel { - fn compute_cost(&self, usage: &Vec) -> Result; + fn compute_cost(&self, usage: &[f64]) -> Result; fn expected_usage_len(&self) -> usize; } diff --git a/agent/provider/src/payments/payments.rs b/agent/provider/src/payments/payments.rs index 750425f90f..29268deee4 100644 --- a/agent/provider/src/payments/payments.rs +++ b/agent/provider/src/payments/payments.rs @@ -188,8 +188,8 @@ impl Payments { let provider_ctx = ProviderCtx { activity_api: Arc::new(activity_api), payment_api: Arc::new(payment_api), - debit_checker: DeadlineChecker::new().start(), - payment_checker: DeadlineChecker::new().start(), + debit_checker: DeadlineChecker::default().start(), + payment_checker: DeadlineChecker::default().start(), config, }; @@ -198,7 +198,7 @@ impl Payments { context: Arc::new(provider_ctx), invoices_to_pay: vec![], earnings: BigDecimal::zero(), - break_agreement_signal: SignalSlot::::new(), + break_agreement_signal: SignalSlot::::default(), invoice_signal: SignalSlot::::new(), } } @@ -337,10 +337,8 @@ async fn send_debit_note( Ok(debit_note) } -async fn check_invoice_events(provider_ctx: Arc, payments_addr: Addr) -> () { +async fn check_invoice_events(provider_ctx: Arc, payments_addr: Addr) { let config = &provider_ctx.config; - let timeout = config.get_events_timeout.clone(); - let error_timeout = config.get_events_error_timeout.clone(); let mut after_timestamp = Utc::now(); loop { @@ -348,7 +346,7 @@ async fn check_invoice_events(provider_ctx: Arc, payments_addr: Add .payment_api .get_invoice_events( Some(&after_timestamp), - Some(timeout), + Some(config.get_events_timeout), None, Some(config.session_id.clone()), ) @@ -357,7 +355,7 @@ async fn check_invoice_events(provider_ctx: Arc, payments_addr: Add Ok(events) => events, Err(e) => { log::error!("Can't query invoice events: {}", e); - tokio::time::sleep(error_timeout).await; + tokio::time::sleep(config.get_events_error_timeout).await; vec![] } }; @@ -487,8 +485,7 @@ async fn handle_debit_note_event( }) .log_err_msg(&format!( "Failed to send BreakAgreement for [{}] with reason: {}", - debit_note.agreement_id, - reason.to_string() + debit_note.agreement_id, reason )) .ok() } @@ -535,10 +532,7 @@ impl Handler for Payments { let agreement = self .agreements .get_mut(&msg.agreement_id) - .ok_or(anyhow!( - "Agreement [{}] wasn't registered.", - &msg.agreement_id - )) + .ok_or_else(|| anyhow!("Agreement [{}] wasn't registered.", &msg.agreement_id)) .log_warn_msg("[ActivityCreated]")?; log::info!( @@ -650,7 +644,7 @@ impl Handler for Payments { } .into_actor(self); - return ActorResponse::r#async(future.map(|_, _, _| Ok(()))); + ActorResponse::r#async(future.map(|_, _, _| Ok(()))) } } @@ -661,10 +655,12 @@ impl Handler for Payments { let agreement = match self .agreements .get(&msg.invoice_info.agreement_id) - .ok_or(anyhow!( - "Not my activity - agreement [{}].", - &msg.invoice_info.agreement_id - )) + .ok_or_else(|| { + anyhow!( + "Not my activity - agreement [{}].", + &msg.invoice_info.agreement_id + ) + }) .log_warn_msg("[UpdateCost]") { Ok(agreement) => agreement, @@ -696,30 +692,32 @@ impl Handler for Payments { // We break Agreement, if we weren't able to send any DebitNote lately. match result { Err(_) => { - if accept_timeout.is_some() && Utc::now() > last_debit_note + accept_timeout.unwrap() { - myself.break_agreement_signal - .send_signal(BreakAgreement { - agreement_id: msg.invoice_info.agreement_id.clone(), - reason: BreakReason::RequestorUnreachable(accept_timeout.unwrap()), - }) - .log_err_msg(&format!( - "Failed to send BreakAgreement for [{}], when Requestor is unreachable.", - msg.invoice_info.agreement_id - )) - .ok(); + if let Some(accept_timeout) = accept_timeout { + if Utc::now() > last_debit_note + accept_timeout { + myself.break_agreement_signal + .send_signal(BreakAgreement { + agreement_id: msg.invoice_info.agreement_id.clone(), + reason: BreakReason::RequestorUnreachable(accept_timeout), + }) + .log_err_msg(&format!( + "Failed to send BreakAgreement for [{}], when Requestor is unreachable.", + msg.invoice_info.agreement_id + )) + .ok(); + } } }, Ok(debit_note) => { - myself.agreements + // Payment due date is always set _before_ sending the DebitNote. + // The following synchronises the acceptance timeout check. + if let Some(agreement) = myself.agreements .get_mut(&msg.invoice_info.agreement_id) - // Payment due date is always set _before_ sending the DebitNote. - // The following synchronises the acceptance timeout check. - .map(|agreement| { + { agreement.last_send_debit_note = debit_note.timestamp; if debit_note.payment_due_date.is_some() { agreement.last_payable_debit_note = debit_note.timestamp } - }); + } } } @@ -761,10 +759,12 @@ impl Handler for Payments { if let Ok(agreement) = self .agreements .get_mut(&msg.debit_info.agreement_id) - .ok_or(anyhow!( - "Not my activity - agreement [{}].", - &msg.debit_info.agreement_id - )) + .ok_or_else(|| { + anyhow!( + "Not my activity - agreement [{}].", + &msg.debit_info.agreement_id + ) + }) .log_warn_msg("[FinalizeActivity]") { agreement @@ -790,7 +790,7 @@ impl Handler for Payments { let activities_watch = agreement.activities_watch.clone(); let agreement_id = msg.agreement_id.clone(); let payment_timeout = agreement.payment_timeout; - let myself = ctx.address().clone(); + let myself = ctx.address(); let ctx = self.context.clone(); let future = async move { @@ -822,7 +822,7 @@ impl Handler for Payments { return ActorResponse::r#async(future); } - return ActorResponse::reply(Err(anyhow!("Not my agreement {}.", &msg.agreement_id))); + ActorResponse::reply(Err(anyhow!("Not my agreement {}.", &msg.agreement_id))) } } @@ -912,16 +912,16 @@ impl Handler for Payments { return ActorResponse::reply(Ok(())); } - let address = ctx.address().clone(); + let address = ctx.address(); let future = async move { let msg = AgreementClosed { agreement_id: msg.agreement_id, send_terminate: false, }; - Ok(address.send(msg).await??) + address.send(msg).await? }; - return ActorResponse::r#async(future.into_actor(self)); + ActorResponse::r#async(future.into_actor(self)) } } @@ -947,7 +947,7 @@ impl Handler for Payments { Err(e) => Err(anyhow!("Cannot get invoice: {}", e)), }); - return ActorResponse::r#async(future); + ActorResponse::r#async(future) } } @@ -1084,8 +1084,7 @@ impl Handler for Payments { }) .log_err_msg(&format!( "Failed to send BreakAgreement for [{}] with reason: {}", - msg.category, - reason.to_string(), + msg.category, reason, )) .ok(); } @@ -1106,7 +1105,7 @@ impl Handler for Payments { }; return Ok(summary); } - return Err(anyhow!("Not my agreement {}.", &msg.agreement_id)); + Err(anyhow!("Not my agreement {}.", &msg.agreement_id)) } } @@ -1124,7 +1123,7 @@ impl Actor for Payments { payment_addr.clone(), )); tokio::task::spawn_local(async move { - for checker in vec![&provider_ctx.debit_checker, &provider_ctx.payment_checker] { + for checker in &[&provider_ctx.debit_checker, &provider_ctx.payment_checker] { let _ = checker .send(Subscribe(payment_addr.clone().recipient())) .await @@ -1146,8 +1145,8 @@ fn get_backoff() -> backoff::ExponentialBackoff { } } -const ACCEPT_PREFIX: &'static str = "debit-"; -const PAYMENT_PREFIX: &'static str = "payment-"; +const ACCEPT_PREFIX: &str = "debit-"; +const PAYMENT_PREFIX: &str = "payment-"; #[inline(always)] fn note_accept_id(id: impl AsRef) -> String { diff --git a/agent/provider/src/payments/pricing.rs b/agent/provider/src/payments/pricing.rs index 47bed990ef..746a3a8529 100644 --- a/agent/provider/src/payments/pricing.rs +++ b/agent/provider/src/payments/pricing.rs @@ -42,7 +42,7 @@ pub struct LinearPricing { } impl PaymentModel for LinearPricing { - fn compute_cost(&self, usage: &Vec) -> Result { + fn compute_cost(&self, usage: &[f64]) -> Result { // Note: last element of usage_coeffs contains constant initial cost // of computing task, so we don't multiply it. let const_coeff_idx = self.usage_coeffs.len() - 1; diff --git a/agent/provider/src/provider_agent.rs b/agent/provider/src/provider_agent.rs index 73042e95d4..3436e7fb39 100644 --- a/agent/provider/src/provider_agent.rs +++ b/agent/provider/src/provider_agent.rs @@ -1,6 +1,8 @@ use actix::prelude::*; use anyhow::{anyhow, Error}; use futures::{FutureExt, StreamExt, TryFutureExt}; +use ya_client::net::NetApi; +use ya_manifest_utils::matching::domain::{DomainPatterns, DomainWhitelistState, DomainsMatcher}; use std::convert::TryFrom; use std::path::{Path, PathBuf}; @@ -13,13 +15,14 @@ use ya_agreement_utils::*; use ya_client::cli::ProviderApi; use ya_core_model::payment::local::NetworkName; use ya_file_logging::{start_logger, LoggerHandle}; -use ya_manifest_utils::Keystore; +use ya_manifest_utils::{manifest, Feature, Keystore}; use crate::config::globals::GlobalsState; use crate::dir::clean_provider_dir; use crate::events::Event; use crate::execution::{ - GetExeUnit, GetOfferTemplates, Shutdown as ShutdownExecution, TaskRunner, UpdateActivity, + ExeUnitDesc, GetExeUnit, GetOfferTemplates, Shutdown as ShutdownExecution, TaskRunner, + UpdateActivity, }; use crate::hardware; use crate::market::provider_market::{OfferKind, Shutdown as MarketShutdown, Unsubscribe}; @@ -64,6 +67,57 @@ impl GlobalsManager { } } +/// Stores current whitelist state. +/// Starts and stops whitelist config file monitor. +struct WhitelistManager { + state: DomainWhitelistState, + monitor: Option, +} + +impl WhitelistManager { + fn try_new(whitelist_file: &Path) -> anyhow::Result { + let patterns = DomainPatterns::load_or_create(whitelist_file)?; + let state = DomainWhitelistState::try_new(patterns)?; + Ok(Self { + state, + monitor: None, + }) + } + + fn spawn_monitor(&mut self, whitelist_file: &Path) -> anyhow::Result<()> { + let state = self.state.clone(); + let handler = move |p: PathBuf| match DomainPatterns::load(&p) { + Ok(patterns) => { + match DomainsMatcher::try_from(&patterns) { + Ok(matcher) => { + *state.matchers.write().unwrap() = matcher; + *state.patterns.lock().unwrap() = patterns; + } + Err(err) => log::error!("Failed to update domain whitelist: {err}"), + }; + } + Err(e) => log::warn!( + "Error updating whitelist configuration from {:?}: {:?}", + p, + e + ), + }; + let monitor = FileMonitor::spawn(whitelist_file, FileMonitor::on_modified(handler))?; + self.monitor = Some(monitor); + Ok(()) + } + + fn get_state(&self) -> DomainWhitelistState { + self.state.clone() + } + + fn stop(&mut self) { + if let Some(monitor) = &mut self.monitor { + monitor.stop(); + } + } +} + pub struct ProviderAgent { globals: GlobalsManager, market: Addr, @@ -75,6 +129,8 @@ pub struct ProviderAgent { log_handler: LoggerHandle, networks: Vec, keystore_monitor: FileMonitor, + net_api: NetApi, + domain_whitelist: WhitelistManager, } impl ProviderAgent { @@ -82,11 +138,7 @@ impl ProviderAgent { let data_dir = config.data_dir.get_or_create()?; //log_dir is the same as data_dir by default, but can be changed using --log-dir option - let log_dir = if let Some(log_dir) = &config.log_dir { - log_dir.get_or_create()? - } else { - data_dir.clone() - }; + let log_dir = config.log_dir_path()?; //start_logger is using env var RUST_LOG internally. //args.debug options sets default logger to debug @@ -130,9 +182,14 @@ impl ProviderAgent { .clone() .unwrap_or_else(|| app_name.to_string()); + let cert_dir = config.cert_dir_path()?; + let keystore = load_keystore(&cert_dir)?; + args.market.session_id = format!("{}-{}", name, std::process::id()); args.runner.session_id = args.market.session_id.clone(); args.payment.session_id = args.market.session_id.clone(); + let policy_config = &mut args.market.negotiator_config.composite_config.policy_config; + policy_config.trusted_keys = Some(keystore.clone()); let networks = args.node.account.networks.clone(); for n in networks.iter() { @@ -146,12 +203,17 @@ impl ProviderAgent { }; log::info!("Using payment network: {}", net_color.paint(&n)); } + let mut globals = GlobalsManager::try_new(&config.globals_file, args.node)?; globals.spawn_monitor(&config.globals_file)?; let mut presets = PresetManager::load_or_create(&config.presets_file)?; presets.spawn_monitor(&config.presets_file)?; let mut hardware = hardware::Manager::try_new(&config)?; hardware.spawn_monitor(&config.hardware_file)?; + let keystore_monitor = spawn_keystore_monitor(cert_dir, keystore)?; + let mut domain_whitelist = WhitelistManager::try_new(&config.domain_whitelist_file)?; + domain_whitelist.spawn_monitor(&config.domain_whitelist_file)?; + policy_config.domain_patterns = domain_whitelist.get_state(); let market = ProviderMarket::new(api.market, &data_dir, args.market)?.start(); let payments = Payments::new(api.activity.clone(), api.payment, args.payment).start(); @@ -159,6 +221,7 @@ impl ProviderAgent { TaskRunner::new(api.activity, args.runner, registry, data_dir.clone())?.start(); let task_manager = TaskManager::new(market.clone(), runner.clone(), payments, args.tasks)?.start(); + let net_api = api.net; let keystore_path = data_dir.join(&config.trusted_keys_file); let keystore_monitor = spawn_keystore_monitor(market.clone(), &keystore_path)?; @@ -174,6 +237,8 @@ impl ProviderAgent { log_handler, networks, keystore_monitor, + net_api, + domain_whitelist, }) } @@ -192,51 +257,67 @@ impl ProviderAgent { let preset_names = presets.iter().map(|p| &p.name).collect::>(); log::debug!("Preset names: {:?}", preset_names); let offer_templates = runner.send(GetOfferTemplates(presets.clone())).await??; - let subnet = &node_info.subnet; for preset in presets { - let pricing_model: Box = match preset.pricing_model.as_str() { - "linear" => Box::new(LinearPricingOffer::default()), - other => return Err(anyhow!("Unsupported pricing model: {}", other)), - }; - let mut offer: OfferTemplate = offer_templates + let offer: OfferTemplate = offer_templates .get(&preset.name) .ok_or_else(|| anyhow!("Offer template not found for preset [{}]", preset.name))? .clone(); - - let (initial_price, prices) = get_prices(pricing_model.as_ref(), &preset, &offer)?; - offer.set_property("golem.com.usage.vector", get_usage_vector_value(&prices)); - offer.add_constraints(Self::build_constraints(subnet.clone())?); - - let com_info = pricing_model.build(&accounts, initial_price, prices)?; - let name = preset.exeunit_name.clone(); - let exeunit_desc = runner.send(GetExeUnit { name }).await?.map_err(|error| { - anyhow!( - "Failed to create offer for preset [{}]. Error: {}", - preset.name, - error - ) - })?; - - let srv_info = ServiceInfo::new(inf_node_info.clone(), exeunit_desc.build()) - .support_multi_activity(true); - - // Create simple offer on market. - let create_offer_message = CreateOffer { + let exeunit_name = preset.exeunit_name.clone(); + let exeunit_desc = runner + .send(GetExeUnit { name: exeunit_name }) + .await? + .map_err(|error| { + anyhow!( + "Failed to create offer for preset [{}]. Error: {}", + preset.name, + error + ) + })?; + + let offer = Self::build_offer( + node_info.clone(), + inf_node_info.clone(), + &accounts, preset, - offer_definition: OfferDefinition { - node_info: node_info.clone(), - srv_info, - com_info, - offer, - }, - }; + offer, + exeunit_desc, + )?; - market.send(create_offer_message).await??; + market.send(offer).await??; } Ok(()) } + fn build_offer( + node_info: NodeInfo, + inf_node_info: InfNodeInfo, + accounts: &[AccountView], + preset: Preset, + mut offer: OfferTemplate, + exeunit_desc: ExeUnitDesc, + ) -> anyhow::Result { + let pricing_model: Box = match preset.pricing_model.as_str() { + "linear" => Box::new(LinearPricingOffer::default()), + other => return Err(anyhow!("Unsupported pricing model: {}", other)), + }; + let (initial_price, prices) = get_prices(pricing_model.as_ref(), &preset, &offer)?; + offer.set_property("golem.com.usage.vector", get_usage_vector_value(&prices)); + offer.add_constraints(Self::build_constraints(node_info.subnet.clone())?); + let com_info = pricing_model.build(accounts, initial_price, prices)?; + let srv_info = Self::build_service_info(inf_node_info, exeunit_desc, &offer)?; + let offer_definition = OfferDefinition { + node_info, + srv_info, + com_info, + offer, + }; + Ok(CreateOffer { + preset, + offer_definition, + }) + } + fn build_constraints(subnet: Option) -> anyhow::Result { let mut cnts = constraints!["golem.srv.comp.expiration" > chrono::Utc::now().timestamp_millis(),]; @@ -246,18 +327,36 @@ impl ProviderAgent { Ok(cnts.to_string()) } - fn create_node_info(&self) -> NodeInfo { - let globals = self.globals.get_state(); - + async fn build_node_info(globals: GlobalsState, net_api: NetApi) -> anyhow::Result { if let Some(subnet) = &globals.subnet { log::info!("Using subnet: {}", yansi::Color::Fixed(184).paint(subnet)); } - - NodeInfo { + let status = net_api.get_status().await?; + Ok(NodeInfo { name: globals.node_name, subnet: globals.subnet, geo_country_code: None, - } + is_public: status.public_ip.is_some(), + }) + } + + fn build_service_info( + inf_node_info: InfNodeInfo, + exeunit_desc: ExeUnitDesc, + offer: &OfferTemplate, + ) -> anyhow::Result { + let exeunit_desc = exeunit_desc.build(); + let support_payload_manifest = match offer.property(manifest::CAPABILITIES_PROPERTY) { + Some(value) => { + serde_json::from_value(value.clone()).map(|capabilities: Vec| { + capabilities.contains(&Feature::ManifestSupport) + })? + } + None => false, + }; + Ok(ServiceInfo::new(inf_node_info, exeunit_desc) + .support_payload_manifest(support_payload_manifest) + .support_multi_activity(true)) } fn accounts(&self, networks: &Vec) -> anyhow::Result> { @@ -313,6 +412,20 @@ impl ProviderAgent { } } +fn load_keystore(cert_dir: &PathBuf) -> anyhow::Result { + let keystore = match Keystore::load(cert_dir) { + Ok(keystore) => { + log::info!("Trusted key store loaded from {}", cert_dir.display()); + keystore + } + Err(err) => { + log::error!("Failed to load keystore: {}", err); + Default::default() + } + }; + Ok(keystore) +} + fn get_prices( pricing_model: &dyn PricingOffer, preset: &Preset, @@ -327,7 +440,7 @@ fn get_prices( .get_initial_price() .ok_or_else(|| anyhow!("Preset [{}] is missing the initial price", preset.name))?; let prices = pricing_model - .prices(&preset) + .prices(preset) .into_iter() .filter_map(|(prop, v)| match offer_usage_vec.contains(&prop.as_str()) { true => Some((prop, v)), @@ -450,10 +563,8 @@ impl Handler for ProviderAgent { { let mut state = preset_state.lock().unwrap(); new_names.retain(|n| { - if state.active.contains(n) { - if !updated.contains(n) { - return false; - } + if state.active.contains(n) && !updated.contains(n) { + return false; } true }); @@ -501,6 +612,7 @@ impl Handler for ProviderAgent { let runner = self.runner.clone(); let log_handler = self.log_handler.clone(); self.keystore_monitor.stop(); + self.domain_whitelist.stop(); async move { market.send(MarketShutdown).await??; @@ -519,7 +631,6 @@ impl Handler for ProviderAgent { fn handle(&mut self, msg: CreateOffers, _: &mut Context) -> Self::Result { let runner = self.runner.clone(); let market = self.market.clone(); - let node_info = self.create_node_info(); let accounts = match self.accounts(&self.networks) { Ok(acc) => acc, Err(e) => return Box::pin(async { Err(e) }), @@ -533,9 +644,12 @@ impl Handler for ProviderAgent { vec![] } }; - let presets = self.presets.list_matching(&preset_names); + let globals = self.globals.get_state(); + let net_api = self.net_api.clone(); + async move { + let node_info = Self::build_node_info(globals, net_api).await?; Self::create_offers(presets?, node_info, inf_node_info, runner, market, accounts).await } .boxed_local() @@ -562,3 +676,107 @@ pub struct Shutdown; #[derive(Message)] #[rtype(result = "Result<(), Error>")] struct CreateOffers(pub OfferKind); + +/// Tests + +#[cfg(test)] +mod tests { + use test_case::test_case; + use ya_agreement_utils::{InfNodeInfo, NodeInfo, OfferTemplate}; + use ya_manifest_utils::manifest; + + use crate::{ + execution::ExeUnitDesc, market::Preset, payments::AccountView, + provider_agent::ProviderAgent, + }; + + #[test_case(true, r#"["inet", "vpn", "manifest-support"]"# ; "Supported with 'inet', 'vpn', and 'manifest-support'")] + #[test_case(true, r#"["manifest-support"]"# ; "Supported with 'manifest-support' only")] + #[test_case(false, r#"["inet"]"# ; "Not supported with 'inet' only")] + #[test_case(false, r#"["no_such_capability"]"# ; "Not supported with unknown 'no_such_capability' capability")] + #[test_case(false, r#"[]"# ; "Not supported with empty capabilities")] + fn payload_manifest_support_test(expected_manifest_suport: bool, runtime_capabilities: &str) { + let mut fake = fake_data(); + fake.offer_template + .properties + .as_object_mut() + .expect("Template properties are object") + .insert( + manifest::CAPABILITIES_PROPERTY.to_string(), + serde_json::from_str(runtime_capabilities).expect("Failed to serialize property"), + ); + + let offer = ProviderAgent::build_offer( + fake.node_info, + fake.inf_node_info, + &fake.accounts, + fake.preset, + fake.offer_template, + fake.exeunit_desc, + ) + .expect("Failed to build offer"); + + let offer_definition = offer.offer_definition.into_json(); + let payload_manifest_prop = offer_definition + .get("golem.srv.caps.payload-manifest") + .expect("Offer property golem.srv.caps.payload-manifest does not exist") + .as_bool() + .expect("Offer property golem.srv.caps.payload-manifest is not bool"); + assert_eq!(payload_manifest_prop, expected_manifest_suport); + } + + /// Test utilities + + struct FakeData { + node_info: NodeInfo, + inf_node_info: InfNodeInfo, + accounts: Vec, + preset: Preset, + offer_template: OfferTemplate, + exeunit_desc: ExeUnitDesc, + } + + fn fake_data() -> FakeData { + let node_info = NodeInfo { + name: Some("node_name".to_string()), + subnet: Some("subnet".to_string()), + geo_country_code: None, + is_public: true, + }; + let inf_node_info = InfNodeInfo::default(); + let accounts = Vec::new(); + + let preset = Preset { + pricing_model: "linear".to_string(), + usage_coeffs: std::collections::HashMap::from([("test_coefficient".to_string(), 1.0)]), + ..Default::default() + }; + + let offer_template = OfferTemplate { + properties: serde_json::json!({ + "golem.com.usage.vector": ["test_coefficient"] + }), + ..Default::default() + }; + + let exeunit_desc = ExeUnitDesc { + name: Default::default(), + version: semver::Version::new(0, 0, 1), + description: None, + supervisor_path: Default::default(), + extra_args: Default::default(), + runtime_path: None, + properties: Default::default(), + config: None, + }; + + FakeData { + node_info, + inf_node_info, + accounts, + preset, + offer_template, + exeunit_desc, + } + } +} diff --git a/agent/provider/src/signal.rs b/agent/provider/src/signal.rs index cf8b2035d9..c2fdf8d68c 100644 --- a/agent/provider/src/signal.rs +++ b/agent/provider/src/signal.rs @@ -53,11 +53,9 @@ impl Future for SignalMonitor { impl Drop for SignalMonitor { fn drop(&mut self) { - std::mem::replace(&mut self.hooks, Vec::new()) - .into_iter() - .for_each(|s| { - unregister(s); - }); + std::mem::take(&mut self.hooks).into_iter().for_each(|s| { + unregister(s); + }); } } diff --git a/agent/provider/src/startup_config.rs b/agent/provider/src/startup_config.rs index 8f95b8e38d..3399891a2e 100644 --- a/agent/provider/src/startup_config.rs +++ b/agent/provider/src/startup_config.rs @@ -21,6 +21,7 @@ use crate::cli::exe_unit::ExeUnitsConfig; use crate::cli::keystore::KeystoreConfig; pub use crate::cli::preset::PresetsConfig; use crate::cli::profile::ProfileConfig; +use crate::cli::whitelist::WhitelistConfig; pub(crate) use crate::config::globals::GLOBALS_JSON; use crate::execution::{ExeUnitsRegistry, TaskRunnerConfig}; use crate::market::config::MarketConfig; @@ -28,13 +29,15 @@ use crate::payments::PaymentsConfig; use crate::tasks::config::TaskConfig; lazy_static::lazy_static! { - pub static ref DEFAULT_DATA_DIR: String = DataDir::new(clap::crate_name!()).to_string(); - + pub static ref DEFAULT_DATA_DIR: String = default_data_dir(); pub static ref DEFAULT_PLUGINS_DIR : PathBuf = default_plugins(); } -pub(crate) const PRESETS_JSON: &'static str = "presets.json"; -pub(crate) const HARDWARE_JSON: &'static str = "hardware.json"; -pub(crate) const TRUSTED_KEYS_FILE: &'static str = "trusted_keys"; +pub(crate) const DOMAIN_WHITELIST_JSON: &str = "domain_whitelist.json"; +pub(crate) const PRESETS_JSON: &str = "presets.json"; +pub(crate) const HARDWARE_JSON: &str = "hardware.json"; +pub(crate) const CERT_DIR: &str = "cert_dir"; + +const DATA_DIR_ENV: &str = "DATA_DIR"; /// Common configuration for all Provider commands. #[derive(StructOpt, Clone, Debug)] @@ -53,7 +56,7 @@ pub struct ProviderConfig { #[structopt( long, set = clap::ArgSettings::Global, - env = "DATA_DIR", + env = DATA_DIR_ENV, default_value = &*DEFAULT_DATA_DIR, )] pub data_dir: DataDir, @@ -63,36 +66,42 @@ pub struct ProviderConfig { set = clap::ArgSettings::Global, env = "PROVIDER_LOG_DIR", )] - pub log_dir: Option, - + log_dir: Option, + /// Certificates directory + #[structopt( + long, + set = clap::ArgSettings::Global, + env = "PROVIDER_CERT_DIR", + )] + cert_dir: Option, + #[structopt(skip = DOMAIN_WHITELIST_JSON)] + pub domain_whitelist_file: PathBuf, #[structopt(skip = GLOBALS_JSON)] pub globals_file: PathBuf, #[structopt(skip = PRESETS_JSON)] pub presets_file: PathBuf, #[structopt(skip = HARDWARE_JSON)] pub hardware_file: PathBuf, - #[structopt(skip = TRUSTED_KEYS_FILE)] - pub trusted_keys_file: PathBuf, /// Max number of available CPU cores #[structopt( long, set = clap::ArgSettings::Global, - env = "YA_RT_CORES") - ] + env = "YA_RT_CORES" + )] pub rt_cores: Option, /// Max amount of available RAM (GiB) #[structopt( long, set = clap::ArgSettings::Global, - env = "YA_RT_MEM") - ] + env = "YA_RT_MEM" + )] pub rt_mem: Option, /// Max amount of available storage (GiB) #[structopt( long, set = clap::ArgSettings::Global, - env = "YA_RT_STORAGE") - ] + env = "YA_RT_STORAGE" + )] pub rt_storage: Option, #[structopt(long, set = clap::ArgSettings::Global)] @@ -101,16 +110,37 @@ pub struct ProviderConfig { impl ProviderConfig { pub fn registry(&self) -> anyhow::Result { - let mut r = ExeUnitsRegistry::new(); + let mut r = ExeUnitsRegistry::default(); r.register_from_file_pattern(&self.exe_unit_path)?; Ok(r) } + + pub fn log_dir_path(&self) -> anyhow::Result { + let log_dir = if let Some(log_dir) = &self.log_dir { + log_dir.get_or_create()? + } else { + self.data_dir.get_or_create()? + }; + Ok(log_dir) + } + + pub fn cert_dir_path(&self) -> anyhow::Result { + let cert_dir = if let Some(cert_dir) = &self.cert_dir { + cert_dir.get_or_create()? + } else { + let mut cert_dir = self.data_dir.get_or_create()?; + cert_dir.push("cert_dir"); + std::fs::create_dir_all(&cert_dir)?; + cert_dir + }; + Ok(cert_dir) + } } #[derive(StructOpt, Clone, Debug, Serialize, Deserialize, derive_more::Display)] #[display( fmt = "{}Networks: {:?}", - "account.map(|a| format!(\"Address: {}\n\", a)).unwrap_or(\"\".into())", + "account.map(|a| format!(\"Address: {}\n\", a)).unwrap_or_else(|| \"\".into())", networks )] pub struct ReceiverAccount { @@ -190,6 +220,7 @@ pub struct StartupConfig { pub commands: Commands, } +#[allow(clippy::large_enum_variant)] #[derive(StructOpt, Clone)] pub enum Commands { /// Run provider agent @@ -204,6 +235,8 @@ pub enum Commands { ExeUnit(ExeUnitsConfig), /// Manage trusted keys Keystore(KeystoreConfig), + /// Manage domain whitelist + Whitelist(WhitelistConfig), /// Clean up disk space Clean(CleanConfig), } @@ -245,7 +278,7 @@ impl FileMonitor { pub fn spawn(path: P, handler: H) -> std::result::Result where P: AsRef, - H: Fn(DebouncedEvent) -> () + Send + 'static, + H: Fn(DebouncedEvent) + Send + 'static, { Self::spawn_with(path, handler, Default::default()) } @@ -257,7 +290,7 @@ impl FileMonitor { ) -> std::result::Result where P: AsRef, - H: Fn(DebouncedEvent) -> () + Send + 'static, + H: Fn(DebouncedEvent) + Send + 'static, { let path = path.as_ref().to_path_buf(); let path_th = path.clone(); @@ -270,7 +303,7 @@ impl FileMonitor { let mut active = false; loop { if !active { - match watcher.watch(&path_th, RecursiveMode::NonRecursive) { + match watcher.watch(&path_th, RecursiveMode::Recursive) { Ok(_) => active = true, Err(e) => { if config.verbose { @@ -314,9 +347,9 @@ impl FileMonitor { } } - pub fn on_modified(f: F) -> impl Fn(DebouncedEvent) -> () + pub fn on_modified(f: F) -> impl Fn(DebouncedEvent) where - F: Fn(PathBuf) -> () + Send + 'static, + F: Fn(PathBuf) + Send + 'static, { move |e| match e { DebouncedEvent::Write(p) @@ -352,8 +385,12 @@ where Ok((s[..pos].parse()?, s[pos + 1..].parse()?)) } +fn default_data_dir() -> String { + DataDir::new(clap::crate_name!()).to_string() +} + fn default_plugins() -> PathBuf { - if let Some(mut exe) = env::current_exe().ok() { + if let Ok(mut exe) = env::current_exe() { exe.pop(); exe.push("plugins"); if exe.is_dir() { diff --git a/agent/provider/src/tasks/task_info.rs b/agent/provider/src/tasks/task_info.rs index c00a9c4d3c..ed2f86683a 100644 --- a/agent/provider/src/tasks/task_info.rs +++ b/agent/provider/src/tasks/task_info.rs @@ -84,7 +84,7 @@ mod test { .unwrap(); let info = TaskInfo::from(&view).unwrap(); - assert_eq!(info.multi_activity, true); + assert!(info.multi_activity); } pub static SAMPLE_AGREEMENT_MULTI_FALSE: &str = r#"{ @@ -105,7 +105,7 @@ mod test { .unwrap(); let info = TaskInfo::from(&view).unwrap(); - assert_eq!(info.multi_activity, false); + assert!(!info.multi_activity); } pub static SAMPLE_AGREEMENT_MULTI_EMPTY: &str = r#"{ @@ -125,6 +125,6 @@ mod test { .unwrap(); let info = TaskInfo::from(&view).unwrap(); - assert_eq!(info.multi_activity, false); + assert!(!info.multi_activity); } } diff --git a/agent/provider/src/tasks/task_manager.rs b/agent/provider/src/tasks/task_manager.rs index 4de1f8650d..7478edca7b 100644 --- a/agent/provider/src/tasks/task_manager.rs +++ b/agent/provider/src/tasks/task_manager.rs @@ -40,7 +40,7 @@ pub struct BreakAgreement { pub reason: BreakReason, } -#[derive(Clone, PartialEq)] +#[derive(Clone, PartialEq, Eq)] pub enum ClosingCause { ApprovalFail, Termination, @@ -506,7 +506,7 @@ impl Handler for TaskManager { start_transition(&actx.myself, &msg.agreement_id, new_state.clone()).await?; - let result = async move { + async move { let msg = AgreementBroken::from(msg.clone()); actx.runner.send(msg.clone()).await??; // Notify market, but we don't care about result. @@ -521,9 +521,7 @@ impl Handler for TaskManager { log::info!("Agreement [{}] cleanup finished.", msg.agreement_id); Ok(()) } - .await; - - result + .await } .map_err(move |error: Error| log::error!("Can't break agreement. Error: {}", error)); @@ -581,7 +579,7 @@ async fn start_transition( agreement_id: agreement_id.to_string(), new_state, }; - Ok(myself.clone().send(msg).await??) + myself.clone().send(msg).await? } async fn finish_transition( @@ -593,7 +591,7 @@ async fn finish_transition( agreement_id: agreement_id.to_string(), new_state, }; - Ok(myself.clone().send(msg).await??) + myself.clone().send(msg).await? } /// Helper struct storing TaskManager sub-actors addresses to use in async functions. diff --git a/agent/provider/src/tasks/task_state.rs b/agent/provider/src/tasks/task_state.rs index f8abfed5ab..42cc8fadf6 100644 --- a/agent/provider/src/tasks/task_state.rs +++ b/agent/provider/src/tasks/task_state.rs @@ -269,12 +269,11 @@ impl TasksStates { impl StateWaiter { /// Returns final state of Agreement. pub async fn transition_finished(&mut self) -> anyhow::Result { - while let Some(change) = self + while let Ok(change) = self .changed_receiver .changed() .await .map(|_| self.changed_receiver.borrow().clone()) - .ok() { if let StateChange::TransitionFinished(state) = change { return Ok(state); diff --git a/agent/provider/tests/keystore_cli.rs b/agent/provider/tests/keystore_cli.rs new file mode 100644 index 0000000000..87781a0a05 --- /dev/null +++ b/agent/provider/tests/keystore_cli.rs @@ -0,0 +1,102 @@ +use std::path::PathBuf; + +use assert_cmd::Command; +use serial_test::serial; + +#[serial] +#[test] +fn keystore_list_cmd_creates_cert_dir_in_data_dir_set_by_env() { + // Having + let mut data_dir = PathBuf::from(env!("CARGO_TARGET_TMPDIR")); + data_dir.push("data_dir"); + let mut cert_dir = data_dir.clone(); + cert_dir.push("cert_dir"); + #[allow(unused_must_use)] + { + std::fs::remove_dir_all(&data_dir); // ignores error if does not exist + std::fs::remove_dir_all(&cert_dir); // ignores error if does not exist + } + // When + Command::cargo_bin("ya-provider") + .unwrap() + .env("DATA_DIR", data_dir.as_path().to_str().unwrap()) + .arg("keystore") + .arg("list") + .arg("--json") + .assert() + .stdout("[]\n") + .success(); + // Then + assert!( + cert_dir.exists(), + "Cert dir has been created inside of data dir" + ); +} + +#[serial] +#[test] +fn keystore_list_cmd_creates_cert_dir_in_dir_set_by_env() { + // Having + let mut data_dir = PathBuf::from(env!("CARGO_TARGET_TMPDIR")); + data_dir.push("data_dir"); + let mut cert_dir = PathBuf::from(env!("CARGO_TARGET_TMPDIR")); + cert_dir.push("cert_dir"); + #[allow(unused_must_use)] + { + std::fs::remove_dir_all(&data_dir); // ignores error if does not exist + std::fs::remove_dir_all(&cert_dir); // ignores error if does not exist + } + let mut wrong_cert_dir = data_dir.clone(); + wrong_cert_dir.push("cert_dir"); + // When + Command::cargo_bin("ya-provider") + .unwrap() + .env("DATA_DIR", data_dir.as_path().to_str().unwrap()) + .env("PROVIDER_CERT_DIR", cert_dir.as_path().to_str().unwrap()) + .arg("keystore") + .arg("list") + .arg("--json") + .assert() + .stdout("[]\n") + .success(); + // Then + assert!( + cert_dir.exists(), + "Cert dir has been created inside of dir pointed by $PROVIDER_CERT_DIR" + ); + assert!( + !wrong_cert_dir.exists(), + "Cert dir has not been created inside of data dir pointed by $DATA_DIR" + ); +} + +#[serial] +#[test] +fn keystore_list_cmd_creates_cert_dir_in_dir_set_by_arg() { + // Having + let mut data_dir = PathBuf::from(env!("CARGO_TARGET_TMPDIR")); + data_dir.push("data_dir"); + let mut cert_dir = PathBuf::from(env!("CARGO_TARGET_TMPDIR")); + cert_dir.push("cert_dir"); + #[allow(unused_must_use)] + { + std::fs::remove_dir_all(&data_dir); // ignores error if does not exist + std::fs::remove_dir_all(&cert_dir); // ignores error if does not exist + } + // When + Command::cargo_bin("ya-provider") + .unwrap() + .env("DATA_DIR", data_dir.as_path().to_str().unwrap()) + .args(["--cert-dir", cert_dir.as_path().to_str().unwrap()]) + .arg("keystore") + .arg("list") + .arg("--json") + .assert() + .stdout("[]\n") + .success(); + // Then + assert!( + cert_dir.exists(), + "Cert dir has been created inside of dir pointed by --cert-dir arg" + ); +} diff --git a/agent/provider/tests/manifest_negotiator.rs b/agent/provider/tests/manifest_negotiator.rs new file mode 100644 index 0000000000..26b4053281 --- /dev/null +++ b/agent/provider/tests/manifest_negotiator.rs @@ -0,0 +1,336 @@ +#[macro_use] +extern crate serial_test; + +use std::collections::HashMap; +use std::fs; +use std::path::PathBuf; + +use serde_json::{json, Value}; +use test_case::test_case; +use ya_agreement_utils::AgreementView; +use ya_manifest_test_utils::{load_certificates_from_dir, TestResources}; +use ya_manifest_utils::matching::domain::{DomainPatterns, DomainWhitelistState}; +use ya_manifest_utils::{Keystore, Policy, PolicyConfig}; +use ya_provider::market::negotiator::builtin::ManifestSignature; +use ya_provider::market::negotiator::*; + +static MANIFEST_TEST_RESOURCES: TestResources = TestResources { + temp_dir: env!("CARGO_TARGET_TMPDIR"), +}; + +#[test_case( + r#"{ "patterns": [{ "domain": "domain.com", "type": "strict" }] }"#, // data_dir/domain_whitelist.json + r#"["https://domain.com"]"#, // compManifest.net.inet.out.urls + r#"{ "any": "thing" }"#, // offer + None, // private key file + None, // sig alg + None, // cert + None; // error msg + "Manifest without singature accepted because domain whitelisted" +)] +#[test_case( + r#"{ "patterns": [{ "domain": "do.*ain.com", "type": "regex" }, { "domain": "another.com", "type": "strict" }] }"#, // data_dir/domain_whitelist.json + r#"["https://domain.com"]"#, // compManifest.net.inet.out.urls + r#"{ "any": "thing" }"#, // offer + None, // private key file + None, // sig alg + None, // cert + None; // error msg + "Manifest without singature accepted because domain whitelisted (regex pattern)" +)] +#[test_case( + r#"{ "patterns": [{ "domain": "different_domain.com", "type": "strict" }] }"#, // data_dir/domain_whitelist.json + r#"["https://domain.com"]"#, // compManifest.net.inet.out.urls + r#"{ "any": "thing" }"#, // offer + None, // private key file + None, // sig alg + None, // cert + Some("manifest requires signature but it has none"); // error msg + "Manifest without singature rejected because domain NOT whitelisted" +)] +#[test_case( + r#"{ "patterns": [{ "domain": "domain.com", "type": "regex" }, { "domain": "another.whitelisted.com", "type": "strict" }] }"#, // data_dir/domain_whitelist.json + r#"["https://domain.com", "https://not.whitelisted.com"]"#, // compManifest.net.inet.out.urls + r#"{ "any": "thing" }"#, // offer + None, // private key file + None, // sig alg + None, // cert + Some("manifest requires signature but it has none"); // error msg + "Manifest without singature rejected because ONE of domains NOT whitelisted" +)] +#[test_case( + r#"{ "patterns": [{ "domain": "domain.com", "type": "regex" }] }"#, // data_dir/domain_whitelist.json + r#"[]"#, // compManifest.net.inet.out.urls + r#"{ "any": "thing" }"#, // offer + None, // private key file + None, // sig alg + None, // cert + None; // error msg + "Manifest accepted because its urls list is empty" +)] +#[test_case( + r#"{ "patterns": [{ "domain": "domain.com", "type": "regex" }] }"#, // data_dir/domain_whitelist.json + r#"["https://domain.com"]"#, // compManifest.net.inet.out.urls + r#"{ "any": "thing" }"#, // offer + Some("foo_req.key.pem"), // private key file + Some("sha256"), // sig alg + Some("foo_req.cert.pem"), // cert + None; // error msg + "Manifest accepted with url NOT whitelisted because signature valid" +)] +#[test_case( + r#"{ "patterns": [{ "domain": "domain.com", "type": "strict" }] }"#, // data_dir/domain_whitelist.json + r#"["https://domain.com"]"#, // compManifest.net.inet.out.urls + r#"{ "any": "thing" }"#, // offer + Some("foo_req.key.pem"), // private key file + Some("sha256"), // sig alg + Some("dummy_inter.cert.pem"), // untrusted cert + Some("failed to verify manifest signature: Invalid certificate"); // error msg + "Manifest rejected because of invalid certificate even when urls domains are whitelisted" +)] +#[test_case( + r#"{ "patterns": [{ "domain": "domain.com", "type": "strict" }] }"#, // data_dir/domain_whitelist.json + r#"["https://domain.com"]"#, // compManifest.net.inet.out.urls + r#"{ "any": "thing" }"#, // offer + Some("foo_inter.key.pem"), // private key file not matching certificate + Some("sha256"), // sig alg + Some("foo_req.cert.pem"), // certificate not matching private key + Some("failed to verify manifest signature: Invalid signature"); // error msg + "Manifest rejected because of invalid signature (signed using incorrect private key) even when urls domains are whitelisted" +)] +#[serial] +fn manifest_negotiator_test( + whitelist: &str, + urls: &str, + offer: &str, + signing_key: Option<&str>, + signature_alg: Option<&str>, + cert: Option<&str>, + error_msg: Option<&str>, +) { + let comp_manifest_b64 = create_comp_manifest_b64(urls); + + let signature_b64 = signing_key.map(|signing_key| { + MANIFEST_TEST_RESOURCES.sign_data(comp_manifest_b64.as_bytes(), signing_key) + }); + + let cert_b64 = cert.map(cert_file_to_cert_b64); + + manifest_negotiator_test_encoded_manifest_sign_and_cert( + whitelist, + comp_manifest_b64, + offer, + signature_b64, + signature_alg, + cert_b64, + error_msg, + ) +} + +#[test_case( + r#"{ "patterns": [{ "domain": "domain.com", "type": "strict" }] }"#, // data_dir/domain_whitelist.json + r#"["https://domain.com"]"#, // compManifest.net.inet.out.urls + r#"{ "any": "thing" }"#, // offer + Some("broken_signature"), // signature (broken) + Some("sha256"), // sig alg + Some("foo_req.cert.pem"), // cert + Some("failed to verify manifest signature: Invalid signature"); // error msg + "Manifest rejected because of invalid signature" +)] +#[serial] +fn manifest_negotiator_test_encoded_sign_and_cert( + whitelist: &str, + urls: &str, + offer: &str, + signature_b64: Option<&str>, + signature_alg: Option<&str>, + cert: Option<&str>, + error_msg: Option<&str>, +) { + let comp_manifest_b64 = create_comp_manifest_b64(urls); + let signature_b64 = signature_b64.map(|signature| signature.to_string()); + + let cert_b64 = cert.map(cert_file_to_cert_b64); + manifest_negotiator_test_encoded_manifest_sign_and_cert( + whitelist, + comp_manifest_b64, + offer, + signature_b64, + signature_alg, + cert_b64, + error_msg, + ) +} + +fn manifest_negotiator_test_encoded_manifest_sign_and_cert( + whitelist: &str, + comp_manifest_b64: String, + offer: &str, + signature_b64: Option, + signature_alg: Option<&str>, + cert_b64: Option, + error_msg: Option<&str>, +) { + // Having + let whitelist_state = create_whitelist(whitelist); + let (resource_cert_dir, test_cert_dir) = MANIFEST_TEST_RESOURCES.init_cert_dirs(); + + if signature_b64.is_some() { + load_certificates_from_dir( + &resource_cert_dir, + &test_cert_dir, + &["foo_ca-chain.cert.pem"], + ); + } + let keystore = Keystore::load(&test_cert_dir).expect("Can load test certificates"); + + let mut config = create_manifest_signature_validating_policy_config(); + config.domain_patterns = whitelist_state; + config.trusted_keys = Some(keystore); + let mut manifest_negotiator = ManifestSignature::from(config); + + let demand = create_demand_json(&comp_manifest_b64, signature_b64, signature_alg, cert_b64); + let demand: Value = serde_json::from_str(&demand).unwrap(); + let demand = AgreementView { + json: demand, + agreement_id: "id".to_string(), + }; + let offer: Value = serde_json::from_str(offer).unwrap(); + let offer = AgreementView { + json: offer, + agreement_id: "id".to_string(), + }; + + // When + let negotiation_result = manifest_negotiator.negotiate_step(&demand, offer.clone()); + + // Then + let negotiation_result = negotiation_result.expect("Negotiator had not failed"); + if let Some(message) = error_msg { + assert_eq!( + negotiation_result, + NegotiationResult::Reject { + message: message.to_string(), + is_final: true + } + ); + } else { + assert_eq!(negotiation_result, NegotiationResult::Ready { offer }); + } +} + +fn cert_file_to_cert_b64(cert_file: &str) -> String { + let (resource_cert_dir, _) = MANIFEST_TEST_RESOURCES.init_cert_dirs(); + let mut cert_path = resource_cert_dir; + cert_path.push(cert_file); + println!("{cert_path:?}"); + let cert = fs::read(cert_path).expect("Can read cert from resources"); + base64::encode(cert) +} + +fn create_comp_manifest_b64(urls: &str) -> String { + let manifest_template = r#"{ + "version": "0.1.0", + "createdAt": "2022-09-07T02:57:00.000000Z", + "expiresAt": "2100-01-01T00:01:00.000000Z", + "metadata": { "name": "App", "version": "0.1.0" }, + "payload": [], + "compManifest": { + "version": "0.1.0", + "script": { "commands": [], "match": "regex" }, + "net": { + "inet": { + "out": { + "protocols": ["https"], + "urls": __URLS__ + } + } + } + } + }"#; + let manifest = manifest_template.replace("__URLS__", urls); + base64::encode(manifest) +} + +fn create_demand_json( + comp_manifest_b64: &str, + signature_b64: Option, + signature_alg_b64: Option<&str>, + cert_b64: Option, +) -> String { + let mut payload = HashMap::new(); + payload.insert("@tag", json!(comp_manifest_b64)); + if signature_b64.is_some() && signature_alg_b64.is_some() { + payload.insert( + "sig", + json!({ + "@tag": signature_b64.unwrap(), + "algorithm": signature_alg_b64.unwrap().to_string() + }), + ); + } else if signature_b64.is_some() { + payload.insert("sig", json!(signature_b64.unwrap())); + } else if signature_alg_b64.is_some() { + payload.insert( + "sig", + json!({ "algorithm": signature_alg_b64.unwrap().to_string() }), + ); + } + if let Some(cert_b64) = cert_b64 { + payload.insert("cert", json!(cert_b64)); + } + // let mut payload = manifest.to_string(); + let manifest = json!({ + "golem": { + "srv": { + "comp": { + "payload": payload + } + } + }, + }); + let demand = serde_json::to_string_pretty(&manifest).unwrap(); + println!("Tested demand:\n{demand}"); + demand +} + +fn create_manifest_signature_validating_policy_config() -> PolicyConfig { + let mut config = PolicyConfig::default(); + config + .policy_disable_component + .push(Policy::ManifestCompliance); + config + .policy_disable_component + .push(Policy::ManifestInetUrlCompliance); + config + .policy_disable_component + .push(Policy::ManifestScriptCompliance); + config +} + +fn create_whitelist(whitelist_json: &str) -> DomainWhitelistState { + let whitelist = create_whitelist_file(whitelist_json); + let whitelist_patterns = + DomainPatterns::load(&whitelist).expect("Can deserialize whitelist patterns"); + DomainWhitelistState::try_new(whitelist_patterns) + .expect("Can create whitelist state from patterns") +} + +fn create_whitelist_file(whitelist_json: &str) -> PathBuf { + let whitelist_file = whitelist_file(); + if whitelist_file.exists() { + fs::remove_file(&whitelist_file).expect("Can delete whitelist file"); + } + fs::write(whitelist_file.as_path(), whitelist_json).expect("Can write whitelist file"); + whitelist_file +} + +fn whitelist_file() -> PathBuf { + tmp_resource("whitelist.json") +} + +fn tmp_resource(name: &str) -> PathBuf { + let mut resource = PathBuf::from(env!("CARGO_TARGET_TMPDIR")); + resource.push(name); + resource +} diff --git a/agent/provider/tests/whitelist_cli.rs b/agent/provider/tests/whitelist_cli.rs new file mode 100644 index 0000000000..c6baa4b9a4 --- /dev/null +++ b/agent/provider/tests/whitelist_cli.rs @@ -0,0 +1,176 @@ +#[macro_use] +extern crate serial_test; + +use std::fs; +use std::path::PathBuf; + +use assert_cmd::{assert::Assert, Command}; +use serde_json::Value; +use test_case::test_case; + +#[serial] +#[test] +fn empty_whitelist_json() { + clean_data_dir(); + Command::cargo_bin("ya-provider") + .unwrap() + .arg("whitelist") + .arg("list") + .arg("--json") + .args(data_dir_args()) + .assert() + .stdout("[]\n") + .success(); +} + +#[test_case( + &["domain.com"], + "strict", + r#"[{ "ID": "5ce448a6", "Pattern": "domain.com", "Type": "strict" }]"#; + "Adding one pattern" +)] +#[test_case( + &["dom.*\\.com", "myapp.com", "other\\.*"], + "regex", + r#"[ + { "ID": "979b6b99", "Pattern": "dom.*\\.com", "Type": "regex" }, + { "ID": "89dcf5f6", "Pattern": "myapp.com", "Type": "regex" }, + { "ID": "c31deaea", "Pattern": "other\\.*", "Type": "regex" } + ]"#; + "Adding multiple patterns" +)] +#[test_case( + &["domain.com", "domain.com"], + "strict", + r#"[ + { "ID": "5ce448a6", "Pattern": "domain.com", "Type": "strict" } + ]"#; + "Adding duplicated patterns results in one result" +)] +#[serial] +fn whitelist_add_test(add: &[&str], pattern_type: &str, expected_add_json_out: &str) { + clean_data_dir(); + let expected_add_json_out = json_to_printed_output(expected_add_json_out); + whitelist_add(add, pattern_type) + .stdout(expected_add_json_out) + .success(); +} + +#[test_case( + &["domain.com"], + "strict", + &["5ce448a6"], + r#"[{ "ID": "5ce448a6", "Pattern": "domain.com", "Type": "strict" }]"#, + "[]"; + "Adding one pattern, removing it, empty list." +)] +#[test_case( + &["domain.com"], + "strict", + &["5ce448a6", "5ce448a6"], + r#"[{ "ID": "5ce448a6", "Pattern": "domain.com", "Type": "strict" }]"#, + "[]"; + "Adding one pattern, removing with duplicated ids, empty list." +)] +#[test_case( + &["domain.com", "another.com"], + "strict", + &["5ce448a6"], + r#"[{ "ID": "5ce448a6", "Pattern": "domain.com", "Type": "strict" }]"#, + r#"[ { "ID": "ee0cc088", "Pattern": "another.com", "Type": "strict" }]"#; + "Adding two patterns, removing one, one on the list." +)] +#[test_case( + &["domain.com"], + "strict", + &["no_such_id"], + "[]", + r#"[{ "ID": "5ce448a6", "Pattern": "domain.com", "Type": "strict" }]"#; + "Adding one pattern, removing not existing, one on the list." +)] +#[test_case( + &[], + "", + &["no_such_id", "nope"], + "[]", + "[]"; + "Removing on empty list. List on empty produces empty list." +)] +#[serial] +fn whitelist_remove_test( + add: &[&str], + pattern_type: &str, + remove_ids: &[&str], + remove_out: &str, + list_out: &str, +) { + clean_data_dir(); + if !add.is_empty() { + whitelist_add(add, pattern_type).success(); + } + let remove_out = json_to_printed_output(remove_out); + whitelist_remove(remove_ids).stdout(remove_out).success(); + let list_out = json_to_printed_output(list_out); + whitelist_list().stdout(list_out).success(); +} + +// Whitelist test utils + +fn whitelist_add(add: &[&str], pattern_type: &str) -> Assert { + let mut cmd = Command::cargo_bin("ya-provider").unwrap(); + cmd.arg("whitelist") + .arg("add") + .arg("-p") + .args(add) + .arg("-t") + .arg(pattern_type) + .arg("--json") + .args(data_dir_args()) + .assert() +} + +fn whitelist_remove(ids: &[&str]) -> Assert { + let mut cmd = Command::cargo_bin("ya-provider").unwrap(); + cmd.arg("whitelist") + .arg("remove") + .args(ids) + .arg("--json") + .args(data_dir_args()) + .assert() +} + +fn whitelist_list() -> Assert { + let mut cmd = Command::cargo_bin("ya-provider").unwrap(); + cmd.arg("whitelist") + .arg("list") + .arg("--json") + .args(data_dir_args()) + .assert() +} + +fn json_to_printed_output(out: &str) -> String { + let out: Value = serde_json::from_str(out).unwrap(); + let out = serde_json::to_string_pretty(&out).unwrap(); + format!("{out}\n") +} + +fn clean_data_dir() { + let data_dir = data_dir_path(); + if data_dir.exists() { + fs::remove_dir_all(&data_dir).expect("Can delete data dir"); + } + // Without creating dir eprintln breaks tests https://github.com/golemfactory/yagna/blob/master/utils/path/src/data_dir.rs#L23 + fs::create_dir(data_dir).unwrap(); +} + +fn data_dir_args() -> [String; 2] { + let data_dir = data_dir_path(); + let data_dir = data_dir.to_str().unwrap().to_string(); + ["--data-dir".to_string(), data_dir] +} + +fn data_dir_path() -> PathBuf { + let mut data_dir = PathBuf::from(env!("CARGO_TARGET_TMPDIR")); + data_dir.push("data_dir"); + data_dir +} diff --git a/core/activity/Cargo.toml b/core/activity/Cargo.toml index e0c7480be5..86e9b6c9e4 100644 --- a/core/activity/Cargo.toml +++ b/core/activity/Cargo.toml @@ -1,18 +1,18 @@ [package] name = "ya-activity" -version = "0.3.0" +version = "0.4.0" authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-core-model = { version = "^0.6", features = ["activity", "market"] } -ya-client-model = { version = "0.4", features = ["sgx"] } -ya-net = "0.2" -ya-persistence = "0.2" +ya-core-model = { version = "0.8", features = ["activity", "market"] } +ya-client-model = { version = "0.5", features = ["sgx"] } +ya-net = "0.3" +ya-persistence = "0.3" ya-service-api = "0.1" -ya-service-api-interfaces = "0.1" -ya-service-api-web = "0.1" -ya-service-bus = "0.4" +ya-service-api-interfaces = "0.2" +ya-service-api-web = "0.2" +ya-service-bus = "0.6" actix-web = "4" actix-http = "3" @@ -39,6 +39,6 @@ uuid = { version = "0.8", features = ["serde", "v4"] } structopt = "0.3.7" [dev-dependencies] -ya-sb-router = "0.4" +ya-sb-router = "0.6" actix-rt = "2.7" diff --git a/core/activity/src/api.rs b/core/activity/src/api.rs index 22904e946d..82415f66e5 100644 --- a/core/activity/src/api.rs +++ b/core/activity/src/api.rs @@ -9,7 +9,7 @@ use ya_service_api_web::scope::ExtendableScope; pub fn web_scope(db: &DbExecutor, tracker: TrackerRef) -> Scope { actix_web::web::scope(crate::ACTIVITY_API_PATH) .app_data(Data::new(db.clone())) - .app_data(Data::new(tracker.clone())) + .app_data(Data::new(tracker)) .extend(common::extend_web_scope) .extend(crate::provider::extend_web_scope) .extend(crate::requestor::control::extend_web_scope) @@ -21,7 +21,8 @@ mod common { use actix_web::{web, HttpResponse, Responder}; use futures::prelude::*; - use ya_core_model::{activity, NodeId, Role}; + use ya_client_model::market::Role; + use ya_core_model::{activity, NodeId}; use ya_persistence::executor::DbExecutor; use ya_service_api_web::middleware::Identity; use ya_service_bus::{timeout::IntoTimeoutFuture, RpcEndpoint}; @@ -85,7 +86,7 @@ mod common { let state = provider_service .send(activity::GetState { activity_id: path.activity_id.to_string(), - timeout: query.timeout.clone(), + timeout: query.timeout, }) .timeout(timeout_margin(query.timeout)) .await???; @@ -129,7 +130,7 @@ mod common { let usage = provider_service .send(activity::GetUsage { activity_id: path.activity_id.to_string(), - timeout: query.timeout.clone(), + timeout: query.timeout, }) .timeout(timeout_margin(query.timeout)) .await???; @@ -153,10 +154,7 @@ mod common { ); (Ok(web::Bytes::from(line)), Some(stream)) } - Err(err) => ( - Err(actix_web::error::ErrorInternalServerError(err).into()), - None, - ), + Err(err) => (Err(actix_web::error::ErrorInternalServerError(err)), None), }) } else { None diff --git a/core/activity/src/cli.rs b/core/activity/src/cli.rs index f656da7722..6adc1a0c8a 100644 --- a/core/activity/src/cli.rs +++ b/core/activity/src/cli.rs @@ -21,7 +21,7 @@ impl ActivityCli { .await .map_err(anyhow::Error::msg)? .map_err(anyhow::Error::msg)? - .ok_or(anyhow::Error::msg("Identity not found")) + .ok_or_else(|| anyhow::Error::msg("Identity not found")) } pub async fn run_command(self, _ctx: &CliCtx) -> anyhow::Result { @@ -32,9 +32,7 @@ impl ActivityCli { if id.starts_with("0x") { id.parse()? } else { - Self::get_identity(idm::Get::ByAlias(id.into())) - .await? - .node_id + Self::get_identity(idm::Get::ByAlias(id)).await?.node_id } } None => Self::get_identity(idm::Get::ByDefault).await?.node_id, diff --git a/core/activity/src/common.rs b/core/activity/src/common.rs index 8273ec9024..15476543b3 100644 --- a/core/activity/src/common.rs +++ b/core/activity/src/common.rs @@ -7,10 +7,10 @@ use uuid::Uuid; use ya_client_model::{ activity::{ActivityState, ActivityUsage}, - market::Agreement, + market::{Agreement, Role}, NodeId, }; -use ya_core_model::{activity, market, Role}; +use ya_core_model::{activity, market}; use ya_net::RemoteEndpoint; use ya_persistence::executor::DbExecutor; use ya_service_api_web::middleware::Identity; @@ -94,7 +94,7 @@ pub(crate) fn agreement_provider_service( agreement: &Agreement, ) -> Result { Ok(ya_net::from(id.identity) - .to(agreement.provider_id().clone()) + .to(*agreement.provider_id()) .service(activity::BUS_ID)) } @@ -102,7 +102,7 @@ pub(crate) async fn get_persisted_usage( db: &DbExecutor, activity_id: &str, ) -> Result { - Ok(db.as_dao::().get(&activity_id).await?) + Ok(db.as_dao::().get(activity_id).await?) } pub(crate) async fn set_persisted_usage( diff --git a/core/activity/src/dao/activity.rs b/core/activity/src/dao/activity.rs index 7385c0470d..201e4a67a4 100644 --- a/core/activity/src/dao/activity.rs +++ b/core/activity/src/dao/activity.rs @@ -1,6 +1,5 @@ use chrono::Utc; use diesel::prelude::*; -use serde_json; use ya_client_model::activity::{State, StatePair}; use ya_persistence::executor::{do_with_transaction, AsDao, PoolType}; @@ -90,7 +89,7 @@ impl<'c> ActivityDao<'c> { } pub async fn create_if_not_exists(&self, activity_id: &str, agreement_id: &str) -> Result<()> { - if let Err(e) = self.create(&activity_id, &agreement_id).await { + if let Err(e) = self.create(activity_id, agreement_id).await { if !self.exists(activity_id, agreement_id).await? { return Err(e); } diff --git a/core/activity/src/dao/activity_state.rs b/core/activity/src/dao/activity_state.rs index c817fe04a4..e975ff90b5 100644 --- a/core/activity/src/dao/activity_state.rs +++ b/core/activity/src/dao/activity_state.rs @@ -2,7 +2,7 @@ use chrono::Utc; use diesel::expression::dsl::exists; use diesel::prelude::*; use diesel::QueryableByName; -use serde_json; + use std::{convert::TryInto, time::Duration}; use tokio::time::sleep; diff --git a/core/activity/src/dao/activity_usage.rs b/core/activity/src/dao/activity_usage.rs index 29690986c1..8d44509aed 100644 --- a/core/activity/src/dao/activity_usage.rs +++ b/core/activity/src/dao/activity_usage.rs @@ -1,7 +1,7 @@ use chrono::Utc; use diesel::expression::dsl::exists; use diesel::prelude::*; -use serde_json; + use std::convert::TryInto; use ya_client_model::activity::activity_usage::ActivityUsage; diff --git a/core/activity/src/dao/event.rs b/core/activity/src/dao/event.rs index 36be57820b..f98b595932 100644 --- a/core/activity/src/dao/event.rs +++ b/core/activity/src/dao/event.rs @@ -11,6 +11,7 @@ use crate::dao::Result; use crate::db::{models::ActivityEventType, schema}; use ya_client_model::activity::provider_event::ProviderEventType; use ya_client_model::NodeId; +use ya_persistence::types::AdaptTimestamp; pub const MAX_EVENTS: i64 = 100; @@ -71,13 +72,13 @@ impl<'c> EventDao<'c> { let identity_id = identity_id.to_owned(); do_with_transaction(self.pool, move |conn| { - let now = Utc::now().naive_utc(); + let now = Utc::now().adapt(); diesel::insert_into(dsl_event::activity_event) .values( dsl::activity .select(( dsl::id, - identity_id.clone().into_sql::(), + identity_id.into_sql::(), now.into_sql::(), event_type.into_sql::(), requestor_pub_key.into_sql(), @@ -134,7 +135,7 @@ impl<'c> EventDao<'c> { dsl::agreement_id, dsl_event::requestor_pub_key, )) - .filter(dsl_event::event_date.gt(after_timestamp.naive_utc())) + .filter(dsl_event::event_date.gt(after_timestamp.adapt())) .into_boxed(); if let Some(app_sid) = app_session_id { @@ -166,7 +167,7 @@ impl<'c> EventDao<'c> { .get_events(identity_id, app_session_id, after_timestamp, max_events) .await? { - if events.len() > 0 { + if !events.is_empty() { return Ok(events); } } diff --git a/core/activity/src/error.rs b/core/activity/src/error.rs index 77691737cb..85d6ef654e 100644 --- a/core/activity/src/error.rs +++ b/core/activity/src/error.rs @@ -32,7 +32,7 @@ impl From for Error { impl From for actix_web::HttpResponse { fn from(err: Error) -> Self { - err.error_response().into() + err.error_response() } } diff --git a/core/activity/src/provider/mod.rs b/core/activity/src/provider/mod.rs index 745e1b9f43..c64f61a468 100644 --- a/core/activity/src/provider/mod.rs +++ b/core/activity/src/provider/mod.rs @@ -1,9 +1,9 @@ //! Provider side operations use actix_web::{web, Responder}; use ya_client_model::activity::{ActivityState, ProviderEvent}; +use ya_client_model::market::Role; use ya_service_bus::timeout::IntoTimeoutFuture; -use ya_core_model::Role; use ya_persistence::executor::DbExecutor; use ya_service_api_web::middleware::Identity; diff --git a/core/activity/src/provider/service.rs b/core/activity/src/provider/service.rs index a2fcec47ee..8933b09d0b 100644 --- a/core/activity/src/provider/service.rs +++ b/core/activity/src/provider/service.rs @@ -1,3 +1,5 @@ +#![allow(clippy::let_unit_value)] + use chrono::Utc; use futures::future::LocalBoxFuture; use futures::prelude::*; @@ -6,12 +8,11 @@ use std::convert::From; use std::time::Duration; use ya_client_model::activity::{ActivityState, ActivityUsage, State, StatePair}; -use ya_client_model::market::agreement::State as AgreementState; +use ya_client_model::market::{agreement::State as AgreementState, Role}; use ya_client_model::NodeId; use ya_core_model::activity; use ya_core_model::activity::local::Credentials; use ya_core_model::activity::RpcMessageError; -use ya_core_model::Role; use ya_persistence::executor::DbExecutor; use ya_service_bus::{timeout::*, typed::ServiceBinder}; @@ -102,8 +103,6 @@ async fn create_activity_gsb( return Err(RpcMessageError::BadRequest(msg)); } - let provider_id = agreement.provider_id().clone(); - db.as_dao::() .create_if_not_exists(&activity_id, &msg.agreement_id) .await @@ -114,7 +113,7 @@ async fn create_activity_gsb( db.as_dao::() .create( &activity_id, - &provider_id, + agreement.provider_id(), ActivityEventType::CreateActivity, msg.requestor_pub_key, &agreement.app_session_id, @@ -130,7 +129,7 @@ async fn create_activity_gsb( db.clone(), tracker.clone(), &activity_id, - provider_id.clone(), + *agreement.provider_id(), app_session_id.clone(), msg.timeout, ) @@ -140,10 +139,10 @@ async fn create_activity_gsb( db.clone(), tracker.clone(), &activity_id, - provider_id, + *agreement.provider_id(), app_session_id, )); - Error::from(e) + e })?; counter!("activity.provider.created", 1); @@ -170,7 +169,7 @@ async fn activity_credentials( let activity_state = db .as_dao::() .get_state_wait( - &activity_id, + activity_id, vec![State::Initialized.into(), State::Terminated.into()], ) .timeout(timeout) @@ -200,7 +199,7 @@ async fn activity_credentials( let credentials = db .as_dao::() - .get(&activity_id) + .get(activity_id) .await? .map(|c| serde_json::from_str(&c.credentials).map_err(|e| Error::Service(e.to_string()))) .transpose()?; @@ -272,8 +271,8 @@ async fn get_activity_progress( db: &DbExecutor, activity_id: &str, ) -> Result<(ActivityState, ActivityUsage), Error> { - let state = db.as_dao::().get(&activity_id).await?; - let usage = db.as_dao::().get(&activity_id).await?; + let state = db.as_dao::().get(activity_id).await?; + let usage = db.as_dao::().get(activity_id).await?; Ok((state, usage)) } diff --git a/core/activity/src/requestor/control.rs b/core/activity/src/requestor/control.rs index 6d9e5ae57e..c99273ad09 100644 --- a/core/activity/src/requestor/control.rs +++ b/core/activity/src/requestor/control.rs @@ -12,8 +12,8 @@ use ya_client_model::activity::{ ActivityState, CreateActivityRequest, CreateActivityResult, Credentials, ExeScriptCommand, ExeScriptRequest, SgxCredentials, State, }; -use ya_client_model::market::Agreement; -use ya_core_model::{activity, Role}; +use ya_client_model::market::{Agreement, Role}; +use ya_core_model::activity; use ya_net::{self as net, RemoteEndpoint}; use ya_persistence::executor::DbExecutor; use ya_service_api_web::middleware::Identity; @@ -84,16 +84,15 @@ async fn create_activity( let agreement = get_agreement(&agreement_id, Role::Requestor).await?; log::debug!("agreement: {:#?}", agreement); - let provider_id = agreement.provider_id().clone(); let msg = activity::Create { - provider_id, + provider_id: *agreement.provider_id(), agreement_id: agreement_id.to_string(), - timeout: query.timeout.clone(), + timeout: query.timeout, requestor_pub_key: body.pub_key()?, }; let create_resp = net::from(id.identity) - .to(provider_id) + .to(*agreement.provider_id()) .service(activity::BUS_ID) .send(msg) .timeout(timeout_margin(query.timeout)) @@ -101,7 +100,7 @@ async fn create_activity( log::debug!("activity created: {}, inserting", create_resp.activity_id()); db.as_dao::() - .create_if_not_exists(&create_resp.activity_id(), agreement_id) + .create_if_not_exists(create_resp.activity_id(), agreement_id) .await?; let create_result = CreateActivityResult { @@ -136,7 +135,7 @@ async fn destroy_activity( let msg = activity::Destroy { activity_id: path.activity_id.to_string(), agreement_id: agreement.agreement_id.clone(), - timeout: query.timeout.clone(), + timeout: query.timeout, }; agreement_provider_service(&id, &agreement)? .send(msg) @@ -183,11 +182,11 @@ async fn exec( activity_id: path.activity_id.clone(), batch_id: batch_id.clone(), exe_script: commands, - timeout: query.timeout.clone(), + timeout: query.timeout, }; ya_net::from(id.identity) - .to(agreement.provider_id().clone()) + .to(*agreement.provider_id()) .service(&activity::exeunit::bus_id(&path.activity_id)) .send(msg) .timeout(timeout_margin(query.timeout)) @@ -233,8 +232,8 @@ async fn await_results( }; let results = ya_net::from(id.identity) - .to(agreement.provider_id().clone()) - .service(&activity::exeunit::bus_id(&path.activity_id)) + .to(*agreement.provider_id()) + .service_transfer(&activity::exeunit::bus_id(&path.activity_id)) .send(msg) .timeout(timeout_margin(query.timeout)) .await???; @@ -254,8 +253,8 @@ fn stream_results( let seq = AtomicU64::new(0); let stream = ya_net::from(id.identity) - .to(agreement.provider_id().clone()) - .service(&activity::exeunit::bus_id(&path.activity_id)) + .to(*agreement.provider_id()) + .service_transfer(&activity::exeunit::bus_id(&path.activity_id)) .call_streaming(msg) .map(|item| match item { Ok(result) => result.map_err(Error::from), @@ -318,7 +317,7 @@ async fn encrypted( }; let result = ya_net::from(id.identity) - .to(agreement.provider_id().clone()) + .to(*agreement.provider_id()) .service(&activity::exeunit::bus_id(&path.activity_id)) .send(msg) .timeout(query.timeout) diff --git a/core/activity/src/requestor/state.rs b/core/activity/src/requestor/state.rs index a7aa605517..f5a22aa384 100644 --- a/core/activity/src/requestor/state.rs +++ b/core/activity/src/requestor/state.rs @@ -1,6 +1,7 @@ use actix_web::{web, Responder}; -use ya_core_model::{activity, Role}; +use ya_client_model::market::Role; +use ya_core_model::activity; use ya_persistence::executor::DbExecutor; use ya_service_api_web::middleware::Identity; use ya_service_bus::{timeout::IntoTimeoutFuture, RpcEndpoint}; @@ -25,7 +26,7 @@ async fn get_running_command( let agreement = get_activity_agreement(&db, &path.activity_id, Role::Requestor).await?; let msg = activity::GetRunningCommand { activity_id: path.activity_id.to_string(), - timeout: query.timeout.clone(), + timeout: query.timeout, }; let cmd = agreement_provider_service(&id, &agreement)? diff --git a/core/activity/src/tracker.rs b/core/activity/src/tracker.rs index eb1ebf608f..7d945ed4de 100644 --- a/core/activity/src/tracker.rs +++ b/core/activity/src/tracker.rs @@ -92,7 +92,7 @@ impl TrackerRef { &mut self, ) -> anyhow::Result<(TrackingEvent, broadcast::Receiver)> { let (tx, rx) = oneshot::channel(); - if let Ok(_) = self.tx.send(Command::Subscribe { tx }).await { + if (self.tx.send(Command::Subscribe { tx }).await).is_ok() { return Ok(rx.await?); } anyhow::bail!("Fatal error activity state tracker is unavailable"); @@ -209,7 +209,7 @@ pub fn start_tracker() -> (TrackerRef, broadcast::Receiver) { exe_unit_states.emit_state(); } Command::Subscribe { tx } => { - if let Err(_) = tx.send(exe_unit_states.subscribe()) { + if tx.send(exe_unit_states.subscribe()).is_err() { log::debug!("dead subscribe"); } } diff --git a/core/activity/src/tracker/state_manager.rs b/core/activity/src/tracker/state_manager.rs index 4098b7fb2a..9a4751e37a 100644 --- a/core/activity/src/tracker/state_manager.rs +++ b/core/activity/src/tracker/state_manager.rs @@ -101,7 +101,7 @@ impl StateManager { .map(|state| ActivityStateModel { id: String::from(state.activity_id.as_str()), agreement_id: state.agreement_id.clone(), - state: state.last_state.clone(), + state: state.last_state, usage: state.usage(), exe_unit: state.exe_unit.as_ref().map(|v| v.to_string()), provider_id: Some(state.identity_id), @@ -111,9 +111,8 @@ impl StateManager { } pub fn emit_state(&self) { - match self.events.send(self.current_state()) { - Ok(cnt) => log::trace!("send to {} recievers", cnt), - Err(_) => (), + if let Ok(cnt) = self.events.send(self.current_state()) { + log::trace!("send to {} recievers", cnt) } } } diff --git a/core/gftp/Cargo.toml b/core/gftp/Cargo.toml index b588c35fdd..49bef9de4a 100644 --- a/core/gftp/Cargo.toml +++ b/core/gftp/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "gftp" -version = "0.2.5" +version = "0.3.0" authors = ["Golem Factory "] edition = "2018" homepage = "https://github.com/golemfactory/yagna" @@ -19,8 +19,8 @@ required-features=['bin'] [dependencies] ya-compile-time-utils = "0.2" -ya-core-model = { version = "^0.6", features = ["gftp", "identity", "net"] } -ya-service-bus = "0.4" +ya-core-model = { version = "^0.8", features = ["gftp", "identity", "net"] } +ya-service-bus = "0.6" actix-rt = "2.7" anyhow = "1.0" diff --git a/core/gftp/examples/gftp-server.rs b/core/gftp/examples/gftp-server.rs index 0ca7358321..cc4be517b3 100644 --- a/core/gftp/examples/gftp-server.rs +++ b/core/gftp/examples/gftp-server.rs @@ -72,16 +72,17 @@ async fn send( let res = reader.read_message().await?; match res.id { - Some(RpcId::Int(v)) => match v == id { - false => return Err(anyhow!("Invalid response ID: {}, expected {}", v, id)), - _ => (), - }, + Some(RpcId::Int(v)) => { + if v != id { + return Err(anyhow!("Invalid response ID: {}, expected {}", v, id)); + } + } id => return Err(anyhow!("Invalid response ID: {:?}", id)), } match res.body { - RpcBody::Error { error } => return Err(anyhow!("Request {:?} failed: {:?}", id, error)), - RpcBody::Request { .. } => return Err(anyhow!("Unexpected message: {:?}", res)), + RpcBody::Error { error } => Err(anyhow!("Request {:?} failed: {:?}", id, error)), + RpcBody::Request { .. } => Err(anyhow!("Unexpected message: {:?}", res)), RpcBody::Result { result } => Ok(result), } } @@ -106,7 +107,7 @@ async fn main() -> Result<()> { dotenv::dotenv().ok(); std::env::set_var( "RUST_LOG", - std::env::var("RUST_LOG").unwrap_or("info".into()), + std::env::var("RUST_LOG").unwrap_or_else(|_| "info".into()), ); env_logger::init(); diff --git a/core/gftp/src/bin/gftp.rs b/core/gftp/src/bin/gftp.rs index 4da09b6cfe..abd79637df 100644 --- a/core/gftp/src/bin/gftp.rs +++ b/core/gftp/src/bin/gftp.rs @@ -1,7 +1,7 @@ use anyhow::Result; use env_logger::{Builder, Env, Target}; use gftp::rpc::{RpcBody, RpcId, RpcMessage, RpcRequest, RpcResult, RpcStatusResult}; -use std::mem; + use structopt::{clap, StructOpt}; use tokio::io; use tokio::io::AsyncBufReadExt; @@ -116,7 +116,7 @@ async fn server_loop() { 0 => break, _ => match buffer.trim().is_empty() { true => continue, - _ => mem::replace(&mut buffer, String::new()), + _ => std::mem::take(&mut buffer), }, }, Err(error) => { diff --git a/core/gftp/src/gftp.rs b/core/gftp/src/gftp.rs index a91a023224..09bad9080b 100644 --- a/core/gftp/src/gftp.rs +++ b/core/gftp/src/gftp.rs @@ -14,7 +14,7 @@ use url::{quirks::hostname, Position, Url}; use ya_core_model::gftp as model; use ya_core_model::identity; -use ya_core_model::net::TryRemoteEndpoint; +use ya_core_model::net::{RemoteEndpoint, TryRemoteEndpoint}; use ya_core_model::NodeId; use ya_service_bus::{typed as bus, RpcEndpoint}; @@ -102,7 +102,7 @@ pub async fn publish(path: &Path) -> Result { let filedesc = FileDesc::open(path)?; filedesc.bind_handlers(); - Ok(gftp_url(&filedesc.hash).await?) + gftp_url(&filedesc.hash).await } pub async fn close(url: &Url) -> Result { @@ -129,7 +129,7 @@ pub async fn download_from_url(url: &Url, dst_path: &Path) -> Result<()> { } pub async fn download_file(node_id: NodeId, hash: &str, dst_path: &Path) -> Result<()> { - let remote = node_id.try_service(&model::file_bus_id(hash))?; + let remote = node_id.service_transfer(&model::file_bus_id(hash)); log::debug!("Creating target file {}", dst_path.display()); let mut file = create_dest_file(dst_path)?; @@ -176,22 +176,22 @@ pub async fn open_for_upload(filepath: &Path) -> Result { .take(65) .collect::(); - let file = Arc::new(Mutex::new(create_dest_file(&filepath)?)); + let file = Arc::new(Mutex::new(create_dest_file(filepath)?)); let gsb_address = model::file_bus_id(&hash_name); let file_clone = file.clone(); let _ = bus::bind(&gsb_address, move |msg: model::UploadChunk| { let file = file_clone.clone(); - async move { Ok(chunk_uploaded(file.clone(), msg).await?) } + async move { chunk_uploaded(file.clone(), msg).await } }); let file_clone = file.clone(); let _ = bus::bind(&gsb_address, move |msg: model::UploadFinished| { let file = file_clone.clone(); - async move { Ok(upload_finished(file.clone(), msg).await?) } + async move { upload_finished(file.clone(), msg).await } }); - Ok(gftp_url(&hash_name).await?) + gftp_url(&hash_name).await } async fn chunk_uploaded( @@ -320,7 +320,7 @@ fn hash_file_sha256(mut file: &mut fs::File) -> Result { let mut hasher = Sha3_256::new(); file.seek(SeekFrom::Start(0)) - .with_context(|| format!("Can't seek file at offset 0."))?; + .with_context(|| "Can't seek file at offset 0.".to_string())?; io::copy(&mut file, &mut hasher)?; Ok(format!("{:x}", hasher.result())) @@ -337,7 +337,7 @@ pub fn extract_url(url: &Url) -> Result<(NodeId, String)> { ))); } - let node_id = NodeId::from_str(hostname(&url)) + let node_id = NodeId::from_str(hostname(url)) .with_context(|| format!("Url {} has invalid node_id.", url))?; // Note: Remove slash from beginning of path. @@ -368,11 +368,11 @@ fn create_dest_file(file_path: &Path) -> Result { file_path.display() ) })?; - Ok(OpenOptions::new() + OpenOptions::new() .read(true) .write(true) .create(true) .truncate(true) .open(file_path) - .with_context(|| format!("Can't create destination file: [{}].", file_path.display()))?) + .with_context(|| format!("Can't create destination file: [{}].", file_path.display())) } diff --git a/core/identity/Cargo.toml b/core/identity/Cargo.toml index 86692e88c4..aa585bd004 100644 --- a/core/identity/Cargo.toml +++ b/core/identity/Cargo.toml @@ -1,21 +1,22 @@ [package] name = "ya-identity" -version = "0.2.1" +version = "0.3.0" description = "Yagna identity management" authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-client-model = { version = "0.4", features = ["with-diesel"] } -ya-core-model = { version = "^0.6", features = ["identity", "appkey"] } -ya-persistence = "0.2" +ya-client-model = { version = "0.5", features = ["with-diesel"] } +ya-core-model = { version = "^0.8", features = ["identity", "appkey"] } +ya-persistence = "0.3" ya-service-api = "0.1" -ya-service-api-interfaces = "0.1" -ya-service-bus = "0.4" +ya-service-api-interfaces = "0.2" +ya-service-bus = "0.6" anyhow = "1.0" appdirs = "0.2" chrono = { version = "0.4", features = ["serde"] } +ctrlc = "3.2" diesel = { version = "1.4", features = ["sqlite", "r2d2", "chrono"] } diesel_migrations = "1.4" ethsign = "0.8" @@ -32,10 +33,11 @@ thiserror = "1.0" tokio = { version = "1", features = ["fs"] } uuid = { version = "0.8", features = ["v4"] } rustc-hex = "2.1.0" +yansi = "0.5.0" [dev-dependencies] -ya-service-api-derive = "0.1" -ya-sb-router = "0.4" +ya-service-api-derive = "0.2" +ya-sb-router = "0.6" actix-rt = "2.7" actix-service = "2" diff --git a/core/identity/examples/app-key-token.rs b/core/identity/examples/app-key-token.rs index 15bf6b30d6..2489b2dbc7 100644 --- a/core/identity/examples/app-key-token.rs +++ b/core/identity/examples/app-key-token.rs @@ -93,7 +93,7 @@ async fn main() -> anyhow::Result<()> { .json() .await .map_err(|e| anyhow::Error::msg(e.to_string()))?; - eprintln!(""); + eprintln!(); eprintln!("response={}", serde_json::to_string_pretty(&resp)?); } Args::List => { diff --git a/core/identity/examples/appkey_service.rs b/core/identity/examples/appkey_service.rs index 3c1d04e412..93ebaa132d 100644 --- a/core/identity/examples/appkey_service.rs +++ b/core/identity/examples/appkey_service.rs @@ -36,11 +36,11 @@ async fn main() -> anyhow::Result<()> { } Args::ClientAK(cmd) => { let ctx = CliCtx::default(); - ctx.output(cmd.run_command(&ctx).await?); + ctx.output(cmd.run_command(&ctx).await?)?; } Args::ClientID(cmd) => { let ctx = CliCtx::default(); - ctx.output(cmd.run_command(&ctx).await?); + ctx.output(cmd.run_command(&ctx).await?)?; } } Ok(()) diff --git a/core/identity/examples/id-dao-test.rs b/core/identity/examples/id-dao-test.rs index ac10efd25f..88a1750c09 100644 --- a/core/identity/examples/id-dao-test.rs +++ b/core/identity/examples/id-dao-test.rs @@ -1,4 +1,3 @@ -use actix_rt; use chrono::Utc; use ya_identity::dao::{identity::*, init}; diff --git a/core/identity/src/cli/appkey.rs b/core/identity/src/cli/appkey.rs index b5e9d2c7fe..4c29880d78 100644 --- a/core/identity/src/cli/appkey.rs +++ b/core/identity/src/cli/appkey.rs @@ -13,7 +13,7 @@ use ya_service_bus::{typed as bus, RpcEndpoint}; pub enum AppKeyCommand { Create { name: String, - #[structopt(default_value = model::DEFAULT_ROLE, long)] + #[structopt(skip = model::DEFAULT_ROLE)] role: String, #[structopt(long)] id: Option, @@ -40,7 +40,7 @@ impl AppKeyCommand { .await .map_err(anyhow::Error::msg)? .map_err(anyhow::Error::msg)? - .ok_or(anyhow::Error::msg("Identity not found")) + .ok_or_else(|| anyhow::Error::msg("Identity not found")) } pub async fn run_command(&self, _ctx: &CliCtx) -> Result { @@ -63,11 +63,7 @@ impl AppKeyCommand { role: role.clone(), identity, }; - let key = bus::service(model::BUS_ID) - .send(create) - .await - .map_err(anyhow::Error::msg)? - .unwrap(); + let key = bus::service(model::BUS_ID).send(create).await??; Ok(CommandOutput::Object(serde_json::to_value(key)?)) } AppKeyCommand::Drop { name, id } => { @@ -75,7 +71,7 @@ impl AppKeyCommand { name: name.clone(), identity: id.clone(), }; - let _ = bus::service(model::BUS_ID) + bus::service(model::BUS_ID) .send(remove) .await .map_err(anyhow::Error::msg)? @@ -85,8 +81,8 @@ impl AppKeyCommand { AppKeyCommand::List { id, page, per_page } => { let list = model::List { identity: id.clone(), - page: page.clone(), - per_page: per_page.clone(), + page: *page, + per_page: *per_page, }; let result: (Vec, u32) = bus::service(model::BUS_ID) .send(list) diff --git a/core/identity/src/cli/identity.rs b/core/identity/src/cli/identity.rs index cbf8d8c872..9520f0d132 100644 --- a/core/identity/src/cli/identity.rs +++ b/core/identity/src/cli/identity.rs @@ -51,7 +51,7 @@ impl std::str::FromStr for NodeOrAlias { impl NodeOrAlias { async fn resolve(&self) -> anyhow::Result { match self { - NodeOrAlias::Node(node_id) => Ok(node_id.clone()), + NodeOrAlias::Node(node_id) => Ok(*node_id), NodeOrAlias::Alias(alias) => { let id = bus::service(identity::BUS_ID) .send(identity::Get::ByAlias(alias.to_owned())) @@ -76,9 +76,9 @@ impl NodeOrAlias { } } -impl Into for NodeOrAlias { - fn into(self) -> identity::Get { - match self { +impl From for identity::Get { + fn from(na: NodeOrAlias) -> Self { + match na { NodeOrAlias::DefaultNode => identity::Get::ByDefault, NodeOrAlias::Alias(alias) => identity::Get::ByAlias(alias), NodeOrAlias::Node(node_id) => identity::Get::ByNodeId(node_id), @@ -180,7 +180,7 @@ impl IdentityCommand { let mut identities: Vec = bus::service(identity::BUS_ID) .send(identity::List::default()) .await - .map_err(|e| anyhow::Error::msg(e)) + .map_err(anyhow::Error::msg) .context("sending id List to BUS")? .unwrap(); identities.sort_by_key(|id| Reverse((id.is_default, id.alias.clone()))); @@ -211,7 +211,7 @@ impl IdentityCommand { bus::service(identity::BUS_ID) .send(command) .await - .map_err(|e| anyhow::Error::msg(e))?, + .map_err(anyhow::Error::msg)?, ) } IdentityCommand::PubKey { node_or_alias } => { @@ -220,7 +220,7 @@ impl IdentityCommand { bus::service(identity::BUS_ID) .send(identity::GetPubKey(node_id)) .await - .map_err(|e| anyhow::Error::msg(e))? + .map_err(anyhow::Error::msg)? .map(|v| { let key = v.to_hex::(); serde_json::json! {{ "pubKey": key }} @@ -263,7 +263,7 @@ impl IdentityCommand { bus::service(identity::BUS_ID) .send(identity::Sign { node_id, payload }) .await - .map_err(|e| anyhow::Error::msg(e))? + .map_err(anyhow::Error::msg)? .map(|v| { let sig = v.to_hex::(); serde_json::json! {{ "sig": sig }} @@ -283,7 +283,7 @@ impl IdentityCommand { .with_default(*set_default), ) .await - .map_err(|e| anyhow::Error::msg(e))?; + .map_err(anyhow::Error::msg)?; CommandOutput::object(id) } IdentityCommand::Create { @@ -315,7 +315,7 @@ impl IdentityCommand { from_keystore: Some(key_file), }) .await - .map_err(|e| anyhow::Error::msg(e))?; + .map_err(anyhow::Error::msg)?; CommandOutput::object(id) } IdentityCommand::Lock { @@ -324,10 +324,9 @@ impl IdentityCommand { } => { let node_id = node_or_alias.clone().unwrap_or_default().resolve().await?; let password = if *new_password { - let password: String = - rpassword::read_password_from_tty(Some("Password: "))?.into(); + let password: String = rpassword::read_password_from_tty(Some("Password: "))?; let password2: String = - rpassword::read_password_from_tty(Some("Confirm password: "))?.into(); + rpassword::read_password_from_tty(Some("Confirm password: "))?; if password != password2 { anyhow::bail!("Password and confirmation do not match.") } @@ -339,7 +338,7 @@ impl IdentityCommand { bus::service(identity::BUS_ID) .send(identity::Lock::with_id(node_id).with_set_password(password)) .await - .map_err(|e| anyhow::Error::msg(e))?, + .map_err(anyhow::Error::msg)?, ) } IdentityCommand::Unlock { node_or_alias } => { @@ -349,7 +348,7 @@ impl IdentityCommand { bus::service(identity::BUS_ID) .send(identity::Unlock::with_id(node_id, password)) .await - .map_err(|e| anyhow::Error::msg(e))?, + .map_err(anyhow::Error::msg)?, ) } IdentityCommand::Drop { node_or_alias } => { @@ -357,7 +356,7 @@ impl IdentityCommand { let id = bus::service(identity::BUS_ID) .send(command) .await - .map_err(|e| anyhow::Error::msg(e))?; + .map_err(anyhow::Error::msg)?; let id = match id { Ok(Some(v)) => v, Err(e) => return CommandOutput::object(Err::<(), _>(e)), @@ -371,7 +370,7 @@ impl IdentityCommand { bus::service(identity::BUS_ID) .send(identity::DropId::with_id(id.node_id)) .await - .map_err(|e| anyhow::Error::msg(e))?, + .map_err(anyhow::Error::msg)?, ) } IdentityCommand::Export { @@ -382,7 +381,7 @@ impl IdentityCommand { let key_file = bus::service(identity::BUS_ID) .send(identity::GetKeyFile(node_id)) .await? - .map_err(|e| anyhow::Error::msg(e))?; + .map_err(anyhow::Error::msg)?; match file_path { Some(file) => { diff --git a/core/identity/src/dao.rs b/core/identity/src/dao.rs index 7c6eb137f5..926b25229f 100644 --- a/core/identity/src/dao.rs +++ b/core/identity/src/dao.rs @@ -44,25 +44,25 @@ impl From for Error { } macro_rules! into_error { - ($self:ident, $code:expr) => { + ($e:ident, $code:expr) => { model::Error { code: $code, - message: format!("{:?}", $self), + message: format!("{:?}", $e), } }; } -impl Into for Error { - fn into(self) -> model::Error { - match self { - Error::Db(_) => into_error!(self, 500), - Error::Dao(_) => into_error!(self, 500), - Error::Gsb(_) => into_error!(self, 500), - Error::RuntimeError(_) => into_error!(self, 500), - Error::Internal(_) => into_error!(self, 500), - Error::AlreadyExists => into_error!(self, 400), - Error::NotFound => into_error!(self, 404), - Error::Forbidden => into_error!(self, 403), +impl From for model::Error { + fn from(e: Error) -> Self { + match e { + Error::Db(_) => into_error!(e, 500), + Error::Dao(_) => into_error!(e, 500), + Error::Gsb(_) => into_error!(e, 500), + Error::RuntimeError(_) => into_error!(e, 500), + Error::Internal(_) => into_error!(e, 500), + Error::AlreadyExists => into_error!(e, 400), + Error::NotFound => into_error!(e, 404), + Error::Forbidden => into_error!(e, 403), } } } diff --git a/core/identity/src/dao/appkey.rs b/core/identity/src/dao/appkey.rs index e0a28059af..3e5e33c325 100644 --- a/core/identity/src/dao/appkey.rs +++ b/core/identity/src/dao/appkey.rs @@ -27,7 +27,7 @@ impl<'c> AppKeyDao<'c> { where F: Send + 'static + FnOnce(&ConnType) -> Result, { - readonly_transaction(&self.pool, f).await + readonly_transaction(self.pool, f).await } #[inline] @@ -38,7 +38,7 @@ impl<'c> AppKeyDao<'c> { &self, f: F, ) -> Result { - do_with_transaction(&self.pool, f).await + do_with_transaction(self.pool, f).await } pub async fn create( @@ -101,6 +101,21 @@ impl<'c> AppKeyDao<'c> { .await } + pub async fn get_for_name(&self, name: String) -> Result<(AppKey, Role)> { + use crate::db::schema::app_key as app_key_dsl; + use crate::db::schema::role as role_dsl; + + readonly_transaction(self.pool, |conn| { + let result = app_key_dsl::table + .inner_join(role_dsl::table) + .filter(app_key_dsl::name.eq(name)) + .first(conn)?; + + Ok(result) + }) + .await + } + pub async fn list( &self, identity: Option, diff --git a/core/identity/src/id_key.rs b/core/identity/src/id_key.rs index c1482f2600..eba46b87a1 100644 --- a/core/identity/src/id_key.rs +++ b/core/identity/src/id_key.rs @@ -155,7 +155,7 @@ fn key_file_from_secret(secret: &SecretKey, password: Protected) -> KeyFile { pub fn generate_new_keyfile(password: Protected) -> anyhow::Result { let (key_file, _) = generate_new_secret(password); - Ok(serde_json::to_string(&key_file).context("serialize keyfile")?) + serde_json::to_string(&key_file).context("serialize keyfile") } #[cfg(test)] diff --git a/core/identity/src/service.rs b/core/identity/src/service.rs index ca3a2e9541..15941cd3d3 100644 --- a/core/identity/src/service.rs +++ b/core/identity/src/service.rs @@ -25,6 +25,8 @@ impl Identity { )); identity::IdentityService::bind_service(service); + identity::wait_for_default_account_unlock().await?; + appkey::activate(&db).await?; Ok(()) } diff --git a/core/identity/src/service/appkey.rs b/core/identity/src/service/appkey.rs index f0dd934e0e..a313391e09 100644 --- a/core/identity/src/service/appkey.rs +++ b/core/identity/src/service/appkey.rs @@ -53,33 +53,48 @@ pub async fn activate(db: &DbExecutor) -> anyhow::Result<()> { { let subscription = subscription.clone(); tokio::task::spawn_local(async move { - let _ = rx - .for_each(|event| send_events(subscription.borrow(), event)) + rx.for_each(|event| send_events(subscription.borrow(), event)) .await; }); } - let _ = bus::bind(&model::BUS_ID, move |s: model::Subscribe| { + let _ = bus::bind(model::BUS_ID, move |s: model::Subscribe| { let id = subscription.borrow_mut().subscribe(s.endpoint); future::ok(id) }); - let create_tx = tx.clone(); + let create_tx = tx; // Create a new application key entry - let _ = bus::bind(&model::BUS_ID, move |create: model::Create| { + let _ = bus::bind(model::BUS_ID, move |create: model::Create| { let key = Uuid::new_v4().to_simple().to_string(); let db = dbx.clone(); let mut create_tx = create_tx.clone(); - let identity = create.identity.clone(); async move { - let result = db - .as_dao::() - .create(key.clone(), create.name, create.role, create.identity) - .await - .map_err(|e| model::Error::internal(e)) - .map(|_| key)?; + let dao = db.as_dao::(); + + let result = match dao.get_for_name(create.name.clone()).await { + Ok((app_key, _)) => { + if app_key.identity_id == create.identity { + Ok(app_key.key) + } else { + Err(model::Error::bad_request(format!( + "app-key with name {} already defined with identity {}", + app_key.name, app_key.identity_id + ))) + } + } + Err(crate::dao::Error::Dao(diesel::result::Error::NotFound)) => dao + .create(key.clone(), create.name, create.role, create.identity) + .await + .map_err(model::Error::internal) + .map(|_| key), + Err(e) => Err(model::Error::internal(e)), + }?; + let _ = create_tx - .send(model::event::Event::NewKey { identity }) + .send(model::event::Event::NewKey { + identity: create.identity, + }) .await; Ok(result) } @@ -90,7 +105,7 @@ pub async fn activate(db: &DbExecutor) -> anyhow::Result<()> { let preconfigured_node_id = crate::autoconf::preconfigured_node_id()?; let start_datetime = Utc::now().naive_utc(); // Retrieve an application key entry based on the key itself - let _ = bus::bind(&model::BUS_ID, move |get: model::Get| { + let _ = bus::bind(model::BUS_ID, move |get: model::Get| { let db = dbx.clone(); let preconfigured_appkey = preconfigured_appkey.clone(); async move { @@ -112,7 +127,7 @@ pub async fn activate(db: &DbExecutor) -> anyhow::Result<()> { key: get.key.clone(), role: model::DEFAULT_ROLE.to_string(), identity: node_id, - created_date: start_datetime.clone(), + created_date: start_datetime, }) } else { let (appkey, role) = db @@ -141,7 +156,7 @@ pub async fn activate(db: &DbExecutor) -> anyhow::Result<()> { .as_dao::() .list(list.identity, list.page, list.per_page) .await - .map_err(Into::into)?; + .map_err(Into::::into)?; let keys = result .0 @@ -160,13 +175,13 @@ pub async fn activate(db: &DbExecutor) -> anyhow::Result<()> { }); let dbx = db.clone(); - let _ = bus::bind(&model::BUS_ID, move |rm: model::Remove| { + let _ = bus::bind(model::BUS_ID, move |rm: model::Remove| { let db = dbx.clone(); async move { db.as_dao::() .remove(rm.name, rm.identity) .await - .map_err(Into::into)?; + .map_err(Into::::into)?; Ok(()) } }); diff --git a/core/identity/src/service/identity.rs b/core/identity/src/service/identity.rs index 3a563859bd..d715e8feb5 100644 --- a/core/identity/src/service/identity.rs +++ b/core/identity/src/service/identity.rs @@ -4,13 +4,14 @@ use std::convert::{TryFrom, TryInto}; use std::rc::Rc; use std::sync::Arc; +use anyhow::bail; use chrono::Utc; use ethsign::{KeyFile, Protected, PublicKey}; use futures::lock::Mutex; use futures::prelude::*; use ya_client_model::NodeId; -use ya_service_bus::typed as bus; +use ya_service_bus::{typed as bus, RpcEndpoint, RpcMessage}; use ya_core_model::identity as model; use ya_persistence::executor::DbExecutor; @@ -28,6 +29,10 @@ impl Subscription { fn subscribe(&mut self, endpoint: String) { self.subscriptions.push(endpoint); } + + fn unsubscribe(&mut self, endpoint: String) { + self.subscriptions.retain(|s| s != &endpoint); + } } pub struct IdentityService { @@ -78,7 +83,7 @@ impl IdentityService { { let subscription = subscription.clone(); tokio::task::spawn_local(async move { - let _ = receiver + receiver .for_each(|event| send_event(subscription.borrow(), event)) .await; }); @@ -102,11 +107,11 @@ impl IdentityService { db.as_dao::() .init_default_key(|| { log::info!("generating new default identity"); - let key: IdentityKey = generate_new(None, "".into()).into(); + let key: IdentityKey = generate_new(None, "".into()); Ok(Identity { identity_id: key.id(), - key_file_json: key.to_key_file().map_err(|e| DaoError::internal(e))?, + key_file_json: key.to_key_file().map_err(DaoError::internal)?, is_default: true, is_deleted: false, alias: None, @@ -154,7 +159,7 @@ impl IdentityService { None => return Ok(None), Some(id) => id, }; - Ok(Some(to_info(&self.default_key, &id))) + Ok(Some(to_info(&self.default_key, id))) } pub fn get_by_id(&self, node_id: &NodeId) -> Result, model::Error> { @@ -162,7 +167,7 @@ impl IdentityService { None => return Ok(None), Some(id) => id, }; - Ok(Some(to_info(&self.default_key, &id))) + Ok(Some(to_info(&self.default_key, id))) } pub fn get_default_id(&self) -> Result, model::Error> { @@ -170,7 +175,7 @@ impl IdentityService { None => return Ok(None), Some(id) => id, }; - Ok(Some(to_info(&self.default_key, &id))) + Ok(Some(to_info(&self.default_key, id))) } pub fn list_ids(&self) -> Result, model::Error> { @@ -252,7 +257,7 @@ impl IdentityService { fn get_key_by_id(&mut self, node_id: &NodeId) -> Result<&mut IdentityKey, model::Error> { Ok(match self.ids.get_mut(node_id) { Some(v) => v, - None => return Err(model::Error::NodeNotFound(Box::new(node_id.clone()))), + None => return Err(model::Error::NodeNotFound(Box::new(*node_id))), }) } @@ -289,9 +294,11 @@ impl IdentityService { ) -> Result { let default_key = self.default_key; let key = self.get_key_by_id(&node_id)?; - key.unlock(password).map_err(model::Error::new_err_msg)?; - let output = to_info(&default_key, key); - Ok(output) + if key.unlock(password).map_err(model::Error::new_err_msg)? { + Ok(to_info(&default_key, key)) + } else { + Err(model::Error::InvalidPassword) + } } pub async fn sign(&mut self, node_id: NodeId, data: Vec) -> Result, model::Error> { @@ -310,7 +317,7 @@ impl IdentityService { let node_id = update.node_id; let key = match self.ids.get_mut(&node_id) { Some(v) => v, - None => return Err(model::Error::NodeNotFound(Box::new(node_id.clone()))), + None => return Err(model::Error::NodeNotFound(Box::new(node_id))), }; let update_alias = update.alias.clone(); if let Some(new_alias) = update.alias { @@ -367,12 +374,22 @@ impl IdentityService { Ok(model::Ack {}) } + pub async fn unsubscribe( + &mut self, + unsubscribe: model::Unsubscribe, + ) -> Result { + self.subscription + .borrow_mut() + .unsubscribe(unsubscribe.endpoint); + Ok(model::Ack {}) + } + pub async fn get_pub_key( &mut self, key_id: model::GetPubKey, ) -> Result { let key = self.get_key_by_id(&key_id.0)?; - key.to_pub_key().map_err(|e| model::Error::new_err_msg(e)) + key.to_pub_key().map_err(model::Error::new_err_msg) } pub async fn get_key_file( @@ -380,7 +397,7 @@ impl IdentityService { key_id: model::GetKeyFile, ) -> Result { let key = self.get_key_by_id(&key_id.0)?; - key.to_key_file().map_err(|e| model::Error::new_err_msg(e)) + key.to_key_file().map_err(model::Error::new_err_msg) } pub fn bind_service(me: Arc>) { @@ -487,6 +504,11 @@ impl IdentityService { async move { this.lock().await.subscribe(subscribe).await } }); let this = me.clone(); + let _ = bus::bind(model::BUS_ID, move |unsubscribe: model::Unsubscribe| { + let this = this.clone(); + async move { this.lock().await.unsubscribe(unsubscribe).await } + }); + let this = me.clone(); let _ = bus::bind(model::BUS_ID, move |node_id: model::GetPubKey| { let this = this.clone(); async move { @@ -497,10 +519,95 @@ impl IdentityService { .map(|key| key.bytes().to_vec()) } }); - let this = me.clone(); + let this = me; let _ = bus::bind(model::BUS_ID, move |node_id: model::GetKeyFile| { let this = this.clone(); async move { this.lock().await.get_key_file(node_id).await } }); } } + +pub async fn wait_for_default_account_unlock() -> anyhow::Result<()> { + let identity_key = get_default_identity_key().await?; + + if identity_key.is_locked { + let locked_identity = identity_key.node_id; + let (tx, rx) = futures::channel::mpsc::unbounded(); + let endpoint = format!("{}/await_unlock", model::BUS_ID); + + let _ = bus::bind(&endpoint, move |e: model::event::Event| { + let mut tx_clone = tx.clone(); + async move { + match e { + model::event::Event::AccountLocked { .. } => {} + model::event::Event::AccountUnlocked { identity } => { + if locked_identity == identity { + log::debug!("Got unlocked event for default locked account with nodeId: {locked_identity}"); + tx_clone.send(()).await.expect("Receiver is closed"); + } + } + }; + Ok(()) + } + }); + subscribe(endpoint.clone()).await?; + + log::info!("{}", yansi::Color::RGB(0xFF, 0xA5, 0x00).paint( + "Daemon cannot start because default account is locked. Unlock it by running 'yagna id unlock'" + )); + + wait_for_unlock(rx).await?; + + unsubscribe(endpoint.clone()).await?; + unbind(endpoint).await?; + } + + Ok(()) +} + +async fn wait_for_unlock( + mut rx: futures::channel::mpsc::UnboundedReceiver<()>, +) -> anyhow::Result<()> { + // Check lock second time because user could unlocked database before subscription + if get_default_identity_key().await?.is_locked { + tokio::select! { + _ = rx.next() => { + log::info!("Default account unlocked"); + } + _ = tokio::signal::ctrl_c() => { + bail!("Default account is locked"); + } + }; + } + + Ok(()) +} + +async fn subscribe(endpoint: String) -> anyhow::Result<()> { + bus::service(model::BUS_ID) + .send(model::Subscribe { endpoint }) + .await??; + + Ok(()) +} + +async fn unsubscribe(endpoint: String) -> anyhow::Result<()> { + bus::service(model::BUS_ID) + .send(model::Unsubscribe { endpoint }) + .await??; + + Ok(()) +} + +async fn unbind(endpoint: String) -> anyhow::Result<()> { + bus::unbind(&format!("{}/{}", endpoint.clone(), model::event::Event::ID)).await?; + + Ok(()) +} + +async fn get_default_identity_key() -> anyhow::Result { + bus::service(model::BUS_ID) + .send(model::Get::ByDefault {}) + .await?? + .ok_or_else(|| anyhow::anyhow!("No default Identity found")) +} diff --git a/core/market/Cargo.toml b/core/market/Cargo.toml index 6b2d813bde..a48457de34 100644 --- a/core/market/Cargo.toml +++ b/core/market/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ya-market" -version = "0.3.0" +version = "0.4.0" description = "The Distributed Marketplace implementation for Yagna." authors = ["Golem Factory "] edition = "2018" @@ -10,19 +10,19 @@ test-suite = [] bcast-singleton = [] [dependencies] -ya-agreement-utils = { version = "^0.3" } -ya-client = "0.6" -ya-core-model = { version = "^0.6", features = ["market", "net"] } +ya-agreement-utils = { version = "0.4" } +ya-client = "0.7" +ya-core-model = { version = "^0.8", features = ["market", "net"] } ya-diesel-utils = { version = "0.1" } ya-market-resolver = "0.2" -ya-net = "0.2" -ya-persistence = "0.2" +ya-net = "0.3" +ya-persistence = "0.3" ya-service-api = "0.1" -ya-service-api-interfaces = "0.1" -ya-service-api-web = "0.1" -ya-service-bus = "0.4" +ya-service-api-interfaces = "0.2" +ya-service-api-web = "0.2" +ya-service-bus = "0.6" ya-std-utils = "0.1" -ya-utils-actix = "0.1" +ya-utils-actix = "0.2" actix = { version = "0.13", default-features = false } actix-http = "3" diff --git a/core/market/resolver/Cargo.toml b/core/market/resolver/Cargo.toml index 91e226891c..2e0524c38f 100644 --- a/core/market/resolver/Cargo.toml +++ b/core/market/resolver/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-agreement-utils = "0.2" +ya-agreement-utils = "0.4" asnom = { git = "https://github.com/dequbed/asnom.git" } bigdecimal = "0.2" diff --git a/core/market/resolver/src/lib.rs b/core/market/resolver/src/lib.rs index 00b8ce128a..5bf03c004b 100644 --- a/core/market/resolver/src/lib.rs +++ b/core/market/resolver/src/lib.rs @@ -12,7 +12,7 @@ use resolver::error::PrepareError; pub use resolver::matching::{match_weak, MatchResult}; pub use resolver::prepare::{PreparedDemand, PreparedOffer}; -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Eq)] pub enum Match { Yes, No { @@ -60,7 +60,7 @@ pub fn match_demand_offer( } } -fn extract_names(props_vec: &Vec<&PropertyRef>) -> Vec { +fn extract_names(props_vec: &[&PropertyRef]) -> Vec { props_vec .iter() .map(|prop| match prop { diff --git a/core/market/resolver/src/resolver/error.rs b/core/market/resolver/src/resolver/error.rs index a6a589acce..166ae2d418 100644 --- a/core/market/resolver/src/resolver/error.rs +++ b/core/market/resolver/src/resolver/error.rs @@ -3,7 +3,7 @@ use std::fmt; // #region ParseError -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct ParseError { msg: String, } @@ -37,7 +37,7 @@ impl error::Error for ParseError { // #region ResolveError -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct ResolveError { pub msg: String, } @@ -71,7 +71,7 @@ impl error::Error for ResolveError { // #region ExpressionError -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct ExpressionError { msg: String, } @@ -105,7 +105,7 @@ impl error::Error for ExpressionError { // #region PrepareError -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct PrepareError { msg: String, } @@ -139,7 +139,7 @@ impl error::Error for PrepareError { // #region MatchError -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct MatchError { msg: String, } diff --git a/core/market/resolver/src/resolver/expression.rs b/core/market/resolver/src/resolver/expression.rs index 4a08a61119..329dcea540 100644 --- a/core/market/resolver/src/resolver/expression.rs +++ b/core/market/resolver/src/resolver/expression.rs @@ -24,8 +24,8 @@ pub enum Expression { Less(PropertyRef, String), // property ref, value LessEqual(PropertyRef, String), // property ref, value Present(PropertyRef), // property ref - Or(Vec>), // operands - And(Vec>), // operands + Or(Vec), // operands + And(Vec), // operands Not(Box), // operand Empty(bool), // empty expression of specific logical value (true/false) } @@ -45,7 +45,7 @@ impl Expression { } // Resolve the expression to bool value if possible (ie. if an expression has a known boolean value) - pub fn to_value<'a>(&'a self) -> Option { + pub fn to_value(&self) -> Option { match self { Expression::Empty(val) => Some(*val), _ => None, @@ -231,7 +231,7 @@ impl Expression { fn resolve_and<'a>( &'a self, - seq: &'a Vec>, + seq: &'a Vec, property_set: &'a PropertySet, ) -> ResolveResult { let mut undefined_found = false; @@ -245,7 +245,7 @@ impl Expression { match unresolved_expr { Expression::Empty(_) => {} _ => { - unresolved_exprs.push(Box::new(unresolved_expr)); + unresolved_exprs.push(unresolved_expr); } }; return ResolveResult::False(vec![], Expression::Empty(false)); @@ -256,7 +256,7 @@ impl Expression { match unresolved_expr { Expression::Empty(_) => {} _ => { - unresolved_exprs.push(Box::new(unresolved_expr)); + unresolved_exprs.push(unresolved_expr); } }; undefined_found = true; @@ -268,12 +268,10 @@ impl Expression { if undefined_found { ResolveResult::Undefined( unresolved_refs, - if unresolved_exprs.len() > 1 { - Expression::And(unresolved_exprs) - } else if unresolved_exprs.len() == 1 { - *unresolved_exprs.pop().unwrap() - } else { - Expression::Empty(true) + match unresolved_exprs.len().cmp(&1) { + std::cmp::Ordering::Greater => Expression::And(unresolved_exprs), + std::cmp::Ordering::Equal => unresolved_exprs.pop().unwrap(), + std::cmp::Ordering::Less => Expression::Empty(true), }, ) } else { @@ -283,7 +281,7 @@ impl Expression { fn resolve_or<'a>( &'a self, - seq: &'a Vec>, + seq: &'a Vec, property_set: &'a PropertySet, ) -> ResolveResult { let mut undefined_found = false; @@ -299,7 +297,7 @@ impl Expression { match unresolved_expr { Expression::Empty(_) => {} _ => { - unresolved_exprs.push(Box::new(unresolved_expr)); + unresolved_exprs.push(unresolved_expr); } }; } @@ -308,7 +306,7 @@ impl Expression { match unresolved_expr { Expression::Empty(_) => {} _ => { - unresolved_exprs.push(Box::new(unresolved_expr)); + unresolved_exprs.push(unresolved_expr); } }; undefined_found = true; @@ -321,23 +319,19 @@ impl Expression { if undefined_found { ResolveResult::Undefined( all_un_props, - if unresolved_exprs.len() > 1 { - Expression::Or(unresolved_exprs) - } else if unresolved_exprs.len() == 1 { - *unresolved_exprs.pop().unwrap() - } else { - Expression::Empty(true) + match unresolved_exprs.len().cmp(&1) { + std::cmp::Ordering::Greater => Expression::Or(unresolved_exprs), + std::cmp::Ordering::Equal => unresolved_exprs.pop().unwrap(), + std::cmp::Ordering::Less => Expression::Empty(true), }, ) } else { ResolveResult::False( all_un_props, - if unresolved_exprs.len() > 1 { - Expression::Or(unresolved_exprs) - } else if unresolved_exprs.len() == 1 { - *unresolved_exprs.pop().unwrap() - } else { - Expression::Empty(false) + match unresolved_exprs.len().cmp(&1) { + std::cmp::Ordering::Greater => Expression::Or(unresolved_exprs), + std::cmp::Ordering::Equal => unresolved_exprs.pop().unwrap(), + std::cmp::Ordering::Less => Expression::Empty(false), }, ) } @@ -401,7 +395,7 @@ pub fn build_expression(root: &Tag) -> Result { Tag::ExplicitTag(exp_tag) => build_expression_from_explicit_tag(exp_tag), Tag::OctetString(oct_string) => build_expression_from_octet_string(oct_string), Tag::Null(_) => Ok(Expression::Empty(true)), - _ => Err(ExpressionError::new(&format!("Unexpected tag type"))), + _ => Err(ExpressionError::new("Unexpected tag type")), } } @@ -452,7 +446,7 @@ fn build_multi_expression( for tag in sequence { match build_expression(tag) { Ok(expr) => { - expr_vec.push(Box::new(expr)); + expr_vec.push(expr); } Err(err) => return Err(err), } @@ -506,7 +500,7 @@ fn build_simple_expression( } } -fn extract_str_from_octet_string<'a>(tag: &'a Tag) -> Result<&'a str, ExpressionError> { +fn extract_str_from_octet_string(tag: &Tag) -> Result<&str, ExpressionError> { match tag { Tag::OctetString(oct) => match str::from_utf8(&oct.inner) { Ok(s) => Ok(s), @@ -522,22 +516,8 @@ fn extract_two_octet_strings<'a>( sequence: &'a Vec, ) -> Result<(&'a str, &'a str), ExpressionError> { if sequence.len() >= 2 { - let attr: &'a str; - let val: &'a str; - - match extract_str_from_octet_string(&sequence[0]) { - Ok(s) => { - attr = s; - } - Err(err) => return Err(err), - } - - match extract_str_from_octet_string(&sequence[1]) { - Ok(s) => { - val = s; - } - Err(err) => return Err(err), - } + let attr: &'a str = extract_str_from_octet_string(&sequence[0])?; + let val: &'a str = extract_str_from_octet_string(&sequence[1])?; Ok((attr, val)) } else { diff --git a/core/market/resolver/src/resolver/ldap_parser.rs b/core/market/resolver/src/resolver/ldap_parser.rs index 97ab6b5995..47c03ba0b3 100644 --- a/core/market/resolver/src/resolver/ldap_parser.rs +++ b/core/market/resolver/src/resolver/ldap_parser.rs @@ -23,7 +23,7 @@ pub const TAG_LESS_EQUAL: u64 = 11; pub fn parse(input: &str) -> Result { match filter(input.as_bytes()) { IResult::Done(_, t) => Ok(t), - IResult::Error(error_kind) => Err(format!("Parsing error: {}", error_kind.to_string())), + IResult::Error(error_kind) => Err(format!("Parsing error: {}", error_kind)), IResult::Incomplete(needed) => Err(format!("Incomplete expression: {:?}", needed)), } } @@ -142,5 +142,5 @@ named!( ); pub fn is_delimiter(chr: u8) -> bool { - chr == '=' as u8 || chr == '<' as u8 || chr == '>' as u8 || chr == '~' as u8 + chr == b'=' || chr == b'<' || chr == b'>' || chr == b'~' } diff --git a/core/market/resolver/src/resolver/matching.rs b/core/market/resolver/src/resolver/matching.rs index 8a964c8567..1e4cce3861 100644 --- a/core/market/resolver/src/resolver/matching.rs +++ b/core/market/resolver/src/resolver/matching.rs @@ -90,11 +90,9 @@ pub fn match_weak<'a>( (un_props1, result1_unres_expr), (un_props2, result2_unres_expr), )) + } else if result1_binary && result2_binary { + Ok(MatchResult::True) } else { - if result1_binary == true && result2_binary == true { - Ok(MatchResult::True) - } else { - Ok(MatchResult::False(un_props1, un_props2)) - } + Ok(MatchResult::False(un_props1, un_props2)) } } diff --git a/core/market/resolver/src/resolver/prop_parser.rs b/core/market/resolver/src/resolver/prop_parser.rs index 1bd5a20bd2..8f223fd3c6 100644 --- a/core/market/resolver/src/resolver/prop_parser.rs +++ b/core/market/resolver/src/resolver/prop_parser.rs @@ -203,7 +203,7 @@ named!( map!(separated_list!(tag!(","), val_literal), |v: Vec< Literal<'a>, >| { - Literal::List(v.into_iter().map(|item| Box::new(item)).collect()) + Literal::List(v.into_iter().map(Box::new).collect()) }), char!(']') )) @@ -306,7 +306,7 @@ pub fn parse_prop_def(input: &str) -> Result<(&str, Option<&str>), String> { match iresult { IResult::Done(rest, t) => Ok((t, Some(rest))), - IResult::Error(error_kind) => Err(format!("Parsing error: {}", error_kind.to_string())), + IResult::Error(error_kind) => Err(format!("Parsing error: {}", error_kind)), IResult::Incomplete(_needed) => Ok((input, None)), } } @@ -321,7 +321,7 @@ pub fn parse_prop_ref_with_aspect( ) -> Result<(&str, Option<&str>, Option<&str>), String> { match prop_ref_aspect_type(input) { IResult::Done(rest, t) => { - if rest == "" { + if rest.is_empty() { Ok(t) } else { Err(format!( @@ -334,7 +334,7 @@ pub fn parse_prop_ref_with_aspect( // no type, try parsing ref with aspect alone match prop_ref_aspect(input) { IResult::Done(rest, t) => { - if rest == "" { + if rest.is_empty() { Ok(t) } else { Err(format!( @@ -357,7 +357,7 @@ pub fn parse_prop_ref_with_aspect( fn parse_prop_ref_no_aspect(input: &str) -> Result<(&str, Option<&str>, Option<&str>), String> { match prop_ref_type(input) { IResult::Done(rest, t) => { - if rest == "" { + if rest.is_empty() { Ok(t) } else { Err(format!( @@ -368,7 +368,7 @@ fn parse_prop_ref_no_aspect(input: &str) -> Result<(&str, Option<&str>, Option<& } IResult::Incomplete(_) | IResult::Error(_) => match prop_ref_no_type(input) { IResult::Done(rest, t) => { - if rest == "" { + if rest.is_empty() { Ok(t) } else { Err(format!( @@ -391,13 +391,13 @@ fn parse_prop_ref_no_aspect(input: &str) -> Result<(&str, Option<&str>, Option<& pub fn parse_prop_ref_as_list(input: &str) -> Result, String> { match prop_ref_list(input) { IResult::Done(rest, t) => { - if rest == "" { + if rest.is_empty() { Ok(t) } else { Err(format!("Parsing list error: unexpected text {}", rest)) } } - IResult::Error(error_kind) => Err(format!("Parsing error: {}", error_kind.to_string())), + IResult::Error(error_kind) => Err(format!("Parsing error: {}", error_kind)), IResult::Incomplete(needed) => Err(format!("Incomplete expression: {:?}", needed)), } } @@ -414,17 +414,15 @@ pub fn parse_prop_value_literal(input: &str) -> Result { match iresult { IResult::Done(rest, t) => { - if rest.len() == 0 { + if rest.is_empty() { Ok(t) } else { Err(format!("Unknown literal type: {}", input)) } } - IResult::Error(error_kind) => Err(format!( - "Parsing error: {} in text '{}'", - error_kind.to_string(), - input - )), + IResult::Error(error_kind) => { + Err(format!("Parsing error: {} in text '{}'", error_kind, input)) + } IResult::Incomplete(_needed) => Err(format!("Parsing error: {:?}", _needed)), } } diff --git a/core/market/resolver/src/resolver/properties.rs b/core/market/resolver/src/resolver/properties.rs index b61e7c7304..8533648b30 100644 --- a/core/market/resolver/src/resolver/properties.rs +++ b/core/market/resolver/src/resolver/properties.rs @@ -163,8 +163,8 @@ impl<'a> PropertyValue<'a> { // Note: Only str1 may contain wildcard // TODO my be sensible to move the Regex building to the point where property is parsed... fn str_equal_with_wildcard(str1: &str, str2: &str) -> bool { - if str1.contains("*") { - let regex_text = format!("^{}$", str1.replace("*", ".*")); + if str1.contains('*') { + let regex_text = format!("^{}$", str1.replace('*', ".*")); match Regex::new(®ex_text) { Ok(regex) => regex.is_match(str2), Err(_error) => false, @@ -285,14 +285,8 @@ impl<'a> PropertyValue<'a> { // ...then check if all results are successful. for item in results.iter() { - match item { - Err(error) => { - return Err(ParseError::new(&format!( - "Error parsing list: '{}'", - error - ))); - } - _ => {} + if let Err(error) = item { + return Err(ParseError::new(&format!("Error parsing list: '{}'", error))); } } @@ -413,20 +407,17 @@ impl<'a> PropertySet<'a> { aspect_name: &'a str, aspect_value: &'a str, ) { - match self.properties.remove(prop_name) { - Some(prop) => { - let new_prop = match prop { - Property::Explicit(name, val, mut aspects) => { - // remove aspect if already exists - aspects.remove(aspect_name); - aspects.insert(aspect_name, aspect_value); - Property::Explicit(name, val, aspects) - } - _ => unreachable!(), - }; - self.properties.insert(prop_name, new_prop); - } - None => {} + if let Some(prop) = self.properties.remove(prop_name) { + let new_prop = match prop { + Property::Explicit(name, val, mut aspects) => { + // remove aspect if already exists + aspects.remove(aspect_name); + aspects.insert(aspect_name, aspect_value); + Property::Explicit(name, val, aspects) + } + _ => unreachable!(), + }; + self.properties.insert(prop_name, new_prop); } } } @@ -434,13 +425,13 @@ impl<'a> PropertySet<'a> { // #endregion // Property reference (element of filter expression) -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub enum PropertyRef { Value(String, PropertyRefType), // reference to property value (prop name) Aspect(String, String, PropertyRefType), // reference to property aspect (prop name, aspect name) } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub enum PropertyRefType { Any, Decimal, diff --git a/core/market/resolver/tests/expression_build.rs b/core/market/resolver/tests/expression_build.rs index 265255049b..94778fb8c1 100644 --- a/core/market/resolver/tests/expression_build.rs +++ b/core/market/resolver/tests/expression_build.rs @@ -7,23 +7,15 @@ use ya_market_resolver::*; #[test] fn prepare_offer_error_for_empty() { let demand = Demand::default(); - match PreparedDemand::from(&demand) { - Err(_) => {} - _ => { - assert!(false); - } - } + + assert!(PreparedDemand::from(&demand).is_err()); } #[test] fn prepare_demand_error_for_empty() { let offer = Offer::default(); - match PreparedOffer::from(&offer) { - Err(_) => {} - _ => { - assert!(false); - } - } + + assert!(PreparedOffer::from(&offer).is_err()); } #[test] @@ -76,18 +68,18 @@ fn build_expression_and() { let f = "(&(a=b)(b=c)(c=d))"; let expression = Expression::And(vec![ - Box::new(Expression::Equals( + Expression::Equals( PropertyRef::Value(String::from("a"), PropertyRefType::Any), String::from("b"), - )), - Box::new(Expression::Equals( + ), + Expression::Equals( PropertyRef::Value(String::from("b"), PropertyRefType::Any), String::from("c"), - )), - Box::new(Expression::Equals( + ), + Expression::Equals( PropertyRef::Value(String::from("c"), PropertyRefType::Any), String::from("d"), - )), + ), ]); assert_eq!(build_expression(&parse(f).unwrap()), Ok(expression)); diff --git a/core/market/resolver/tests/expression_resolve.rs b/core/market/resolver/tests/expression_resolve.rs index ed912b5f22..6548933a95 100644 --- a/core/market/resolver/tests/expression_resolve.rs +++ b/core/market/resolver/tests/expression_resolve.rs @@ -654,14 +654,14 @@ fn resolve_complex_and_undefined() { &PropertyRef::Value(String::from("b"), PropertyRefType::Any), ], Expression::And(vec![ - Box::new(Expression::Equals( + Expression::Equals( PropertyRef::Value(String::from("a"), PropertyRefType::Any), String::from("b"), - )), - Box::new(Expression::Equals( + ), + Expression::Equals( PropertyRef::Value(String::from("b"), PropertyRefType::Any), String::from("c"), - )), + ), ]), ), ); diff --git a/core/market/resolver/tests/matching.rs b/core/market/resolver/tests/matching.rs index c1ee3cd127..af2b5efffd 100644 --- a/core/market/resolver/tests/matching.rs +++ b/core/market/resolver/tests/matching.rs @@ -36,7 +36,7 @@ fn match_simple_error() { let prep_demand_result = PreparedDemand::from(&demand); match prep_demand_result { - Ok(_) => assert!(false, "Demand content error was not caught!"), + Ok(_) => panic!("Demand content error was not caught!"), Err(prep_error) => assert_eq!( prep_error, PrepareError::new("Error parsing Demand constraints: Parsing error: Alternative") diff --git a/core/market/resolver/tests/properties_list.rs b/core/market/resolver/tests/properties_list.rs index e875104016..a6c32ec86f 100644 --- a/core/market/resolver/tests/properties_list.rs +++ b/core/market/resolver/tests/properties_list.rs @@ -8,8 +8,8 @@ fn equals_for_list_contains_true() { Box::new(PropertyValue::Str("def")), ]); - assert_eq!(prop_value.equals("abc"), true); - assert_eq!(prop_value.equals("def"), true); + assert!(prop_value.equals("abc")); + assert!(prop_value.equals("def")); } #[test] @@ -19,7 +19,7 @@ fn equals_for_list_contains_false() { Box::new(PropertyValue::Str("def")), ]); - assert_eq!(prop_value.equals("fds"), false); + assert!(!prop_value.equals("fds")); } #[test] @@ -29,7 +29,7 @@ fn equals_for_list_list_equals_true() { Box::new(PropertyValue::Str("def")), ]); - assert_eq!(prop_value.equals("[abc,def]"), true); + assert!(prop_value.equals("[abc,def]")); } #[test] @@ -39,7 +39,7 @@ fn equals_for_list_list_different_length_false() { Box::new(PropertyValue::Str("def")), ]); - assert_eq!(prop_value.equals("[abc,def,xyz]"), false); + assert!(!prop_value.equals("[abc,def,xyz]")); } #[test] @@ -49,7 +49,7 @@ fn equals_for_list_list_different_items_false() { Box::new(PropertyValue::Str("def")), ]); - assert_eq!(prop_value.equals("[abc,xyz]"), false); + assert!(!prop_value.equals("[abc,xyz]")); } #[test] @@ -59,7 +59,7 @@ fn greater_for_list_false() { Box::new(PropertyValue::Str("def")), ]); - assert_eq!(prop_value.greater("abc"), false); + assert!(!prop_value.greater("abc")); } #[test] @@ -69,7 +69,7 @@ fn greater_equal_for_list_false() { Box::new(PropertyValue::Str("def")), ]); - assert_eq!(prop_value.greater_equal("abc"), false); + assert!(!prop_value.greater_equal("abc")); } #[test] @@ -79,7 +79,7 @@ fn less_for_list_false() { Box::new(PropertyValue::Str("def")), ]); - assert_eq!(prop_value.less("abc"), false); + assert!(!prop_value.less("abc")); } #[test] @@ -89,7 +89,7 @@ fn less_equal_for_list_false() { Box::new(PropertyValue::Str("def")), ]); - assert_eq!(prop_value.less_equal("abc"), false); + assert!(!prop_value.less_equal("abc")); } // #endregion diff --git a/core/market/resolver/tests/properties_string.rs b/core/market/resolver/tests/properties_string.rs index 748e7657c7..b01a9afd8d 100644 --- a/core/market/resolver/tests/properties_string.rs +++ b/core/market/resolver/tests/properties_string.rs @@ -6,28 +6,28 @@ use ya_market_resolver::resolver::properties::*; fn equals_for_strings_simple_true() { let prop_value = PropertyValue::Str("abc"); - assert_eq!(prop_value.equals("abc"), true); + assert!(prop_value.equals("abc")); } #[test] fn equals_for_strings_simple_false() { let prop_value = PropertyValue::Str("abc"); - assert_eq!(prop_value.equals("abas"), false); + assert!(!prop_value.equals("abas")); } #[test] fn equals_for_strings_wildcard_true() { let prop_value = PropertyValue::Str("abc"); - assert_eq!(prop_value.equals("ab*"), true); + assert!(prop_value.equals("ab*")); } #[test] fn equals_for_strings_wildcard_false() { let prop_value = PropertyValue::Str("abc"); - assert_eq!(prop_value.equals("as*"), false); + assert!(!prop_value.equals("as*")); } // #endregion diff --git a/core/market/resolver/tests/properties_version.rs b/core/market/resolver/tests/properties_version.rs index d5ccf8b697..c14915831d 100644 --- a/core/market/resolver/tests/properties_version.rs +++ b/core/market/resolver/tests/properties_version.rs @@ -8,42 +8,42 @@ use ya_market_resolver::resolver::properties::*; fn equals_for_version_simple_true() { let prop_value = PropertyValue::Version(Version::parse("0.5.0").unwrap()); - assert_eq!(prop_value.equals("0.5.0"), true); + assert!(prop_value.equals("0.5.0")); } #[test] fn equals_for_version_simple_false() { let prop_value = PropertyValue::Version(Version::parse("0.5.0").unwrap()); - assert_eq!(prop_value.equals("0.6.1"), false); + assert!(!prop_value.equals("0.6.1")); } #[test] fn less_for_version_simple_true() { let prop_value = PropertyValue::Version(Version::parse("0.5.0").unwrap()); - assert_eq!(prop_value.less("0.6.0"), true); + assert!(prop_value.less("0.6.0")); } #[test] fn less_equal_for_version_simple_true() { let prop_value = PropertyValue::Version(Version::parse("0.5.0").unwrap()); - assert_eq!(prop_value.less_equal("0.5.0"), true); + assert!(prop_value.less_equal("0.5.0")); } #[test] fn greater_for_version_simple_true() { let prop_value = PropertyValue::Version(Version::parse("0.5.0").unwrap()); - assert_eq!(prop_value.greater("0.4.0"), true); + assert!(prop_value.greater("0.4.0")); } #[test] fn greater_equal_for_version_simple_true() { let prop_value = PropertyValue::Version(Version::parse("0.5.0").unwrap()); - assert_eq!(prop_value.greater_equal("0.5.0"), true); + assert!(prop_value.greater_equal("0.5.0")); } // #endregion diff --git a/core/market/resolver/tests/sample/mod.rs b/core/market/resolver/tests/sample/mod.rs index 78c2443ef0..c590a8d11a 100644 --- a/core/market/resolver/tests/sample/mod.rs +++ b/core/market/resolver/tests/sample/mod.rs @@ -72,7 +72,7 @@ pub static POC_OFFER_PROPERTIES_JSON_DEEP: &str = r#" } }"#; -pub static POC_OFFER_PROPERTIES_FLAT: &'static [&'static str] = &[ +pub static POC_OFFER_PROPERTIES_FLAT: &[&str] = &[ "golem.com.pricing.model=\"linear\"", "golem.com.pricing.model.linear.coeffs=[0.1,0.2,1.0]", "golem.com.scheme=\"payu\"", @@ -109,14 +109,14 @@ pub static POC_DEMAND_PROPERTIES_JSON_DEEP: &str = r#" } }"#; -pub static POC_DEMAND_PROPERTIES_FLAT: &'static [&'static str] = &[ +pub static POC_DEMAND_PROPERTIES_FLAT: &[&str] = &[ "golem.node.debug.subnet=\"piotr\"", "golem.node.id.name=\"test1\"", "golem.srv.comp.expiration=1590765503361", "golem.srv.comp.task_package=\"hash://sha3:D5E31B2EED628572A5898BF8C34447644BFC4B5130CFC1E4F10AEAA1:http://12.34.56.78:8000/rust-wasi-tutorial.zip\"" ]; -pub static POC_DEMAND_CONSTRAINTS: &'static str = r#" +pub static POC_DEMAND_CONSTRAINTS: &str = r#" (& (golem.inf.mem.gib>0.5) (golem.inf.storage.gib>1) @@ -124,5 +124,5 @@ pub static POC_DEMAND_CONSTRAINTS: &'static str = r#" (golem.node.debug.subnet=piotr) )"#; -pub static POC_OFFER_CONSTRAINTS: &'static str = +pub static POC_OFFER_CONSTRAINTS: &str = "(&(golem.node.debug.subnet=piotr)(golem.srv.comp.expiration>0))"; diff --git a/core/market/src/cli.rs b/core/market/src/cli.rs new file mode 100644 index 0000000000..f2cea412de --- /dev/null +++ b/core/market/src/cli.rs @@ -0,0 +1,100 @@ +use chrono::{DateTime, Utc}; +use structopt::StructOpt; +use ya_client::model::market::{agreement::State, Role}; +use ya_core_model::market::{GetAgreement, ListAgreements}; +use ya_service_api::{CliCtx, CommandOutput, ResponseTable}; +use ya_service_bus::{typed as bus, RpcEndpoint}; + +/// Market management +#[derive(StructOpt, Debug)] +pub enum Command { + Agreements(AgreementsCommand), +} + +impl Command { + pub async fn run_command(self, ctx: &CliCtx) -> anyhow::Result { + match self { + Command::Agreements(agreements_cmd) => agreements_cmd.run_command(ctx).await, + } + } +} + +#[derive(StructOpt, Debug)] +pub enum AgreementsCommand { + List { + #[structopt(long, help = "Only show agreements with this state")] + state: Option, + #[structopt(long, help = "Only show agreements after this date, rfc3339")] + before: Option>, + #[structopt(long, help = "Only show agreements before this date, rfc3339")] + after: Option>, + #[structopt(long, help = "Only show agreements with this app session id")] + app_session_id: Option, + }, + Get { + #[structopt(long, help = "Agreement ID, may be obtained via list-agreements")] + id: String, + #[structopt(long, help = "Your role in the agreement (Provider | Requestor)")] + role: Role, + }, +} + +impl AgreementsCommand { + pub async fn run_command(self, _ctx: &CliCtx) -> anyhow::Result { + match self { + AgreementsCommand::List { + state, + before, + after, + app_session_id, + } => { + let request = ListAgreements { + state, + before_date: before, + after_date: after, + app_session_id, + }; + + let agreements = bus::service(ya_core_model::market::BUS_ID) + .send(request) + .await??; + + let mut agreements_json = Vec::new(); + for agreement in agreements { + agreements_json.push(serde_json::to_value([ + agreement.id, + agreement.role.to_string(), + agreement.timestamp.to_rfc3339(), + agreement + .approved_date + .map(|ts| ts.to_rfc3339()) + .unwrap_or_else(|| "N/A".to_owned()), + ])?); + } + + Ok(ResponseTable { + columns: vec![ + "id".to_owned(), + "role".to_owned(), + "created".to_owned(), + "approved".to_owned(), + ], + values: agreements_json, + } + .with_header("\nMatching agreements:\n".to_owned())) + } + AgreementsCommand::Get { id, role } => { + let request = GetAgreement { + agreement_id: id, + role, + }; + + let agreement = bus::service(ya_core_model::market::BUS_ID) + .send(request) + .await??; + + CommandOutput::object(agreement) + } + } + } +} diff --git a/core/market/src/config.rs b/core/market/src/config.rs index f8ca074bbc..e12a63f8d7 100644 --- a/core/market/src/config.rs +++ b/core/market/src/config.rs @@ -61,7 +61,7 @@ impl Config { pub fn from_env() -> Result { // Empty command line arguments, because we want to use ENV fallback // or default values if ENV variables are not set. - Ok(Config::from_iter_safe(&[""])?) + Config::from_iter_safe(&[""]) } } diff --git a/core/market/src/db/dao/agreement.rs b/core/market/src/db/dao/agreement.rs index b337d3e29a..e0f7de4eac 100644 --- a/core/market/src/db/dao/agreement.rs +++ b/core/market/src/db/dao/agreement.rs @@ -1,4 +1,4 @@ -use chrono::{NaiveDateTime, Utc}; +use chrono::{DateTime, NaiveDateTime, Utc}; use diesel::prelude::*; use ya_client::model::market::Reason; @@ -61,6 +61,48 @@ pub enum AgreementDaoError { } impl<'c> AgreementDao<'c> { + pub async fn list( + &self, + node_id: Option, + state: Option, + before: Option>, + after: Option>, + app_session_id: Option, + ) -> Result, AgreementDaoError> { + do_with_transaction(self.pool, move |conn| { + let mut query = market_agreement.into_boxed(); + + if let Some(node_id) = node_id { + query = query.filter( + agreement::provider_id + .eq(node_id) + .or(agreement::requestor_id.eq(node_id)), + ); + }; + + if let Some(app_session_id) = app_session_id { + query = query.filter(agreement::session_id.eq(app_session_id)) + } + + if let Some(state) = state { + query = query.filter(agreement::state.eq(state)); + } + + if let Some(before) = before { + query = query.filter(agreement::creation_ts.lt(before.naive_utc())); + } + + if let Some(after) = after { + query = query.filter(agreement::creation_ts.gt(after.naive_utc())); + } + + let agreements = query.get_results::(conn)?; + + Ok(agreements) + }) + .await + } + pub async fn select( &self, id: &AgreementId, @@ -138,7 +180,7 @@ impl<'c> AgreementDao<'c> { let proposal_id = agreement.offer_proposal_id.clone(); readonly_transaction(self.ram_pool, move |conn| { if has_counter_proposal(conn, &proposal_id)? { - return Err(SaveAgreementError::ProposalCountered(proposal_id.clone())); + return Err(SaveAgreementError::ProposalCountered(proposal_id)); } Ok(()) }) @@ -171,11 +213,11 @@ impl<'c> AgreementDao<'c> { &self, id: &AgreementId, session: &AppSessionId, - signature: &String, + signature: &str, ) -> Result { let id = id.clone(); let session = session.clone(); - let signature = signature.clone(); + let signature = signature.to_owned(); do_with_transaction(self.pool, move |conn| { let mut agreement: Agreement = @@ -198,13 +240,13 @@ impl<'c> AgreementDao<'c> { &self, id: &AgreementId, session: &AppSessionId, - signature: &String, + signature: &str, timestamp: &NaiveDateTime, ) -> Result { let id = id.clone(); let session = session.clone(); - let signature = signature.clone(); - let timestamp = timestamp.clone(); + let signature = signature.to_owned(); + let timestamp = *timestamp; do_with_transaction(self.pool, move |conn| { let mut agreement: Agreement = @@ -229,10 +271,10 @@ impl<'c> AgreementDao<'c> { pub async fn approve( &self, id: &AgreementId, - signature: &String, + signature: &str, ) -> Result { let id = id.clone(); - let signature = signature.clone(); + let signature = signature.to_owned(); do_with_transaction(self.pool, move |conn| { let mut agreement: Agreement = @@ -247,7 +289,9 @@ impl<'c> AgreementDao<'c> { &agreement, None, Owner::Provider, - agreement.approved_ts.unwrap_or(Utc::now().naive_utc()), + agreement + .approved_ts + .unwrap_or_else(|| Utc::now().naive_utc()), )?; Ok(agreement) @@ -262,7 +306,7 @@ impl<'c> AgreementDao<'c> { timestamp: &NaiveDateTime, ) -> Result { let id = id.clone(); - let timestamp = timestamp.clone(); + let timestamp = *timestamp; do_with_transaction(self.pool, move |conn| { let mut agreement: Agreement = @@ -283,7 +327,7 @@ impl<'c> AgreementDao<'c> { timestamp: &NaiveDateTime, ) -> Result { let id = id.clone(); - let timestamp = timestamp.clone(); + let timestamp = *timestamp; do_with_transaction(self.pool, move |conn| { let mut agreement: Agreement = @@ -305,7 +349,7 @@ impl<'c> AgreementDao<'c> { timestamp: &NaiveDateTime, ) -> Result { let id = id.clone(); - let timestamp = timestamp.clone(); + let timestamp = *timestamp; do_with_transaction(self.pool, move |conn| { let mut agreement: Agreement = diff --git a/core/market/src/db/dao/agreement_events.rs b/core/market/src/db/dao/agreement_events.rs index ece4fe44ed..0060d7eb37 100644 --- a/core/market/src/db/dao/agreement_events.rs +++ b/core/market/src/db/dao/agreement_events.rs @@ -5,6 +5,7 @@ use ya_client::model::market::Reason; use ya_client::model::NodeId; use ya_persistence::executor::PoolType; use ya_persistence::executor::{readonly_transaction, ConnType}; +use ya_persistence::types::AdaptTimestamp; use crate::db::dao::AgreementDaoError; use crate::db::model::{Agreement, AgreementEvent, AgreementId, NewAgreementEvent}; @@ -34,7 +35,7 @@ impl<'c> AgreementEventsDao<'c> { after_timestamp: NaiveDateTime, ) -> DbResult> { let session_id = session_id.clone(); - let node_id = node_id.clone(); + let node_id = *node_id; readonly_transaction(self.pool, move |conn| { // We will get only one Agreement, by using this filter. // There will be no way to get Requestor'a Agreement, when being Provider, and vice versa, @@ -56,7 +57,7 @@ impl<'c> AgreementEventsDao<'c> { Ok(market_agreement_event .filter(event::agreement_id.eq_any(select_corresponding_agreement)) - .filter(event::timestamp.gt(after_timestamp)) + .filter(event::timestamp.gt(after_timestamp.adapt())) .order_by(event::timestamp.asc()) .limit(max_events as i64) .load::(conn)?) diff --git a/core/market/src/db/dao/cleaner.rs b/core/market/src/db/dao/cleaner.rs index b88f614f05..55f01894c2 100644 --- a/core/market/src/db/dao/cleaner.rs +++ b/core/market/src/db/dao/cleaner.rs @@ -21,9 +21,8 @@ pub async fn clean(db: DbMixedExecutor, cfg: &DbConfig) { ); let v_results = vec![results.0, results.1, results.2, results.3, results.4]; for db_result in v_results.into_iter() { - match db_result { - Err(e) => log::error!("Market database cleaner error: {}", e), - _ => (), + if let Err(e) = db_result { + log::error!("Market database cleaner error: {}", e) } } } diff --git a/core/market/src/db/dao/negotiation_events.rs b/core/market/src/db/dao/negotiation_events.rs index 3e261b922d..717b765c12 100644 --- a/core/market/src/db/dao/negotiation_events.rs +++ b/core/market/src/db/dao/negotiation_events.rs @@ -90,7 +90,6 @@ impl<'c> NegotiationEventsDao<'c> { let basic_query = dsl::market_negotiation_event.filter(dsl::subscription_id.eq(&subscription_id)); let mut events = basic_query - .clone() .filter(dsl::event_type.ne_all(vec![ EventType::ProviderNewProposal, EventType::RequestorNewProposal, @@ -162,12 +161,12 @@ fn validate_subscription( owner: Owner, ) -> Result<(), TakeEventsError> { match owner { - Owner::Requestor => match demand_status(conn, &subscription_id)? { + Owner::Requestor => match demand_status(conn, subscription_id)? { DemandState::NotFound => Err(TakeEventsError::NotFound(subscription_id.clone()))?, DemandState::Expired(_) => Err(TakeEventsError::Expired(subscription_id.clone()))?, _ => Ok(()), }, - Owner::Provider => match query_state(conn, &subscription_id, &Utc::now().naive_utc())? { + Owner::Provider => match query_state(conn, subscription_id, &Utc::now().naive_utc())? { OfferState::NotFound => Err(TakeEventsError::NotFound(subscription_id.clone()))?, OfferState::Expired(_) => Err(TakeEventsError::Expired(subscription_id.clone()))?, _ => Ok(()), diff --git a/core/market/src/db/dao/offer.rs b/core/market/src/db/dao/offer.rs index 1fba7fc1dd..31d0826c5a 100644 --- a/core/market/src/db/dao/offer.rs +++ b/core/market/src/db/dao/offer.rs @@ -277,7 +277,7 @@ pub(super) fn query_state( id: &SubscriptionId, expiry_validation_ts: &NaiveDateTime, ) -> DbResult { - let offer: Option = query_offer(conn, &id)?; + let offer: Option = query_offer(conn, id)?; if is_unsubscribed(conn, id)? { return Ok(OfferState::Unsubscribed(offer)); diff --git a/core/market/src/db/dao/proposal.rs b/core/market/src/db/dao/proposal.rs index e29ed7dc7f..3c1c9c8af8 100644 --- a/core/market/src/db/dao/proposal.rs +++ b/core/market/src/db/dao/proposal.rs @@ -64,7 +64,7 @@ impl<'c> ProposalDao<'c> { let prev_proposal_id = proposal .prev_proposal_id .clone() - .ok_or(SaveProposalError::NoPrevious(proposal.id.clone()))?; + .ok_or_else(|| SaveProposalError::NoPrevious(proposal.id.clone()))?; if has_counter_proposal(conn, &prev_proposal_id)? { return Err(SaveProposalError::AlreadyCountered(prev_proposal_id)); @@ -74,7 +74,7 @@ impl<'c> ProposalDao<'c> { .filter(dsl::id.eq(&prev_proposal_id)) .first(conn) .optional()? - .ok_or(SaveProposalError::NoPrevious(proposal.id.clone()))?; + .ok_or_else(|| SaveProposalError::NoPrevious(proposal.id.clone()))?; // If previous Proposal was rejected, we must change it's state back. if prev_proposal.state == ProposalState::Rejected { diff --git a/core/market/src/db/model/agreement.rs b/core/market/src/db/model/agreement.rs index a0c20ee306..43cf1a3e09 100644 --- a/core/market/src/db/model/agreement.rs +++ b/core/market/src/db/model/agreement.rs @@ -26,6 +26,7 @@ pub type AppSessionId = Option; AsExpression, FromSqlRow, PartialEq, + Eq, Debug, Clone, Copy, @@ -194,6 +195,20 @@ impl From for ClientAgreementState { } } +impl From for AgreementState { + fn from(agreement_state: ClientAgreementState) -> Self { + match agreement_state { + ClientAgreementState::Proposal => AgreementState::Proposal, + ClientAgreementState::Pending => AgreementState::Pending, + ClientAgreementState::Cancelled => AgreementState::Cancelled, + ClientAgreementState::Rejected => AgreementState::Rejected, + ClientAgreementState::Approved => AgreementState::Approved, + ClientAgreementState::Expired => AgreementState::Expired, + ClientAgreementState::Terminated => AgreementState::Terminated, + } + } +} + pub fn check_transition(from: AgreementState, to: AgreementState) -> Result<(), AgreementDaoError> { log::trace!("Checking Agreement state transition: {} => {}", from, to); match from { @@ -219,10 +234,11 @@ pub fn check_transition(from: AgreementState, to: AgreementState) -> Result<(), }, AgreementState::Cancelled => (), AgreementState::Rejected => (), - AgreementState::Approved => match to { - AgreementState::Terminated => return Ok(()), - _ => (), - }, + AgreementState::Approved => { + if to == AgreementState::Terminated { + return Ok(()); + } + } AgreementState::Expired => (), AgreementState::Terminated => (), }; diff --git a/core/market/src/db/model/agreement_events.rs b/core/market/src/db/model/agreement_events.rs index 0e93ec9c0b..7930f7b6ec 100644 --- a/core/market/src/db/model/agreement_events.rs +++ b/core/market/src/db/model/agreement_events.rs @@ -12,6 +12,7 @@ use ya_client::model::market::{ AgreementEventType as ClientEventType, AgreementOperationEvent as ClientEvent, Reason, }; use ya_diesel_utils::DbTextField; +use ya_persistence::types::{AdaptTimestamp, TimestampAdapter}; #[derive( DbTextField, @@ -20,6 +21,7 @@ use ya_diesel_utils::DbTextField; AsExpression, FromSqlRow, PartialEq, + Eq, Debug, Clone, Copy, @@ -52,7 +54,7 @@ pub struct AgreementEvent { pub struct NewAgreementEvent { pub agreement_id: AgreementId, pub event_type: AgreementEventType, - pub timestamp: NaiveDateTime, + pub timestamp: TimestampAdapter, pub issuer: Owner, pub reason: Option, } @@ -90,9 +92,9 @@ impl NewAgreementEvent { // so we need to have the same value on both nodes. // I don't know, how to solve this problem now, so I leave code that makes it possible to // add this external timestamp to database, but here I will use generated value. - timestamp: Utc::now().naive_utc(), + timestamp: Utc::now().adapt(), issuer: terminator, - reason: reason.map(|reason| DbReason(reason)), + reason: reason.map(DbReason), }) } } diff --git a/core/market/src/db/model/demand.rs b/core/market/src/db/model/demand.rs index 6db0e41878..c136bffe89 100644 --- a/core/market/src/db/model/demand.rs +++ b/core/market/src/db/model/demand.rs @@ -1,5 +1,4 @@ use chrono::{NaiveDateTime, TimeZone, Utc}; -use serde_json; use ya_client::model::{market::Demand as ClientDemand, ErrorMessage, NodeId}; use ya_service_api_web::middleware::Identity; @@ -61,7 +60,7 @@ impl Demand { pub fn into_client_demand(&self) -> Result { Ok(ClientDemand { demand_id: self.id.to_string(), - requestor_id: self.node_id.clone(), + requestor_id: self.node_id, constraints: self.constraints.clone(), properties: serde_json::from_str(&self.properties).map_err(|e| { format!( diff --git a/core/market/src/db/model/negotiation_events.rs b/core/market/src/db/model/negotiation_events.rs index 2febd164ec..13e20b2801 100644 --- a/core/market/src/db/model/negotiation_events.rs +++ b/core/market/src/db/model/negotiation_events.rs @@ -33,6 +33,7 @@ pub enum EventError { AsExpression, FromSqlRow, PartialEq, + Eq, Debug, Clone, Copy, @@ -97,7 +98,7 @@ impl MarketEvent { Owner::Provider => EventType::ProviderProposalRejected, }, artifact_id: proposal.body.id.clone(), - reason: reason.map(|reason| DbReason(reason)), + reason: reason.map(DbReason), } } @@ -142,7 +143,7 @@ impl MarketEvent { .get_proposal(&self.artifact_id) .await .map_err(|e| EventError::GetError(self.artifact_id.clone(), e.to_string()))? - .ok_or(EventError::ProposalNotFound(self.artifact_id.clone()))?; + .ok_or_else(|| EventError::ProposalNotFound(self.artifact_id.clone()))?; Ok(prop.into_client()?) } @@ -156,7 +157,7 @@ impl MarketEvent { .select(&self.artifact_id, None, Utc::now().naive_utc()) .await .map_err(|e| EventError::GetError(self.artifact_id.clone(), e.to_string()))? - .ok_or(EventError::AgreementNotFound(self.artifact_id.clone()))?; + .ok_or_else(|| EventError::AgreementNotFound(self.artifact_id.clone()))?; Ok(agreement.into_client()?) } diff --git a/core/market/src/db/model/offer.rs b/core/market/src/db/model/offer.rs index 3193949564..b7564c03e1 100644 --- a/core/market/src/db/model/offer.rs +++ b/core/market/src/db/model/offer.rs @@ -1,6 +1,5 @@ use chrono::{NaiveDateTime, TimeZone, Utc}; use serde::{Deserialize, Serialize}; -use serde_json; use ya_client::model::{market::Offer as ClientOffer, ErrorMessage, NodeId}; use ya_service_api_web::middleware::Identity; @@ -82,7 +81,7 @@ impl Offer { pub fn into_client_offer(&self) -> Result { Ok(ClientOffer { offer_id: self.id.to_string(), - provider_id: self.node_id.clone(), + provider_id: self.node_id, constraints: self.constraints.clone(), properties: serde_json::from_str(&self.properties).map_err(|e| { format!( @@ -142,7 +141,7 @@ mod tests { let node_id = "0xbabe000000000000000000000000000000000000"; let offer = Offer { - id: SubscriptionId::from_str(&false_subscription_id).unwrap(), + id: SubscriptionId::from_str(false_subscription_id).unwrap(), properties: "{}".to_string(), constraints: "()".to_string(), node_id: NodeId::from_str(node_id).unwrap(), @@ -165,7 +164,7 @@ mod tests { let node_id = "0xbabe000000000000000000000000000000000000"; let offer = Offer { - id: SubscriptionId::from_str(&offer_id).unwrap(), + id: SubscriptionId::from_str(offer_id).unwrap(), properties: "{}".to_string(), constraints: "()".to_string(), node_id: NodeId::from_str(node_id).unwrap(), diff --git a/core/market/src/db/model/proposal.rs b/core/market/src/db/model/proposal.rs index d380b67428..7802b53022 100644 --- a/core/market/src/db/model/proposal.rs +++ b/core/market/src/db/model/proposal.rs @@ -25,6 +25,7 @@ use crate::protocol::negotiation::messages::ProposalContent; AsExpression, FromSqlRow, PartialEq, + Eq, Debug, Clone, Copy, @@ -52,6 +53,7 @@ pub enum ProposalState { AsExpression, FromSqlRow, PartialEq, + Eq, Debug, Clone, Copy, @@ -129,8 +131,8 @@ impl Proposal { let negotiation = Negotiation::from_subscriptions(&demand, &offer, Owner::Requestor); let creation_ts = Utc::now().naive_utc(); let expiration_ts = match demand.expiration_ts < offer.expiration_ts { - true => demand.expiration_ts.clone(), - false => offer.expiration_ts.clone(), + true => demand.expiration_ts, + false => offer.expiration_ts, }; let proposal_id = @@ -169,7 +171,7 @@ impl Proposal { // TODO: Initial proposal id will differ on Requestor and Provider!! let creation_ts = Utc::now().naive_utc(); let proposal_id = - ProposalId::generate_id(&offer.id, &demand_id, &creation_ts, Owner::Provider); + ProposalId::generate_id(&offer.id, demand_id, &creation_ts, Owner::Provider); let proposal = DbProposal { id: proposal_id, @@ -234,7 +236,7 @@ impl Proposal { constraints: proposal.constraints.clone(), state: ProposalState::Draft, creation_ts, - expiration_ts: expiration_ts.clone(), + expiration_ts: *expiration_ts, }; Ok(Proposal { @@ -266,22 +268,22 @@ impl Proposal { pub fn issuer(&self) -> NodeId { match self.body.issuer { Issuer::Us => match self.body.id.owner() { - Owner::Requestor => self.negotiation.requestor_id.clone(), - Owner::Provider => self.negotiation.provider_id.clone(), + Owner::Requestor => self.negotiation.requestor_id, + Owner::Provider => self.negotiation.provider_id, }, Issuer::Them => match self.body.id.owner() { - Owner::Requestor => self.negotiation.provider_id.clone(), - Owner::Provider => self.negotiation.requestor_id.clone(), + Owner::Requestor => self.negotiation.provider_id, + Owner::Provider => self.negotiation.requestor_id, }, } } pub fn validate_id(&self) -> Result<(), ProposalIdValidationError> { - Ok(self.body.id.validate( + self.body.id.validate( &self.negotiation.offer_id, &self.negotiation.demand_id, &self.body.creation_ts, - )?) + ) } } diff --git a/core/market/src/db/model/proposal_id.rs b/core/market/src/db/model/proposal_id.rs index 45553d111d..8710f4ace8 100644 --- a/core/market/src/db/model/proposal_id.rs +++ b/core/market/src/db/model/proposal_id.rs @@ -44,7 +44,7 @@ impl Owner { const HASH_SUFFIX_LEN: usize = 64; -#[derive(Error, Debug, PartialEq)] +#[derive(Error, Debug, PartialEq, Eq)] pub enum ProposalIdParseError { #[error("Id [{0}] has invalid format.")] InvalidFormat(String), @@ -56,7 +56,7 @@ pub enum ProposalIdParseError { InvalidLength(String), } -#[derive(thiserror::Error, Debug, PartialEq, Serialize, Deserialize)] +#[derive(thiserror::Error, Debug, PartialEq, Eq, Serialize, Deserialize)] #[error("Proposal id [{0}] has unexpected hash [{1}].")] pub struct ProposalIdValidationError(ProposalId, String); @@ -77,12 +77,12 @@ impl ProposalId { ) -> ProposalId { ProposalId { owner, - id: hash_proposal(&offer_id, &demand_id, &creation_ts), + id: hash_proposal(offer_id, demand_id, creation_ts), } } pub fn owner(&self) -> Owner { - self.owner.clone() + self.owner } pub fn translate(mut self, new_owner: Owner) -> Self { @@ -101,7 +101,7 @@ impl ProposalId { demand_id: &SubscriptionId, creation_ts: &NaiveDateTime, ) -> Result<(), ProposalIdValidationError> { - let hash = hash_proposal(&offer_id, &demand_id, &creation_ts); + let hash = hash_proposal(offer_id, demand_id, creation_ts); if self.id != hash { return Err(ProposalIdValidationError(self.clone(), hash)); } @@ -167,7 +167,7 @@ impl FromStr for Owner { Err(ProposalIdParseError::InvalidOwner(s.to_string()))?; } - Ok(match s.chars().nth(0).unwrap() { + Ok(match s.chars().next().unwrap() { 'P' => Owner::Provider, 'R' => Owner::Requestor, _ => Err(ProposalIdParseError::InvalidOwner(s.to_string()))?, diff --git a/core/market/src/db/model/subscription_id.rs b/core/market/src/db/model/subscription_id.rs index d2c393c70a..2ca7a69038 100644 --- a/core/market/src/db/model/subscription_id.rs +++ b/core/market/src/db/model/subscription_id.rs @@ -12,7 +12,7 @@ use ya_diesel_utils::DbTextField; const RANDOM_PREFIX_LEN: usize = 32; const HASH_SUFFIX_LEN: usize = 64; -#[derive(thiserror::Error, Debug, PartialEq)] +#[derive(thiserror::Error, Debug, PartialEq, Eq)] pub enum SubscriptionParseError { #[error("Subscription id [{0}] has invalid format.")] InvalidFormat(String), @@ -26,7 +26,7 @@ pub enum SubscriptionParseError { InvalidLength(String), } -#[derive(thiserror::Error, Debug, PartialEq)] +#[derive(thiserror::Error, Debug, PartialEq, Eq)] #[error("Subscription id [{0}] doesn't match content hash [{1}].")] pub struct SubscriptionValidationError(SubscriptionId, String); @@ -108,7 +108,7 @@ impl FromStr for SubscriptionId { if !elements .iter() .map(|slice| slice.chars().all(|character| character.is_ascii_hexdigit())) - .all(|result| result == true) + .all(|result| result) { Err(SubscriptionParseError::NotHexadecimal(s.to_string()))?; } diff --git a/core/market/src/identity.rs b/core/market/src/identity.rs index b0e7747dac..4cec60c5f7 100644 --- a/core/market/src/identity.rs +++ b/core/market/src/identity.rs @@ -51,6 +51,7 @@ impl IdentityApi for IdentityGSB { } } +#[allow(clippy::new_ret_no_self)] impl IdentityGSB { pub fn new() -> Arc { Arc::new(IdentityGSB) diff --git a/core/market/src/lib.rs b/core/market/src/lib.rs index 385d1ffa3f..7b8a335c01 100644 --- a/core/market/src/lib.rs +++ b/core/market/src/lib.rs @@ -1,6 +1,7 @@ #[macro_use] extern crate diesel; +mod cli; mod config; mod db; mod identity; diff --git a/core/market/src/market.rs b/core/market/src/market.rs index f27b523304..d6c18e9eb9 100644 --- a/core/market/src/market.rs +++ b/core/market/src/market.rs @@ -7,7 +7,7 @@ use thiserror::Error; use crate::config::Config; use crate::db::dao::AgreementDao; -use crate::db::model::{AgreementId, AppSessionId, SubscriptionId}; +use crate::db::model::{AgreementId, AppSessionId, Owner, SubscriptionId}; use crate::identity::{IdentityApi, IdentityGSB}; use crate::matcher::error::{ DemandError, MatcherError, MatcherInitError, QueryDemandsError, QueryOfferError, @@ -19,10 +19,11 @@ use crate::negotiation::error::{ }; use crate::negotiation::{EventNotifier, ProviderBroker, RequestorBroker}; use crate::rest_api; +use crate::testing::AgreementState; use ya_client::model::market::{ - Agreement, AgreementOperationEvent as ClientAgreementEvent, Demand, NewDemand, NewOffer, Offer, - Reason, + Agreement, AgreementListEntry, AgreementOperationEvent as ClientAgreementEvent, Demand, + NewDemand, NewOffer, Offer, Reason, Role, }; use ya_core_model::market::{local, BUS_ID}; use ya_service_api_interfaces::{Provider, Service}; @@ -94,7 +95,7 @@ impl MarketService { // We need the same notifier for both Provider and Requestor implementation since we have // single endpoint and both implementations are able to add events. - let agreement_notifier = EventNotifier::::new(); + let agreement_notifier = EventNotifier::::default(); let provider_engine = ProviderBroker::new( db.clone(), @@ -104,7 +105,7 @@ impl MarketService { )?; let requestor_engine = RequestorBroker::new( db.clone(), - store.clone(), + store, listeners.proposal_receiver, agreement_notifier, config.clone(), @@ -234,6 +235,41 @@ impl MarketService { Ok(()) } + pub async fn list_agreements( + &self, + id: &Identity, + state: Option, + before: Option>, + after: Option>, + app_sesssion_id: Option, + ) -> Result, AgreementError> { + let agreements = self + .db + .as_dao::() + .list(Some(id.identity), state, before, after, app_sesssion_id) + .await + .map_err(|e| AgreementError::Internal(e.to_string()))?; + + let mut result = Vec::new(); + let naive_to_utc = |ts| DateTime::::from_utc(ts, Utc); + + for agreement in agreements { + let role = match agreement.id.owner() { + Owner::Provider => Role::Provider, + Owner::Requestor => Role::Requestor, + }; + + result.push(AgreementListEntry { + id: agreement.id.into_client(), + timestamp: naive_to_utc(agreement.creation_ts), + approved_date: agreement.approved_ts.map(naive_to_utc), + role, + }); + } + + Ok(result) + } + pub async fn get_agreement( &self, agreement_id: &AgreementId, @@ -285,7 +321,7 @@ impl MarketService { } impl Service for MarketService { - type Cli = (); + type Cli = crate::cli::Command; } // =========================================== // diff --git a/core/market/src/market/agreement.rs b/core/market/src/market/agreement.rs index dd100025c8..d6e25a84f9 100644 --- a/core/market/src/market/agreement.rs +++ b/core/market/src/market/agreement.rs @@ -1,10 +1,6 @@ -use chrono; - -use ya_client::model::market::Agreement as ClientAgreement; -use ya_core_model::{ - market::{GetAgreement, RpcMessageError}, - Role, -}; +use chrono::{DateTime, Utc}; +use ya_client::model::market::{Agreement as ClientAgreement, AgreementListEntry, Role}; +use ya_core_model::market::{GetAgreement, ListAgreements, RpcMessageError}; use ya_service_bus::typed::ServiceBinder; use crate::db::dao::AgreementDao; @@ -13,10 +9,50 @@ use crate::db::DbMixedExecutor; pub async fn bind_gsb(db: DbMixedExecutor, public_prefix: &str, _local_prefix: &str) { log::trace!("Binding market agreement public service to service bus"); - ServiceBinder::new(public_prefix, &db, ()).bind(get_agreement); + ServiceBinder::new(public_prefix, &db, ()) + .bind(list_agreements) + .bind(get_agreement); log::debug!("Successfully bound market agreement public service to service bus"); } +async fn list_agreements( + db: DbMixedExecutor, + _sender_id: String, + msg: ListAgreements, +) -> Result, RpcMessageError> { + let dao = db.as_dao::(); + + let agreements = dao + .list( + None, + msg.state.map(Into::into), + msg.before_date, + msg.after_date, + msg.app_session_id, + ) + .await + .map_err(|e| RpcMessageError::Market(e.to_string()))?; + + let mut result = Vec::new(); + let naive_to_utc = |ts| DateTime::::from_utc(ts, Utc); + + for agreement in agreements { + let role = match agreement.id.owner() { + Owner::Provider => Role::Provider, + Owner::Requestor => Role::Requestor, + }; + + result.push(AgreementListEntry { + id: agreement.id.into_client(), + timestamp: naive_to_utc(agreement.creation_ts), + approved_date: agreement.approved_ts.map(naive_to_utc), + role, + }); + } + + Ok(result) +} + async fn get_agreement( db: DbMixedExecutor, _sender_id: String, @@ -34,11 +70,10 @@ async fn get_agreement( // I'm not sure we can trust `sender_id` value from gsb now. let dao = db.as_dao::(); let now = chrono::Utc::now().naive_utc(); - Ok(dao - .select(&agreement_id, None, now) + dao.select(&agreement_id, None, now) .await .map_err(|e| RpcMessageError::Market(e.to_string()))? - .ok_or(RpcMessageError::NotFound(msg.agreement_id.clone()))? + .ok_or_else(|| RpcMessageError::NotFound(msg.agreement_id.clone()))? .into_client() - .map_err(|e| RpcMessageError::Market(e.to_string()))?) + .map_err(|e| RpcMessageError::Market(e.to_string())) } diff --git a/core/market/src/matcher.rs b/core/market/src/matcher.rs index c6cbe89ec4..dc5bb55eda 100644 --- a/core/market/src/matcher.rs +++ b/core/market/src/matcher.rs @@ -77,7 +77,7 @@ impl Matcher { discovery, config, identity: identity_api, - expiration_tracker: DeadlineChecker::new().start(), + expiration_tracker: DeadlineChecker::default().start(), }; let listeners = EventsListeners { proposal_receiver }; @@ -122,20 +122,18 @@ impl Matcher { async move { let id = SubscriptionId::from_str(&msg.id); match (&msg.category[..], &id) { - ("Offer", Ok(id)) => match store.get_offer(id).await { - Err(QueryOfferError::Expired(_)) => { + ("Offer", Ok(id)) => { + if let Err(QueryOfferError::Expired(_)) = store.get_offer(id).await { log::info!("Offer [{}] expired.", id); counter!("market.offers.expired", 1) } - _ => (), - }, + } ("Demand", Ok(id)) => { - match store.db.as_dao::().demand_state(id).await { - Ok(DemandState::Expired(_)) => { - log::info!("Demand [{}] expired.", id); - counter!("market.demands.expired", 1) - } - _ => (), + if let Ok(DemandState::Expired(_)) = + store.db.as_dao::().demand_state(id).await + { + log::info!("Demand [{}] expired.", id); + counter!("market.demands.expired", 1) } } _ => {} @@ -238,7 +236,7 @@ impl Matcher { // If re-broadcasts are disabled, fallback to lazy broadcast binding self.discovery.bind_gsb_broadcast().await.map_or_else( |e| { - log::warn!("Failed to subscribe to broadcasts. Error: {:?}.", e,); + log::warn!("Failed to subscribe to broadcasts. Error: {e}."); }, |_| (), ); @@ -273,16 +271,15 @@ impl Matcher { pub async fn get_our_active_offer_ids(&self) -> Result, QueryOffersError> { let our_node_ids = self.identity.list().await?; - Ok(self.store.get_active_offer_ids(Some(our_node_ids)).await?) + self.store.get_active_offer_ids(Some(our_node_ids)).await } pub async fn get_our_unsubscribed_offer_ids( &self, ) -> Result, QueryOffersError> { let our_node_ids = self.identity.list().await?; - Ok(self - .store + self.store .get_unsubscribed_offer_ids(Some(our_node_ids)) - .await?) + .await } } diff --git a/core/market/src/matcher/cyclic.rs b/core/market/src/matcher/cyclic.rs index 0ca20a88ef..c60da500f0 100644 --- a/core/market/src/matcher/cyclic.rs +++ b/core/market/src/matcher/cyclic.rs @@ -9,7 +9,7 @@ use super::Matcher; use std::time::Instant; pub(super) async fn bcast_offers(matcher: Matcher) { - if matcher.config.discovery.max_bcasted_offers <= 0 { + if matcher.config.discovery.max_bcasted_offers == 0 { return; } @@ -26,18 +26,22 @@ pub(super) async fn bcast_offers(matcher: Matcher) { // Add some random subset of Offers to broadcast. let num_our_offers = our_ids.len(); - // let num_to_bcast = matcher.config.discovery.max_bcasted_offers; - // - // let all_ids = matcher.store.get_active_offer_ids(None).await?; - // let our_and_random_ids = randomize_ids(our_ids, all_ids, num_to_bcast as usize); + let num_to_bcast = matcher.config.discovery.max_bcasted_offers; + + let offers_to_broadcast = if matcher.discovery.is_hybrid_net() { + let all_ids = matcher.store.get_active_offer_ids(None).await?; + randomize_ids(our_ids, all_ids, num_to_bcast as usize) + } else { + our_ids + }; log::trace!( "Broadcasted {} Offers including {} ours.", - our_ids.len(), + offers_to_broadcast.len(), num_our_offers ); - matcher.discovery.bcast_offers(our_ids).await?; + matcher.discovery.bcast_offers(offers_to_broadcast).await?; let end = Instant::now(); counter!("market.offers.broadcasts", 1); @@ -52,7 +56,7 @@ pub(super) async fn bcast_offers(matcher: Matcher) { } pub(super) async fn bcast_unsubscribes(matcher: Matcher) { - if matcher.config.discovery.max_bcasted_unsubscribes <= 0 { + if matcher.config.discovery.max_bcasted_unsubscribes == 0 { return; } @@ -69,18 +73,25 @@ pub(super) async fn bcast_unsubscribes(matcher: Matcher) { // Add some random subset of Offer unsubscribes to bcast. let num_our_unsubscribes = our_ids.len(); - // let max_bcast = matcher.config.discovery.max_bcasted_unsubscribes as usize; - // - // let all_ids = matcher.store.get_unsubscribed_offer_ids(None).await?; - // let our_and_random_ids = randomize_ids(our_ids, all_ids, max_bcast); + let max_bcast = matcher.config.discovery.max_bcasted_unsubscribes as usize; + + let offers_to_broadcast = if matcher.discovery.is_hybrid_net() { + let all_ids = matcher.store.get_unsubscribed_offer_ids(None).await?; + randomize_ids(our_ids, all_ids, max_bcast as usize) + } else { + our_ids + }; log::trace!( "Broadcasted {} unsubscribed Offers including {} ours.", - our_ids.len(), + offers_to_broadcast.len(), num_our_unsubscribes ); - matcher.discovery.bcast_unsubscribes(our_ids).await?; + matcher + .discovery + .bcast_unsubscribes(offers_to_broadcast) + .await?; let end = Instant::now(); counter!("market.offers.unsubscribes.broadcasts", 1); diff --git a/core/market/src/matcher/handlers.rs b/core/market/src/matcher/handlers.rs index a31be1356b..0f518281c1 100644 --- a/core/market/src/matcher/handlers.rs +++ b/core/market/src/matcher/handlers.rs @@ -20,10 +20,10 @@ pub(super) async fn filter_out_known_offer_ids( // We shouldn't propagate Offer, if we already have it in our database. // Note that when we broadcast our Offer, it will reach us too, so it concerns // not only Offers from other nodes. - Ok(store + store .filter_out_known_offer_ids(msg.offer_ids) .await - .map_err(|e| log::warn!("Error filtering Offers. Error: {}", e))?) + .map_err(|e| log::warn!("Error filtering Offers. Error: {}", e)) } /// Returns only ids of those from input offers, that was successfully stored locally. @@ -75,9 +75,9 @@ pub(super) async fn get_local_offers( Ok(offers) => Ok(offers), Err(e) => { log::error!("Failed to get batch offers. Error: {}", e); - Err(DiscoveryRemoteError::InternalError(format!( - "Failed to get offers from db." - ))) + Err(DiscoveryRemoteError::InternalError( + "Failed to get offers from db.".to_string(), + )) } } } diff --git a/core/market/src/matcher/resolver.rs b/core/market/src/matcher/resolver.rs index 5c52985362..f266690093 100644 --- a/core/market/src/matcher/resolver.rs +++ b/core/market/src/matcher/resolver.rs @@ -79,7 +79,7 @@ impl Resolver { .get_demands_before(offer.insertion_ts.unwrap()) .await? .into_iter() - .filter(|demand| matches(&offer, &demand)) + .filter(|demand| matches(&offer, demand)) .for_each(|demand| self.emit_proposal(offer.clone(), demand)); } Subscription::Demand(id) => { @@ -88,7 +88,7 @@ impl Resolver { .get_offers_before(demand.insertion_ts.unwrap()) .await? .into_iter() - .filter(|offer| matches(&offer, &demand)) + .filter(|offer| matches(offer, &demand)) .for_each(|offer| self.emit_proposal(offer, demand.clone())); } } diff --git a/core/market/src/matcher/store.rs b/core/market/src/matcher/store.rs index 964e767d82..4f7a2c1469 100644 --- a/core/market/src/matcher/store.rs +++ b/core/market/src/matcher/store.rs @@ -35,7 +35,7 @@ impl SubscriptionStore { let creation_ts = Utc::now().naive_utc(); // TODO: provider agent should set expiration. let expiration_ts = creation_ts + self.config.subscription.default_ttl; - let offer = Offer::from_new(offer, &id, creation_ts, expiration_ts)?; + let offer = Offer::from_new(offer, id, creation_ts, expiration_ts)?; self.insert_offer(offer).await } @@ -74,24 +74,22 @@ impl SubscriptionStore { &self, node_ids: Option>, ) -> Result, QueryOffersError> { - Ok(self - .db + self.db .as_dao::() .get_offer_ids(node_ids, Utc::now().naive_utc()) .await - .map_err(QueryOffersError::from)?) + .map_err(QueryOffersError::from) } pub async fn get_unsubscribed_offer_ids( &self, node_ids: Option>, ) -> Result, QueryOffersError> { - Ok(self - .db + self.db .as_dao::() .get_unsubscribed_ids(node_ids, Utc::now().naive_utc()) .await - .map_err(QueryOffersError::from)?) + .map_err(QueryOffersError::from) } pub async fn get_client_offers( @@ -124,24 +122,22 @@ impl SubscriptionStore { &self, ids: Vec, ) -> Result, QueryOffersError> { - Ok(self - .db + self.db .as_dao::() .get_offers(Some(ids), None, None, Utc::now().naive_utc()) .await - .map_err(QueryOffersError::from)?) + .map_err(QueryOffersError::from) } pub async fn get_offers_before( &self, inserted_before_ts: NaiveDateTime, ) -> Result, QueryOffersError> { - Ok(self - .db + self.db .as_dao::() .get_offers(None, None, Some(inserted_before_ts), Utc::now().naive_utc()) .await - .map_err(QueryOffersError::from)?) + .map_err(QueryOffersError::from) } /// Returns Offers SubscriptionId from vector, that don't exist in our database. @@ -187,7 +183,7 @@ impl SubscriptionStore { .as_dao::() .unsubscribe(id, Utc::now().naive_utc()) .await - .map_err(|e| ModifyOfferError::Unsubscribe(e.into(), id.clone())) + .map_err(|e| ModifyOfferError::Unsubscribe(e, id.clone())) .and_then(|state| match state { OfferState::Active(_) => Ok(()), OfferState::NotFound => Err(ModifyOfferError::NotFound(id.clone())), @@ -227,7 +223,7 @@ impl SubscriptionStore { } log::debug!("Removing not owned unsubscribed Offer [{}].", offer_id); - match self.db.as_dao::().delete(&offer_id).await { + match self.db.as_dao::().delete(offer_id).await { Ok(true) => Ok(()), Ok(false) => Err(ModifyOfferError::UnsubscribedNotRemoved(offer_id.clone())), Err(e) => Err(ModifyOfferError::Remove(e, offer_id.clone())), @@ -242,12 +238,12 @@ impl SubscriptionStore { let creation_ts = Utc::now().naive_utc(); // TODO: requestor agent should set expiration. let expiration_ts = creation_ts + self.config.subscription.default_ttl; - let demand = Demand::from_new(demand, &id, creation_ts, expiration_ts)?; + let demand = Demand::from_new(demand, id, creation_ts, expiration_ts)?; self.db .as_dao::() .insert(&demand) .await - .map_err(|e| DemandError::Save(e))?; + .map_err(DemandError::Save)?; Ok(demand) } @@ -284,12 +280,11 @@ impl SubscriptionStore { &self, insertion_ts: NaiveDateTime, ) -> Result, DemandError> { - Ok(self - .db + self.db .as_dao::() .get_demands(None, Some(insertion_ts), Utc::now().naive_utc()) .await - .map_err(|e| DemandError::GetMany(e))?) + .map_err(DemandError::GetMany) } pub async fn remove_demand( @@ -305,7 +300,7 @@ impl SubscriptionStore { match self .db .as_dao::() - .delete(&demand_id) + .delete(demand_id) .await .map_err(|e| DemandError::Remove(e, demand_id.clone()))? { diff --git a/core/market/src/negotiation/common.rs b/core/market/src/negotiation/common.rs index e6864c5605..d0c7a93832 100644 --- a/core/market/src/negotiation/common.rs +++ b/core/market/src/negotiation/common.rs @@ -71,10 +71,10 @@ impl CommonBroker { ) -> CommonBroker { CommonBroker { store, - db: db.clone(), - negotiation_notifier: EventNotifier::new(), + db, + negotiation_notifier: EventNotifier::default(), session_notifier, - agreement_notifier: EventNotifier::new(), + agreement_notifier: EventNotifier::default(), config, agreement_lock: AgreementLock::new(), } @@ -150,7 +150,7 @@ impl CommonBroker { // danger of data inconsistency. If we won't reject countering Proposal here, // it will be sent to Provider and his counter Proposal will be rejected later. - let proposal = self.get_proposal(subs_id.clone(), proposal_id).await?; + let proposal = self.get_proposal(subs_id, proposal_id).await?; self.validate_proposal(&proposal, caller_id, caller_role) .await?; @@ -197,7 +197,7 @@ impl CommonBroker { .take_events(subscription_id, max_events, owner) .await?; - if events.len() > 0 { + if !events.is_empty() { return Ok(events); } @@ -255,7 +255,7 @@ impl CommonBroker { .await .map_err(|e| AgreementEventsError::Internal(e.to_string()))?; - if events.len() > 0 { + if !events.is_empty() { counter!("market.agreements.events.queried", events.len() as u64); return Ok(events); } @@ -274,9 +274,10 @@ impl CommonBroker { NotifierError::ChannelClosed(_) => { Err(AgreementEventsError::Internal(error.to_string())) } - NotifierError::Unsubscribed(_) => Err(AgreementEventsError::Internal(format!( + NotifierError::Unsubscribed(_) => Err(AgreementEventsError::Internal( "Code logic error. Shouldn't get Unsubscribe in Agreement events notifier." - ))), + .to_string(), + )), }; } // Ok result means, that event with required sessionId id was added. @@ -292,10 +293,9 @@ impl CommonBroker { subs_id: Option<&SubscriptionId>, id: &ProposalId, ) -> Result { - Ok(self - .db + self.db .as_dao::() - .get_proposal(&id) + .get_proposal(id) .await .map_err(|e| GetProposalError::Internal(id.clone(), subs_id.cloned(), e.to_string()))? .filter(|proposal| { @@ -316,7 +316,7 @@ impl CommonBroker { // that such Proposal exists, but for different subscription_id. false }) - .ok_or(GetProposalError::NotFound(id.clone(), subs_id.cloned()))?) + .ok_or_else(|| GetProposalError::NotFound(id.clone(), subs_id.cloned())) } pub async fn get_client_proposal( @@ -342,11 +342,7 @@ impl CommonBroker { ) -> Result<(), AgreementError> { let dao = self.db.as_dao::(); let agreement = match dao - .select_by_node( - &client_agreement_id, - id.identity.clone(), - Utc::now().naive_utc(), - ) + .select_by_node(&client_agreement_id, id.identity, Utc::now().naive_utc()) .await .map_err(|e| AgreementError::Get(client_agreement_id.clone(), e))? { @@ -368,7 +364,7 @@ impl CommonBroker { &agreement, reason.clone(), "NotSigned".to_string(), - timestamp.clone(), + timestamp, ) .await?; @@ -379,7 +375,7 @@ impl CommonBroker { ×tamp, ) .await - .map_err(|e| AgreementError::UpdateState((&agreement.id).clone(), e))?; + .map_err(|e| AgreementError::UpdateState((agreement.id).clone(), e))?; } self.notify_agreement(&agreement).await; @@ -452,7 +448,7 @@ impl CommonBroker { .select(&agreement_id, None, Utc::now().naive_utc()) .await .map_err(|_e| RemoteAgreementError::NotFound(agreement_id.clone()))? - .ok_or(RemoteAgreementError::NotFound(agreement_id.clone()))?; + .ok_or_else(|| RemoteAgreementError::NotFound(agreement_id.clone()))?; let auth_id = match caller_role { Owner::Provider => agreement.provider_id, @@ -682,6 +678,8 @@ impl CommonBroker { }) } + //TODO https://github.com/golemfactory/yagna/issues/2247 + #[allow(clippy::unnecessary_operation)] pub async fn validate_proposal( &self, proposal: &Proposal, @@ -692,7 +690,7 @@ impl CommonBroker { Owner::Provider => &proposal.negotiation.provider_id != caller_id, Owner::Requestor => &proposal.negotiation.requestor_id != caller_id, } { - ProposalValidationError::Unauthorized(proposal.body.id.clone(), caller_id.clone()); + ProposalValidationError::Unauthorized(proposal.body.id.clone(), *caller_id); } if &proposal.issuer() == caller_id { @@ -719,7 +717,7 @@ impl CommonBroker { let session_notifier = &self.session_notifier; // Notify everyone waiting on Agreement events endpoint. - if let Some(_) = &agreement.session_id { + if agreement.session_id.is_some() { session_notifier.notify(&agreement.session_id.clone()).await; } // Even if session_id was not None, we want to notify everyone else, @@ -796,16 +794,14 @@ pub fn validate_match( | Match::Undefined { demand_mismatch, offer_mismatch, - } => { - return Err(MatchValidationError::NotMatching { - new: new_proposal.body.id.clone(), - prev: prev_proposal.body.id.clone(), - mismatches: format!( - "Mismatched constraints - Offer: {:?}, Demand: {:?}", - offer_mismatch, demand_mismatch - ), - }) - } + } => Err(MatchValidationError::NotMatching { + new: new_proposal.body.id.clone(), + prev: prev_proposal.body.id.clone(), + mismatches: format!( + "Mismatched constraints - Offer: {:?}, Demand: {:?}", + offer_mismatch, demand_mismatch + ), + }), } } @@ -820,14 +816,13 @@ pub fn validate_transition( fn get_reason_code(reason: &Option, key: &str) -> Option { reason .as_ref() - .map(|reason| { + .and_then(|reason| { reason .extra .get(key) .map(|json| json.as_str().map(|code| code.to_string())) }) .flatten() - .flatten() } /// This function extract from Reason additional information about termination reason @@ -842,7 +837,9 @@ pub fn inc_terminate_metrics(reason: &Option, owner: Owner) { let p_code = get_reason_code(reason, "golem.provider.code"); let r_code = get_reason_code(reason, "golem.requestor.code"); - let reason_code = r_code.xor(p_code).unwrap_or("NotSpecified".to_string()); + let reason_code = r_code + .xor(p_code) + .unwrap_or_else(|| "NotSpecified".to_string()); match owner { Owner::Provider => { counter!("market.agreements.provider.terminated.reason", 1, "reason" => reason_code) diff --git a/core/market/src/negotiation/error.rs b/core/market/src/negotiation/error.rs index f135e405e2..86bb1ca7c4 100644 --- a/core/market/src/negotiation/error.rs +++ b/core/market/src/negotiation/error.rs @@ -68,8 +68,12 @@ pub enum AgreementError { Get(String, AgreementDaoError), #[error("Agreement [{0}]. Error: {1}")] UpdateState(AgreementId, AgreementDaoError), + #[error("Invalid date. {0}")] + InvalidDate(#[from] chrono::ParseError), #[error("Invalid Agreement id. {0}")] InvalidId(#[from] ProposalIdParseError), + #[error("Invalid Agreement state. {0}")] + InvalidAgreementState(#[from] strum::ParseError), #[error(transparent)] Gsb(#[from] GsbAgreementError), #[error("Protocol error: {0}")] diff --git a/core/market/src/negotiation/notifier.rs b/core/market/src/negotiation/notifier.rs index 1e9dc339b8..33dc3acb9a 100644 --- a/core/market/src/negotiation/notifier.rs +++ b/core/market/src/negotiation/notifier.rs @@ -51,17 +51,23 @@ where StopEvents(Type), } -impl EventNotifier +impl Default for EventNotifier where Type: Debug + PartialEq + Clone + EnableDisplay + 'static, for<'a> DisplayEnabler<'a, Type>: std::fmt::Display, { - pub fn new() -> EventNotifier { + fn default() -> Self { // We will create receivers later, when someone needs it. let (sender, _receiver) = channel(100); - EventNotifier { sender } + Self { sender } } +} +impl EventNotifier +where + Type: Debug + PartialEq + Clone + EnableDisplay + 'static, + for<'a> DisplayEnabler<'a, Type>: std::fmt::Display, +{ pub async fn notify(&self, subscription_id: &Type) { let sender = self.sender.clone(); let to_send = Notification::::NewEvent(subscription_id.clone()); diff --git a/core/market/src/negotiation/provider.rs b/core/market/src/negotiation/provider.rs index 527f32ea0a..26209ae151 100644 --- a/core/market/src/negotiation/provider.rs +++ b/core/market/src/negotiation/provider.rs @@ -49,7 +49,7 @@ impl ProviderBroker { session_notifier: EventNotifier, config: Arc, ) -> Result { - let broker = CommonBroker::new(db.clone(), store, session_notifier, config); + let broker = CommonBroker::new(db, store, session_notifier, config); let broker1 = broker.clone(); let broker2 = broker.clone(); @@ -229,13 +229,13 @@ impl ProviderBroker { let dao = self.common.db.as_dao::(); let agreement = { - let _hold = self.common.agreement_lock.lock(&agreement_id).await; + let _hold = self.common.agreement_lock.lock(agreement_id).await; let agreement = dao .select(agreement_id, Some(id.identity), Utc::now().naive_utc()) .await .map_err(|e| AgreementError::Get(agreement_id.to_string(), e))? - .ok_or(AgreementError::NotFound(agreement_id.to_string()))?; + .ok_or_else(|| AgreementError::NotFound(agreement_id.to_string()))?; if agreement.state == AgreementState::Cancelled { return Ok(ApprovalResult::Cancelled); @@ -284,7 +284,7 @@ impl ProviderBroker { AgreementState::Cancelled, ))) => return Ok(ApprovalResult::Cancelled), Err(e) => { - let _hold = self.common.agreement_lock.lock(&agreement_id).await; + let _hold = self.common.agreement_lock.lock(agreement_id).await; log::warn!( "Failed to send Approve Agreement [{}] to Requestor. {}. Reverting state to `Pending`.", @@ -308,7 +308,7 @@ impl ProviderBroker { // is supposed to return after approval. match notifier.wait_for_event_until(stop_time).await { Err(NotifierError::Timeout(_)) => { - let _hold = self.common.agreement_lock.lock(&agreement_id).await; + let _hold = self.common.agreement_lock.lock(agreement_id).await; dao.revert_approving(agreement_id).await.log_err().ok(); Err(AgreementProtocolError::Timeout(agreement.id.clone()).into()) @@ -322,21 +322,23 @@ impl ProviderBroker { .log_err()?; { - let _hold = self.common.agreement_lock.lock(&agreement_id).await; + let _hold = self.common.agreement_lock.lock(agreement_id).await; let agreement = dao .select(agreement_id, None, Utc::now().naive_utc()) .await .map_err(|e| AgreementError::Get(agreement_id.to_string(), e))? - .ok_or(AgreementError::Internal(format!( - "Agreement [{}], which existed previously, disappeared.", - agreement_id - )))?; + .ok_or_else(|| { + AgreementError::Internal(format!( + "Agreement [{}], which existed previously, disappeared.", + agreement_id + )) + })?; match agreement.state { AgreementState::Cancelled => Ok(ApprovalResult::Cancelled), AgreementState::Approved => Ok(ApprovalResult::Approved), - AgreementState::Expired => Err(AgreementError::Expired(agreement.id.clone()))?, + AgreementState::Expired => Err(AgreementError::Expired(agreement.id))?, _ => Err(AgreementError::Internal(format!( "Agreement [{}] has unexpected state [{}]", agreement.id, agreement.state @@ -353,24 +355,24 @@ impl ProviderBroker { ) -> Result<(), AgreementError> { let dao = self.common.db.as_dao::(); let agreement = { - let _hold = self.common.agreement_lock.lock(&agreement_id).await; + let _hold = self.common.agreement_lock.lock(agreement_id).await; let agreement = dao .select(agreement_id, Some(id.identity), Utc::now().naive_utc()) .await .map_err(|e| AgreementError::Get(agreement_id.to_string(), e))? - .ok_or(AgreementError::NotFound(agreement_id.to_string()))?; + .ok_or_else(|| AgreementError::NotFound(agreement_id.to_string()))?; validate_transition(&agreement, AgreementState::Rejected)?; let timestamp = Utc::now().naive_utc(); self.api - .reject_agreement(&agreement, reason.clone(), timestamp.clone()) + .reject_agreement(&agreement, reason.clone(), timestamp) .await?; dao.reject(&agreement.id, reason.clone(), ×tamp) .await - .map_err(|e| AgreementError::UpdateState((&agreement.id).clone(), e))? + .map_err(|e| AgreementError::UpdateState((agreement.id).clone(), e))? }; counter!("market.agreements.provider.rejected", 1); @@ -544,16 +546,12 @@ async fn agreement_received( let offer_id = &offer_proposal.negotiation.offer_id.clone(); if offer_proposal.body.issuer != Issuer::Us { - return Err(RemoteProposeAgreementError::RequestorOwn( - offer_proposal_id.clone(), - )); + return Err(RemoteProposeAgreementError::RequestorOwn(offer_proposal_id)); } - let demand_proposal_id = offer_proposal - .body - .prev_proposal_id - .clone() - .ok_or_else(|| RemoteProposeAgreementError::NoNegotiations(offer_proposal_id))?; + let demand_proposal_id = offer_proposal.body.prev_proposal_id.clone().ok_or( + RemoteProposeAgreementError::NoNegotiations(offer_proposal_id), + )?; let demand_proposal = broker.get_proposal(None, &demand_proposal_id).await?; let mut agreement = Agreement::new_with_ts( @@ -586,7 +584,7 @@ async fn agreement_received( RemoteProposeAgreementError::AlreadyCountered(id) } _ => RemoteProposeAgreementError::Unexpected { - public_msg: format!("Failed to save Agreement."), + public_msg: "Failed to save Agreement.".to_string(), original_msg: e.to_string(), }, })?; @@ -599,12 +597,12 @@ async fn agreement_received( .add_agreement_event(&agreement) .await .map_err(|e| RemoteProposeAgreementError::Unexpected { - public_msg: format!("Failed to add event for Agreement."), + public_msg: "Failed to add event for Agreement.".to_string(), original_msg: e.to_string(), })?; // Send channel message to wake all query_events waiting for proposals. - broker.negotiation_notifier.notify(&offer_id).await; + broker.negotiation_notifier.notify(offer_id).await; counter!("market.agreements.provider.proposed", 1); log::info!( @@ -621,9 +619,9 @@ async fn on_agreement_cancelled( msg: AgreementCancelled, ) -> Result<(), AgreementProtocolError> { let caller: NodeId = CommonBroker::parse_caller(&caller)?; - Ok(agreement_cancelled(broker, caller, msg) + agreement_cancelled(broker, caller, msg) .await - .map_err(|e| AgreementProtocolError::Remote(e))?) + .map_err(AgreementProtocolError::Remote) } async fn agreement_cancelled( @@ -640,7 +638,7 @@ async fn agreement_cancelled( .await .log_err() .map_err(|_e| RemoteAgreementError::NotFound(msg.agreement_id.clone()))? - .ok_or(RemoteAgreementError::NotFound(msg.agreement_id.clone()))?; + .ok_or_else(|| RemoteAgreementError::NotFound(msg.agreement_id.clone()))?; if agreement.requestor_id != caller { // Don't reveal, that we know this Agreement id. @@ -648,7 +646,7 @@ async fn agreement_cancelled( } validate_transition(&agreement, AgreementState::Cancelled).map_err(|_| { - RemoteAgreementError::InvalidState(agreement.id.clone(), agreement.state.clone()) + RemoteAgreementError::InvalidState(agreement.id.clone(), agreement.state) })?; dao.cancel(&agreement.id, msg.reason.clone(), &msg.cancellation_ts) @@ -680,7 +678,7 @@ impl From for RemoteProposeAgreementError { GetProposalError::NotFound(id, ..) => RemoteProposeAgreementError::NotFound(id), GetProposalError::Internal(id, _, original_msg) => { RemoteProposeAgreementError::Unexpected { - public_msg: format!("Failed to get proposal from db [{}].", id.to_string()), + public_msg: format!("Failed to get proposal from db [{}].", id), original_msg, } } diff --git a/core/market/src/negotiation/requestor.rs b/core/market/src/negotiation/requestor.rs index 94f3bcc51d..71f4cc4b8c 100644 --- a/core/market/src/negotiation/requestor.rs +++ b/core/market/src/negotiation/requestor.rs @@ -49,7 +49,7 @@ impl RequestorBroker { session_notifier: EventNotifier, config: Arc, ) -> Result { - let broker = CommonBroker::new(db.clone(), store, session_notifier, config); + let broker = CommonBroker::new(db, store, session_notifier, config); let broker1 = broker.clone(); let broker2 = broker.clone(); @@ -317,24 +317,24 @@ impl RequestorBroker { ) -> Result<(), AgreementError> { let dao = self.common.db.as_dao::(); let agreement = { - let _hold = self.common.agreement_lock.lock(&agreement_id).await; + let _hold = self.common.agreement_lock.lock(agreement_id).await; let agreement = dao .select(agreement_id, Some(id.identity), Utc::now().naive_utc()) .await .map_err(|e| AgreementError::Get(agreement_id.to_string(), e))? - .ok_or(AgreementError::NotFound(agreement_id.to_string()))?; + .ok_or_else(|| AgreementError::NotFound(agreement_id.to_string()))?; validate_transition(&agreement, AgreementState::Cancelled)?; let timestamp = Utc::now().naive_utc(); self.api - .cancel_agreement(&agreement, reason.clone(), timestamp.clone()) + .cancel_agreement(&agreement, reason.clone(), timestamp) .await?; dao.cancel(&agreement.id, reason.clone(), ×tamp) .await - .map_err(|e| AgreementError::UpdateState((&agreement.id).clone(), e))? + .map_err(|e| AgreementError::UpdateState((agreement.id).clone(), e))? }; self.common.notify_agreement(&agreement).await; @@ -371,7 +371,7 @@ impl RequestorBroker { .select(id, None, Utc::now().naive_utc()) .await .map_err(|e| WaitForApprovalError::Get(id.clone(), e))? - .ok_or(WaitForApprovalError::NotFound(id.clone()))?; + .ok_or_else(|| WaitForApprovalError::NotFound(id.clone()))?; match agreement.state { AgreementState::Approved => { @@ -418,14 +418,10 @@ impl RequestorBroker { // We won't be able to process `on_agreement_approved`, before we // finish execution under this lock. This avoids errors related to // Provider approving Agreement before we set proper state in database. - let _hold = self.common.agreement_lock.lock(&agreement_id).await; + let _hold = self.common.agreement_lock.lock(agreement_id).await; let mut agreement = match dao - .select( - agreement_id, - Some(id.identity.clone()), - Utc::now().naive_utc(), - ) + .select(agreement_id, Some(id.identity), Utc::now().naive_utc()) .await .map_err(|e| AgreementError::Get(agreement_id.to_string(), e))? { @@ -458,7 +454,7 @@ impl RequestorBroker { &agreement_id ); } - return Ok(()); + Ok(()) } async fn query_reason_for(&self, agreement_id: &AgreementId) -> Option { @@ -485,9 +481,9 @@ async fn on_agreement_approved( msg: AgreementApproved, ) -> Result<(), AgreementProtocolError> { let caller: NodeId = CommonBroker::parse_caller(&caller)?; - Ok(agreement_approved(broker, caller, msg) + agreement_approved(broker, caller, msg) .await - .map_err(|e| AgreementProtocolError::Remote(e))?) + .map_err(AgreementProtocolError::Remote) } async fn agreement_approved( @@ -507,7 +503,7 @@ async fn agreement_approved( .select(&msg.agreement_id, None, Utc::now().naive_utc()) .await .map_err(|_e| RemoteAgreementError::NotFound(msg.agreement_id.clone()))? - .ok_or(RemoteAgreementError::NotFound(msg.agreement_id.clone()))?; + .ok_or_else(|| RemoteAgreementError::NotFound(msg.agreement_id.clone()))?; if agreement.provider_id != caller { // Don't reveal, that we know this Agreement id. @@ -515,7 +511,7 @@ async fn agreement_approved( } validate_transition(&agreement, AgreementState::Approving).map_err(|_| { - RemoteAgreementError::InvalidState(agreement.id.clone(), agreement.state.clone()) + RemoteAgreementError::InvalidState(agreement.id.clone(), agreement.state) })?; // Validate Agreement `valid_to` timestamp. In previous version we got @@ -582,7 +578,7 @@ async fn commit_agreement(broker: CommonBroker, agreement_id: AgreementId) { .select(&agreement_id, None, Utc::now().naive_utc()) .await .map_err(|_e| AgreementError::NotFound(agreement_id.to_string()))? - .ok_or(AgreementError::NotFound(agreement_id.to_string()))?; + .ok_or_else(|| AgreementError::NotFound(agreement_id.to_string()))?; // TODO: Sign Agreement. let signature = "NoSignature".to_string(); @@ -593,10 +589,9 @@ async fn commit_agreement(broker: CommonBroker, agreement_id: AgreementId) { NegotiationApi::commit_agreement(&agreement).await?; // We approve Agreement in database, when we are sure, that committing succeeded. - Ok(dao - .approve(&agreement.id, &signature) + dao.approve(&agreement.id, &signature) .await - .map_err(|e| AgreementError::UpdateState(agreement.id.clone(), e))?) + .map_err(|e| AgreementError::UpdateState(agreement.id.clone(), e)) } .await { @@ -645,9 +640,9 @@ async fn on_agreement_rejected( msg: AgreementRejected, ) -> Result<(), AgreementProtocolError> { let caller: NodeId = CommonBroker::parse_caller(&caller)?; - Ok(agreement_rejected(broker, caller, msg) + agreement_rejected(broker, caller, msg) .await - .map_err(|e| AgreementProtocolError::Remote(e))?) + .map_err(AgreementProtocolError::Remote) } async fn agreement_rejected( @@ -663,7 +658,7 @@ async fn agreement_rejected( .select(&msg.agreement_id, None, Utc::now().naive_utc()) .await .map_err(|_e| RemoteAgreementError::NotFound(msg.agreement_id.clone()))? - .ok_or(RemoteAgreementError::NotFound(msg.agreement_id.clone()))?; + .ok_or_else(|| RemoteAgreementError::NotFound(msg.agreement_id.clone()))?; if agreement.provider_id != caller { // Don't reveal, that we know this Agreement id. @@ -671,7 +666,7 @@ async fn agreement_rejected( } validate_transition(&agreement, AgreementState::Rejected).map_err(|_| { - RemoteAgreementError::InvalidState(agreement.id.clone(), agreement.state.clone()) + RemoteAgreementError::InvalidState(agreement.id.clone(), agreement.state) })?; dao.reject(&agreement.id, msg.reason.clone(), &msg.rejection_ts) @@ -703,14 +698,13 @@ pub async fn proposal_receiver_thread( ) { while let Some(proposal) = proposal_receiver.recv().await { let broker = broker.clone(); - match async move { + if let Err(error) = async move { log::debug!("Got matching Offer-Demand pair; emitting as Proposal to Requestor."); broker.generate_proposal(proposal).await } .await { - Err(error) => log::warn!("Failed to add proposal. Error: {}", error), - Ok(_) => (), + log::warn!("Failed to add proposal. Error: {}", error) } } } diff --git a/core/market/src/protocol/discovery.rs b/core/market/src/protocol/discovery.rs index 5cd087991c..3ffdb91ef7 100644 --- a/core/market/src/protocol/discovery.rs +++ b/core/market/src/protocol/discovery.rs @@ -206,7 +206,7 @@ impl Discovery { // Should never happen, but just to be certain. if offer_ids.is_empty() { - return (); + return; } let default_id = match self.default_identity().await { Ok(id) => id, @@ -244,7 +244,7 @@ impl Discovery { ServiceBinder::new(&get_offers_addr(public_prefix), &(), self.clone()).bind_with_processor( move |_, myself, caller: String, msg: RetrieveOffers| { - let myself = myself.clone(); + let myself = myself; myself.on_get_remote_offers(caller, msg) }, ); @@ -260,7 +260,7 @@ impl Discovery { if self.re_broadcast_enabled() { self.bind_gsb_broadcast().await.map_or_else( |e| { - log::warn!("Failed to subscribe to broadcasts. Error: {:?}.", e,); + log::warn!("Failed to subscribe to broadcasts. Error: {e}."); }, |_| (), ); @@ -386,7 +386,7 @@ impl Discovery { ) -> Result, DiscoveryRemoteError> { log::trace!("[{}] asks for {} Offers.", &caller, msg.offer_ids.len()); let get_local_offers = self.inner.get_local_offers_handler.clone(); - Ok(get_local_offers.call(caller, msg).await?) + get_local_offers.call(caller, msg).await } async fn on_bcast_unsubscribes( @@ -427,7 +427,7 @@ impl Discovery { } async fn default_identity(&self) -> Result { - Ok(self.inner.identity.default_identity().await?) + self.inner.identity.default_identity().await } } diff --git a/core/market/src/protocol/discovery/builder.rs b/core/market/src/protocol/discovery/builder.rs index d93e895a2b..4484e76b65 100644 --- a/core/market/src/protocol/discovery/builder.rs +++ b/core/market/src/protocol/discovery/builder.rs @@ -180,7 +180,7 @@ mod test { let discovery = DiscoveryBuilder::default() .add_data(MockIdentity::new("test") as Arc) - .add_data(7 as usize) + .add_data(7_usize) .add_data("mock data") .add_handler(|_, _: OffersRetrieved| async { Ok(vec![]) }) .add_handler(|_, _: RetrieveOffers| async { panic!("should not be invoked") }) diff --git a/core/market/src/protocol/negotiation/provider.rs b/core/market/src/protocol/negotiation/provider.rs index 534ee00ef4..751656ec3e 100644 --- a/core/market/src/protocol/negotiation/provider.rs +++ b/core/market/src/protocol/negotiation/provider.rs @@ -123,10 +123,10 @@ impl NegotiationApi { signature: agreement .approved_signature .clone() - .ok_or(AgreementProtocolError::NotSigned(id.clone()))?, + .ok_or_else(|| AgreementProtocolError::NotSigned(id.clone()))?, approved_ts: agreement .approved_ts - .ok_or(AgreementProtocolError::NoApprovalTimestamp(id.clone()))?, + .ok_or_else(|| AgreementProtocolError::NoApprovalTimestamp(id.clone()))?, }; let net_send_fut = net::from(agreement.provider_id) .to(agreement.requestor_id) @@ -290,34 +290,34 @@ impl NegotiationApi { ServiceBinder::new(&provider::proposal_addr(public_prefix), &(), self.clone()) .bind_with_processor( move |_, myself, caller: String, msg: InitialProposalReceived| { - let myself = myself.clone(); + let myself = myself; myself.on_initial_proposal_received(caller, msg) }, ) .bind_with_processor(move |_, myself, caller: String, msg: ProposalReceived| { - let myself = myself.clone(); + let myself = myself; myself.on_proposal_received(caller, msg) }) .bind_with_processor(move |_, myself, caller: String, msg: ProposalRejected| { - let myself = myself.clone(); + let myself = myself; myself.on_proposal_rejected(caller, msg) }); ServiceBinder::new(&provider::agreement_addr(public_prefix), &(), self.clone()) .bind_with_processor(move |_, myself, caller: String, msg: AgreementReceived| { - let myself = myself.clone(); + let myself = myself; myself.on_agreement_received(caller, msg) }) .bind_with_processor(move |_, myself, caller: String, msg: AgreementCancelled| { - let myself = myself.clone(); + let myself = myself; myself.on_agreement_cancelled(caller, msg) }) .bind_with_processor(move |_, myself, caller: String, msg: AgreementTerminated| { - let myself = myself.clone(); + let myself = myself; myself.on_agreement_terminated(caller, msg) }) .bind_with_processor(move |_, myself, caller: String, msg: AgreementCommitted| { - let myself = myself.clone(); + let myself = myself; myself.on_agreement_committed(caller, msg) }); Ok(()) diff --git a/core/market/src/protocol/negotiation/requestor.rs b/core/market/src/protocol/negotiation/requestor.rs index 54f24606e5..faa50e9147 100644 --- a/core/market/src/protocol/negotiation/requestor.rs +++ b/core/market/src/protocol/negotiation/requestor.rs @@ -137,21 +137,19 @@ impl NegotiationApi { &self, agreement: &Agreement, ) -> Result<(), ProposeAgreementError> { - let requestor_id = agreement.requestor_id.clone(); - let provider_id = agreement.provider_id.clone(); let id = agreement.id.clone(); let msg = AgreementReceived { agreement_id: agreement.id.clone(), - valid_to: agreement.valid_to.clone(), - creation_ts: agreement.creation_ts.clone(), + valid_to: agreement.valid_to, + creation_ts: agreement.creation_ts, proposal_id: agreement.offer_proposal_id.clone(), signature: agreement .proposed_signature .clone() - .ok_or(ProposeAgreementError::NotSigned(id.clone()))?, + .ok_or_else(|| ProposeAgreementError::NotSigned(id.clone()))?, }; - net::from(requestor_id) - .to(provider_id) + net::from(agreement.requestor_id) + .to(agreement.provider_id) .service(&provider::agreement_addr(BUS_ID)) .send(msg) .map_err(|e| GsbAgreementError(e.to_string(), id)) @@ -160,18 +158,16 @@ impl NegotiationApi { } pub async fn commit_agreement(agreement: &Agreement) -> Result<(), CommitAgreementError> { - let requestor_id = agreement.requestor_id.clone(); - let provider_id = agreement.provider_id.clone(); let id = agreement.id.clone(); let msg = AgreementCommitted { agreement_id: agreement.id.clone(), signature: agreement .committed_signature .clone() - .ok_or(CommitAgreementError::NotSigned(id.clone()))?, + .ok_or_else(|| CommitAgreementError::NotSigned(id.clone()))?, }; - net::from(requestor_id) - .to(provider_id) + net::from(agreement.requestor_id) + .to(agreement.provider_id) .service(&provider::agreement_addr(BUS_ID)) .send(msg) .map_err(|e| GsbAgreementError(e.to_string(), id)) @@ -290,25 +286,25 @@ impl NegotiationApi { ServiceBinder::new(&requestor::proposal_addr(public_prefix), &(), self.clone()) .bind_with_processor(move |_, myself, caller: String, msg: ProposalReceived| { - let myself = myself.clone(); + let myself = myself; myself.on_proposal_received(caller, msg) }) .bind_with_processor(move |_, myself, caller: String, msg: ProposalRejected| { - let myself = myself.clone(); + let myself = myself; myself.on_proposal_rejected(caller, msg) }); ServiceBinder::new(&requestor::agreement_addr(public_prefix), &(), self.clone()) .bind_with_processor(move |_, myself, caller: String, msg: AgreementApproved| { - let myself = myself.clone(); + let myself = myself; myself.on_agreement_approved(caller, msg) }) .bind_with_processor(move |_, myself, caller: String, msg: AgreementRejected| { - let myself = myself.clone(); + let myself = myself; myself.on_agreement_rejected(caller, msg) }) .bind_with_processor(move |_, myself, caller: String, msg: AgreementTerminated| { - let myself = myself.clone(); + let myself = myself; myself.on_agreement_terminated(caller, msg) }); Ok(()) diff --git a/core/market/src/rest_api.rs b/core/market/src/rest_api.rs index 539dd158df..8854e09de0 100644 --- a/core/market/src/rest_api.rs +++ b/core/market/src/rest_api.rs @@ -9,7 +9,7 @@ use actix_web::{error::InternalError, http::StatusCode, web::PathConfig}; use chrono::{DateTime, Utc}; use serde::Deserialize; -use ya_client::model::ErrorMessage; +use ya_client::model::{market::agreement::State, ErrorMessage}; use crate::db::model::{ AgreementId, AppSessionId, Owner, ProposalId, ProposalIdParseError, SubscriptionId, @@ -59,6 +59,15 @@ pub struct PathSubscriptionProposal { pub proposal_id: ProposalId, } +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct QueryAgreementList { + pub state: Option, + pub before_date: Option>, + pub after_date: Option>, + pub app_session_id: Option, +} + #[derive(Deserialize)] pub struct QueryAppSessionId { #[serde(rename = "appSessionId")] @@ -127,7 +136,7 @@ pub(crate) fn default_event_timeout() -> f32 { } impl PathAgreement { - pub fn to_id(self, owner: Owner) -> Result { + pub fn to_id(&self, owner: Owner) -> Result { AgreementId::from_client(&self.agreement_id, owner) } } diff --git a/core/market/src/rest_api/common.rs b/core/market/src/rest_api/common.rs index 0f22824a3e..be23208f22 100644 --- a/core/market/src/rest_api/common.rs +++ b/core/market/src/rest_api/common.rs @@ -11,15 +11,36 @@ use super::PathAgreement; use crate::db::model::Owner; use crate::market::MarketService; use crate::negotiation::error::AgreementError; -use crate::rest_api::QueryAgreementEvents; +use crate::rest_api::{QueryAgreementEvents, QueryAgreementList}; pub fn register_endpoints(scope: Scope) -> Scope { scope + .service(list_agreements) .service(collect_agreement_events) .service(get_agreement) .service(terminate_agreement) } +#[actix_web::get("/agreements")] +async fn list_agreements( + market: Data>, + query: Query, + id: Identity, +) -> impl Responder { + let query = query.into_inner(); + + market + .list_agreements( + &id, + query.state.map(Into::into), + query.before_date, + query.after_date, + query.app_session_id, + ) + .await + .map(|list| HttpResponse::Ok().json(list)) +} + #[actix_web::get("/agreements/{agreement_id}")] async fn get_agreement( market: Data>, @@ -30,7 +51,7 @@ async fn get_agreement( // and check, if any will be returned. Note that we won't get Agreement if we aren't // owner, so here is no danger, that Provider gets Requestor's Offer and opposite. let path = path.into_inner(); - let r_agreement_id = path.clone().to_id(Owner::Requestor)?; + let r_agreement_id = path.to_id(Owner::Requestor)?; let p_agreement_id = r_agreement_id.clone().swap_owner(); let r_result = market.get_agreement(&r_agreement_id, &id).await; @@ -44,7 +65,7 @@ async fn get_agreement( r_result.map(|agreement| HttpResponse::Ok().json(agreement)) } else { // Both calls shouldn't return Agreement. - Err(AgreementError::Internal(format!("We found "))) + Err(AgreementError::Internal("We found ".to_string())) } } @@ -57,7 +78,7 @@ async fn collect_agreement_events( let timeout: f32 = query.timeout; let after_timestamp = query .after_timestamp - .unwrap_or(Utc.ymd(2016, 11, 11).and_hms(15, 12, 0)); + .unwrap_or_else(|| Utc.ymd(2016, 11, 11).and_hms(15, 12, 0)); market .query_agreement_events( diff --git a/core/market/src/rest_api/error.rs b/core/market/src/rest_api/error.rs index 9f1ea53336..34b401c0c5 100644 --- a/core/market/src/rest_api/error.rs +++ b/core/market/src/rest_api/error.rs @@ -21,7 +21,7 @@ use crate::{ impl From for actix_web::HttpResponse { fn from(e: MarketError) -> Self { - e.error_response().into() + e.error_response() } } @@ -54,10 +54,7 @@ impl ResponseError for NegotiationError {} impl ResponseError for ResolverError { fn error_response(&self) -> HttpResponse { - match self { - _ => HttpResponse::InternalServerError().json(ErrorMessage::new(self.to_string())), - } - .into() + HttpResponse::InternalServerError().json(ErrorMessage::new(self.to_string())) } } @@ -69,23 +66,18 @@ impl ResponseError for DemandError { } _ => HttpResponse::InternalServerError().json(ErrorMessage::new(self.to_string())), } - .into() } } impl ResponseError for QueryDemandsError { fn error_response(&self) -> HttpResponse { - HttpResponse::InternalServerError() - .json(ErrorMessage::new(self.to_string())) - .into() + HttpResponse::InternalServerError().json(ErrorMessage::new(self.to_string())) } } impl ResponseError for QueryOffersError { fn error_response(&self) -> HttpResponse { - HttpResponse::InternalServerError() - .json(ErrorMessage::new(self.to_string())) - .into() + HttpResponse::InternalServerError().json(ErrorMessage::new(self.to_string())) } } @@ -96,7 +88,6 @@ impl ResponseError for QueryOfferError { QueryOfferError::NotFound(_) => HttpResponse::NotFound().json(msg), _ => HttpResponse::InternalServerError().json(msg), } - .into() } } @@ -109,7 +100,6 @@ impl ResponseError for SaveOfferError { } _ => HttpResponse::InternalServerError().json(msg), } - .into() } } @@ -123,7 +113,6 @@ impl ResponseError for ModifyOfferError { } _ => HttpResponse::InternalServerError().json(msg), } - .into() } } @@ -140,7 +129,6 @@ impl ResponseError for QueryEventsError { } _ => HttpResponse::InternalServerError().json(msg), } - .into() } } @@ -150,12 +138,12 @@ impl ResponseError for ProposalError { match self { ProposalError::Validation(e) => e.error_response(), ProposalError::Save(SaveProposalError::AlreadyCountered(..)) => { - HttpResponse::Gone().json(msg).into() + HttpResponse::Gone().json(msg) } ProposalError::Get(e) => e.error_response(), ProposalError::Reject(e) => e.error_response(), // TODO: get rid of those `_` patterns as they do not break when error is extended - _ => HttpResponse::InternalServerError().json(msg).into(), + _ => HttpResponse::InternalServerError().json(msg), } } } @@ -170,7 +158,6 @@ impl ResponseError for RejectProposalError { | RejectProposalError::ChangeState(_) | RejectProposalError::CallerParse(_) => HttpResponse::InternalServerError().json(msg), } - .into() } } @@ -186,7 +173,6 @@ impl ResponseError for ProposalValidationError { ProposalValidationError::Unauthorized(_, _) => HttpResponse::Unauthorized().json(msg), ProposalValidationError::Internal(_) => HttpResponse::InternalServerError().json(msg), } - .into() } } @@ -197,7 +183,6 @@ impl ResponseError for GetProposalError { GetProposalError::NotFound(..) => HttpResponse::NotFound().json(msg), _ => HttpResponse::InternalServerError().json(msg), } - .into() } } @@ -205,18 +190,18 @@ impl ResponseError for AgreementError { fn error_response(&self) -> HttpResponse { let msg = ErrorMessage::new(self.to_string()); match self { - AgreementError::NotFound(_) => HttpResponse::NotFound().json(msg).into(), - AgreementError::Expired(_) => HttpResponse::Gone().json(msg).into(), - AgreementError::ProposalAlreadyAccepted(..) => { - HttpResponse::Conflict().json(msg).into() - } + AgreementError::NotFound(_) => HttpResponse::NotFound().json(msg), + AgreementError::Expired(_) => HttpResponse::Gone().json(msg), + AgreementError::ProposalAlreadyAccepted(..) => HttpResponse::Conflict().json(msg), AgreementError::UpdateState(_, e) => e.error_response(), AgreementError::NoNegotiations(_) | AgreementError::ProposalRejected(..) | AgreementError::OwnProposal(..) | AgreementError::ProposalNotFound(..) | AgreementError::ProposalCountered(..) - | AgreementError::InvalidId(..) => HttpResponse::BadRequest().json(msg).into(), + | AgreementError::InvalidDate(..) + | AgreementError::InvalidAgreementState(..) + | AgreementError::InvalidId(..) => HttpResponse::BadRequest().json(msg), AgreementError::GetProposal(..) | AgreementError::Save(..) | AgreementError::Get(..) @@ -225,7 +210,7 @@ impl ResponseError for AgreementError { | AgreementError::Protocol(_) | AgreementError::ProtocolTerminate(_) | AgreementError::ProtocolCommit(_) - | AgreementError::Internal(_) => HttpResponse::InternalServerError().json(msg).into(), + | AgreementError::Internal(_) => HttpResponse::InternalServerError().json(msg), } } } @@ -249,7 +234,6 @@ impl ResponseError for AgreementDaoError { | AgreementDaoError::SessionId(_) | AgreementDaoError::EventError(_) => HttpResponse::InternalServerError().json(msg), } - .into() } } @@ -267,7 +251,6 @@ impl ResponseError for WaitForApprovalError { HttpResponse::InternalServerError().json(msg) } } - .into() } } @@ -278,6 +261,5 @@ impl ResponseError for AgreementEventsError { AgreementEventsError::InvalidMaxEvents(..) => HttpResponse::BadRequest().json(msg), AgreementEventsError::Internal(_) => HttpResponse::InternalServerError().json(msg), } - .into() } } diff --git a/core/market/src/testing/agreement_utils.rs b/core/market/src/testing/agreement_utils.rs index 8dde1f0884..6e3075635f 100644 --- a/core/market/src/testing/agreement_utils.rs +++ b/core/market/src/testing/agreement_utils.rs @@ -33,6 +33,7 @@ pub async fn negotiate_agreement( .await } +#[allow(clippy::too_many_arguments)] pub async fn negotiate_agreement_with_ids( network: &MarketsNetwork, req_name: &str, diff --git a/core/market/src/testing/backtrace_util.rs b/core/market/src/testing/backtrace_util.rs index 26436fac5d..b79db2d74e 100644 --- a/core/market/src/testing/backtrace_util.rs +++ b/core/market/src/testing/backtrace_util.rs @@ -9,22 +9,20 @@ fn get_innermost_backtrace_symbol(fm: &backtrace::BacktraceFrame) -> Option Option { // On some systems backtrace lib doesn't properly set actual_start_index - let mut idx = 0; - for frame in frames.iter() { + for (idx, frame) in frames.iter().enumerate() { if let Some(name) = get_innermost_backtrace_symbol(frame) { // Note: On windows there is no "::" suffix if name.starts_with("ya_market::testing::backtrace_util::generate_backtraced_name") { return Some(idx); } } - idx += 1; } None } fn get_symbol_at_level(bt: &backtrace::Backtrace, lvl: usize) -> Option { let frames = &bt.frames(); - match adjust_backtrace_level(&frames) { + match adjust_backtrace_level(frames) { Some(adjustment) => { let frame = &frames[lvl + adjustment]; return get_innermost_backtrace_symbol(frame); @@ -43,7 +41,7 @@ pub fn generate_backtraced_name(level: Option) -> String { log::trace!("Generated name: {} level: {:?} BT: {:#?}", name, level, bt); return name; } - let u4 = uuid::Uuid::new_v4().to_string().to_string(); + let u4 = uuid::Uuid::new_v4().to_string(); log::error!( "No backtrace support. Generating default name from UUIDv4. uuid4={}, bt={:#?}", u4, diff --git a/core/market/src/testing/bcast.rs b/core/market/src/testing/bcast.rs index 3315c8fcce..8cfd20f520 100644 --- a/core/market/src/testing/bcast.rs +++ b/core/market/src/testing/bcast.rs @@ -43,7 +43,7 @@ impl BCast for BCastService { let me = self.inner.borrow(); me.topics .get(topic) - .map(|receivers| receivers.iter().map(|endpoint| endpoint.clone()).collect()) + .map(|receivers| receivers.to_vec()) .unwrap_or_default() } } diff --git a/core/market/src/testing/events_helper.rs b/core/market/src/testing/events_helper.rs index ff690e3f57..63e2d1812c 100644 --- a/core/market/src/testing/events_helper.rs +++ b/core/market/src/testing/events_helper.rs @@ -64,7 +64,7 @@ pub mod requestor { ) -> anyhow::Result { let events = market .requestor_engine - .query_events(&demand_id, QUERY_EVENTS_TIMEOUT, Some(5)) + .query_events(demand_id, QUERY_EVENTS_TIMEOUT, Some(5)) .await?; expect_proposal(events, stage) } @@ -127,7 +127,7 @@ pub mod provider { ) -> anyhow::Result { let events = market .provider_engine - .query_events(&offer_id, QUERY_EVENTS_TIMEOUT, Some(5)) + .query_events(offer_id, QUERY_EVENTS_TIMEOUT, Some(5)) .await?; expect_proposal(events, stage) } diff --git a/core/market/src/testing/mock_identity.rs b/core/market/src/testing/mock_identity.rs index ce405e76d4..3232bb52dc 100644 --- a/core/market/src/testing/mock_identity.rs +++ b/core/market/src/testing/mock_identity.rs @@ -38,7 +38,7 @@ impl MockIdentity { let mut identities = HashMap::new(); identities .entry(name.to_string()) - .or_insert(default.clone()); + .or_insert_with(|| default.clone()); let mock_identity = MockIdentityInner { default, diff --git a/core/market/src/testing/mock_net.rs b/core/market/src/testing/mock_net.rs index cf00bdc8d9..99fdb1acb6 100644 --- a/core/market/src/testing/mock_net.rs +++ b/core/market/src/testing/mock_net.rs @@ -48,14 +48,14 @@ impl MockNet { pub fn register_node(&self, node_id: &NodeId, prefix: &str) { // Only two first components - let mut iter = prefix.split("/").fuse(); + let mut iter = prefix.split('/').fuse(); let prefix = match (iter.next(), iter.next(), iter.next()) { (Some(""), Some(test_name), Some(name)) => format!("/{}/{}", test_name, name), _ => panic!("[MockNet] Can't register prefix {}", prefix), }; let mut inner = self.inner.lock().unwrap(); - if let Some(_) = inner.nodes.insert(node_id.clone(), prefix) { + if inner.nodes.insert(*node_id, prefix).is_some() { panic!("[MockNet] Node [{}] already existed.", &node_id); } } @@ -66,7 +66,7 @@ impl MockNet { .nodes .remove(node_id) .map(|_| ()) - .ok_or(anyhow::anyhow!("node not registered: {}", node_id)) + .ok_or_else(|| anyhow::anyhow!("node not registered: {}", node_id)) } async fn translate_address(&self, address: String) -> Result<(NodeId, String)> { @@ -75,19 +75,19 @@ impl MockNet { Err(e) => Err(Error::GsbBadRequest(e.to_string()))?, }; - let mut iter = to_addr.split("/").fuse(); + let mut iter = to_addr.split('/').fuse(); let dst_id = match (iter.next(), iter.next(), iter.next()) { (Some(""), Some("net"), Some(dst_id)) => dst_id, _ => panic!("[MockNet] Invalid destination address {}", to_addr), }; - let dest_node_id = NodeId::from_str(&dst_id)?; + let dest_node_id = NodeId::from_str(dst_id)?; let inner = self.inner.lock().unwrap(); let local_prefix = inner.nodes.get(&dest_node_id); if let Some(local_prefix) = local_prefix { let net_prefix = format!("/net/{}", dst_id); - Ok((from_node, to_addr.replacen(&net_prefix, &local_prefix, 1))) + Ok((from_node, to_addr.replacen(&net_prefix, local_prefix, 1))) } else { Err(Error::GsbFailure(format!( "[MockNet] Can't find destination address for endpoint [{}].", @@ -100,7 +100,7 @@ impl MockNet { let inner = self.inner.lock().unwrap(); for (id, prefix) in inner.nodes.iter() { if address.contains(prefix) { - return Some(id.clone()); + return Some(*id); } } None @@ -141,7 +141,7 @@ impl MockNetInner { let stub: SendBroadcastStub = serialization::from_slice(msg).unwrap(); let caller = caller.to_string(); - let msg = msg.iter().copied().collect::>(); + let msg = msg.to_vec(); let topic = stub.topic; let endpoints = bcast.resolve(&caller, &topic); @@ -199,7 +199,7 @@ impl MockNetInner { &caller, &local_addr ); - Ok(local_bus::send(&local_addr, &from.to_string(), &data).await?) + local_bus::send(&local_addr, &from.to_string(), &data).await } }, (), @@ -209,14 +209,14 @@ impl MockNetInner { // Copied from core/net/api.rs pub(crate) fn parse_from_addr(from_addr: &str) -> Result<(NodeId, String)> { - let mut it = from_addr.split("/").fuse(); + let mut it = from_addr.split('/').fuse(); if let (Some(""), Some("from"), Some(from_node_id), Some("to"), Some(to_node_id)) = (it.next(), it.next(), it.next(), it.next(), it.next()) { to_node_id.parse::()?; let prefix = 10 + from_node_id.len(); let service_id = &from_addr[prefix..]; - if let Some(_) = it.next() { + if it.next().is_some() { return Ok((from_node_id.parse()?, net_service(service_id))); } } diff --git a/core/market/src/testing/mock_node.rs b/core/market/src/testing/mock_node.rs index c83b04a39c..2477c08a4e 100644 --- a/core/market/src/testing/mock_node.rs +++ b/core/market/src/testing/mock_node.rs @@ -1,3 +1,5 @@ +#![allow(clippy::too_many_arguments)] + use actix_http::body::BoxBody; use actix_http::Request; use actix_service::Service as ActixService; @@ -106,10 +108,7 @@ impl MockNodeKind { fn testname_from_backtrace(bn: &str) -> String { log::info!("Test name to regex match: {}", &bn); // Extract test name - let captures = Regex::new(r"(.*)::(.*)::.*") - .unwrap() - .captures(&bn) - .unwrap(); + let captures = Regex::new(r"(.*)::(.*)::.*").unwrap().captures(bn).unwrap(); let filename = captures.get(1).unwrap().as_str().to_string(); let testname = captures.get(2).unwrap().as_str().to_string(); @@ -168,7 +167,7 @@ impl MarketsNetwork { kind: node_kind, }; - let node_id = node.mock_identity.get_default_id().clone().identity; + let node_id = node.mock_identity.get_default_id().identity; log::info!("Creating mock node {}: [{}].", name, &node_id); BCastService::default().register(&node_id, &self.test_name); MockNet::default().register_node(&node_id, &public_gsb_prefix); @@ -414,33 +413,31 @@ impl MarketsNetwork { pub fn get_default_id(&self, node_name: &str) -> Identity { self.nodes .iter() - .find(|node| &node.name == node_name) + .find(|node| node.name == node_name) .map(|node| node.mock_identity.clone()) .unwrap() .get_default_id() - .clone() } pub fn create_identity(&self, node_name: &str, id_name: &str) -> Identity { let mock_identity = self .nodes .iter() - .find(|node| &node.name == node_name) + .find(|node| node.name == node_name) .map(|node| node.mock_identity.clone()) .unwrap(); let id = mock_identity.new_identity(id_name); - let node_id = id.identity.clone(); let (public_gsb_prefix, _) = gsb_prefixes(&self.test_name, node_name); - MockNet::default().register_node(&node_id, &public_gsb_prefix); - return id; + MockNet::default().register_node(&id.identity, &public_gsb_prefix); + id } pub fn list_ids(&self, node_name: &str) -> HashMap { self.nodes .iter() - .find(|node| &node.name == node_name) + .find(|node| node.name == node_name) .map(|node| node.mock_identity.list_ids()) .unwrap() } @@ -512,7 +509,7 @@ fn test_data_dir() -> PathBuf { fn escape_path(path: &str) -> String { // Windows can't handle colons - path.replace("::", "_").to_string() + path.replace("::", "_") } pub fn prepare_test_dir(dir_name: &str) -> Result { @@ -718,10 +715,10 @@ where { let subscriptions = subscriptions.into_iter(); let mut all_broadcasted = false; - 'retry: for _i in 0..10 { + 'retry: for _i in 0..30 { for subscription in subscriptions.clone() { for mkt in mkts { - if mkt.get_offer(&subscription).await.is_err() { + if mkt.get_offer(subscription).await.is_err() { // Every 150ms we should get at least one broadcast from each Node. // After a few tries all nodes should have the same knowledge about Offers. tokio::time::sleep(Duration::from_millis(250)).await; @@ -751,7 +748,7 @@ where for subscription in subscriptions.clone() { for mkt in mkts { let expect_error = QueryOfferError::Unsubscribed(subscription.clone()).to_string(); - match mkt.get_offer(&subscription).await { + match mkt.get_offer(subscription).await { Err(e) => assert_eq!(e.to_string(), expect_error), Ok(_) => { // Every 150ms we should get at least one broadcast from each Node. diff --git a/core/market/src/testing/proposal_util.rs b/core/market/src/testing/proposal_util.rs index 4b90484f89..69245a7e75 100644 --- a/core/market/src/testing/proposal_util.rs +++ b/core/market/src/testing/proposal_util.rs @@ -94,8 +94,8 @@ pub async fn exchange_proposals_exclusive_with_ids( prov_name, &exclusive_offer(match_on), &exclusive_demand(match_on), - &req_id, - &prov_id, + req_id, + prov_id, ) .await } @@ -133,8 +133,8 @@ pub async fn exchange_proposals_impl( let req_mkt = network.get_market(req_name); let prov_mkt = network.get_market(prov_name); - let demand_id = req_mkt.subscribe_demand(demand, &req_id).await?; - let offer_id = prov_mkt.subscribe_offer(offer, &prov_id).await?; + let demand_id = req_mkt.subscribe_demand(demand, req_id).await?; + let offer_id = prov_mkt.subscribe_offer(offer, prov_id).await?; // Expect events generated on requestor market. let req_offer_proposal1 = requestor::query_proposal(&req_mkt, &demand_id, "Initial #R").await?; @@ -145,8 +145,8 @@ pub async fn exchange_proposals_impl( .counter_proposal( &demand_id, &req_offer_proposal1.proposal_id.parse()?, - &demand, - &req_id, + demand, + req_id, ) .await?; @@ -158,7 +158,7 @@ pub async fn exchange_proposals_impl( // Provider counters proposal. let _offer_proposal_id = prov_mkt .provider_engine - .counter_proposal(&offer_id, &prov_demand_proposal1_id, &offer, &prov_id) + .counter_proposal(&offer_id, &prov_demand_proposal1_id, offer, prov_id) .await?; // Requestor receives proposal. diff --git a/core/market/src/utils/agreement_lock.rs b/core/market/src/utils/agreement_lock.rs index 4254374a73..872c215d94 100644 --- a/core/market/src/utils/agreement_lock.rs +++ b/core/market/src/utils/agreement_lock.rs @@ -19,20 +19,14 @@ impl AgreementLock { pub async fn lock(&self, agreement_id: &AgreementId) -> OwnedMutexGuard<()> { // Note how important are '{}' around this statement. Otherwise lock isn't freed // and we can't acquire write lock - let potencial_lock = { - self.lock_map - .read() - .await - .get(agreement_id) - .map(|lock| lock.clone()) - }; - match potencial_lock { + let potential_lock = { self.lock_map.read().await.get(agreement_id).cloned() }; + match potential_lock { Some(mutex) => mutex, None => { let mut lock_map = self.lock_map.write().await; lock_map .entry(agreement_id.clone()) - .or_insert(Arc::new(Mutex::new(()))) + .or_insert_with(|| Arc::new(Mutex::new(()))) .clone() } } diff --git a/core/market/tests/test_agreement.rs b/core/market/tests/test_agreement.rs index 1af042c311..b73a3b5cc4 100644 --- a/core/market/tests/test_agreement.rs +++ b/core/market/tests/test_agreement.rs @@ -1,7 +1,8 @@ use actix_web::{http::StatusCode, web::Bytes}; use chrono::{Duration, Utc}; -use ya_core_model::{market, Role}; +use ya_client::model::market::Role; +use ya_core_model::market; use ya_market::assert_err_eq; use ya_market::testing::{ agreement_utils::{gen_reason, negotiate_agreement}, @@ -67,6 +68,51 @@ async fn test_gsb_get_agreement() { assert_eq!(agreement.app_session_id, sess_id); } +#[cfg_attr(not(feature = "test-suite"), ignore)] +#[serial_test::serial] +async fn test_gsb_list_agreements() { + let network = MarketsNetwork::new(None) + .await + .add_market_instance(REQ_NAME) + .await + .add_market_instance(PROV_NAME) + .await; + + let proposal_id = exchange_draft_proposals(&network, REQ_NAME, PROV_NAME) + .await + .unwrap() + .proposal_id; + let req_market = network.get_market(REQ_NAME); + let req_engine = &req_market.requestor_engine; + let req_id = network.get_default_id(REQ_NAME); + + let agreement_id = req_engine + .create_agreement( + req_id.clone(), + &proposal_id, + Utc::now() + Duration::hours(1), + ) + .await + .unwrap(); + + // than: confirm agreement with app_session_id + let sess_id = Some("sess-iksde".into()); + req_engine + .confirm_agreement(req_id.clone(), &agreement_id, sess_id.clone()) + .await + .unwrap(); + + let agreements = bus::service(network.node_gsb_prefixes(REQ_NAME).0) + .send(market::ListAgreements::default()) + .await + .unwrap() + .unwrap(); + + assert_eq!(agreements.len(), 1); + assert_eq!(agreements[0].id, agreement_id.into_client()); + assert_eq!(agreements[0].role, Role::Requestor); +} + #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_get_agreement() { @@ -1190,7 +1236,7 @@ async fn test_terminate_invalid_reason() { .unwrap() .r_agreement; - let mut app = network.get_rest_app(REQ_NAME).await; + let app = network.get_rest_app(REQ_NAME).await; let url = format!( "/market-api/v1/agreements/{}/terminate", agreement_id.into_client(), @@ -1202,7 +1248,7 @@ async fn test_terminate_invalid_reason() { .set_payload(Bytes::copy_from_slice(reason.as_bytes())) .to_request(); - let resp = actix_web::test::call_service(&mut app, req).await; + let resp = actix_web::test::call_service(&app, req).await; assert_eq!(resp.status(), StatusCode::BAD_REQUEST); let reason = "{'no_message_field': 'Reason expects message field'}".to_string(); @@ -1211,6 +1257,6 @@ async fn test_terminate_invalid_reason() { .set_payload(Bytes::copy_from_slice(reason.as_bytes())) .to_request(); - let resp = actix_web::test::call_service(&mut app, req).await; + let resp = actix_web::test::call_service(&app, req).await; assert_eq!(resp.status(), StatusCode::BAD_REQUEST); } diff --git a/core/market/tests/test_agreement_events.rs b/core/market/tests/test_agreement_events.rs index 77c17cf9e4..d28ed94007 100644 --- a/core/market/tests/test_agreement_events.rs +++ b/core/market/tests/test_agreement_events.rs @@ -49,7 +49,7 @@ async fn test_agreement_approved_event() { // Provider will approve agreement after some delay. let agr_id = agreement_id.clone(); - let from_timestamp = confirm_timestamp.clone(); + let from_timestamp = confirm_timestamp; let query_handle = tokio::task::spawn_local(async move { tokio::time::sleep(std::time::Duration::from_millis(20)).await; prov_market @@ -374,7 +374,7 @@ async fn test_agreement_rejected_event() { // Provider will approve agreement after some delay. let agr_id = agreement_id.clone(); - let from_timestamp = confirm_timestamp.clone(); + let from_timestamp = confirm_timestamp; let query_handle = tokio::task::spawn_local(async move { tokio::time::sleep(std::time::Duration::from_millis(20)).await; prov_market diff --git a/core/market/tests/test_app_session_id.rs b/core/market/tests/test_app_session_id.rs index 618d7b2e89..3e01c19ebd 100644 --- a/core/market/tests/test_app_session_id.rs +++ b/core/market/tests/test_app_session_id.rs @@ -44,11 +44,7 @@ async fn test_session_events_filtering() { let mut agreements = vec![]; for proposal_id in proposals.iter() { let agreement_id = req_engine - .create_agreement( - req_id.clone(), - &proposal_id, - Utc::now() + Duration::hours(1), - ) + .create_agreement(req_id.clone(), proposal_id, Utc::now() + Duration::hours(1)) .await .unwrap(); agreements.push(agreement_id); @@ -64,7 +60,7 @@ async fn test_session_events_filtering() { let confirm_timestamp = Utc::now(); for (agreement_id, session_id) in agreements.iter().zip(sessions.iter()) { req_engine - .confirm_agreement(req_id.clone(), &agreement_id, Some(session_id.clone())) + .confirm_agreement(req_id.clone(), agreement_id, Some(session_id.clone())) .await .unwrap(); } @@ -582,7 +578,7 @@ async fn test_common_event_flow() { // Use max_events to query one event at the time. let mut current_timestamp = timestamp_before; - for i in 0..agreements.len() { + for agreement in &agreements { let events = req_market .query_agreement_events( &Some("r-session".to_string()), @@ -595,7 +591,7 @@ async fn test_common_event_flow() { .unwrap(); assert_eq!(events.len(), 1); - assert_eq!(events[0].agreement_id, agreements[i].into_client()); + assert_eq!(events[0].agreement_id, agreement.into_client()); match &events[0].event_type { AgreementEventType::AgreementApprovedEvent {} => (), @@ -604,7 +600,7 @@ async fn test_common_event_flow() { e ), } - current_timestamp = events[0].event_date.clone(); + current_timestamp = events[0].event_date; } // We don't expect any events anymore. diff --git a/core/market/tests/test_cleaner.rs b/core/market/tests/test_cleaner.rs index 19c9cb974d..7b1db860c3 100644 --- a/core/market/tests/test_cleaner.rs +++ b/core/market/tests/test_cleaner.rs @@ -1,4 +1,5 @@ use chrono::{Duration, NaiveDateTime, Utc}; +use std::ops::Not; use structopt::StructOpt; use ya_market::testing::cleaner::clean; @@ -40,32 +41,30 @@ async fn test_agreement() { // create agreement event in 3 steps agreement_dao - .confirm(&expired_agreement.id, &None, &"signature,".to_string()) + .confirm(&expired_agreement.id, &None, "signature,") .await .unwrap(); agreement_dao .approving( &expired_agreement.id, &None, - &"signature,".to_string(), + "signature,", &Utc::now().naive_utc(), ) .await .unwrap(); agreement_dao - .approve(&expired_agreement.id, &"signature,".to_string()) + .approve(&expired_agreement.id, "signature,") .await .unwrap(); clean(db.clone(), &db_config()).await; - assert_eq!( - >::exists(&db.disk_db.pool, valid_agreement.id).await, - true - ); - assert_eq!( - >::exists(&db.disk_db.pool, expired_agreement.id).await, - false + assert!( + >::exists(&db.disk_db.pool, valid_agreement.id).await ); + assert!(Not::not( + >::exists(&db.disk_db.pool, expired_agreement.id).await + )); } #[cfg_attr(not(feature = "test-suite"), ignore)] @@ -85,14 +84,10 @@ async fn test_demand() { demand_dao.insert(&valid_demand).await.unwrap(); demand_dao.insert(&expired_demand).await.unwrap(); clean(db.clone(), &db_config()).await; - assert_eq!( - >::exists(&db.ram_db.pool, valid_demand.id).await, - true - ); - assert_eq!( - >::exists(&db.ram_db.pool, expired_demand.id).await, - false - ); + assert!(>::exists(&db.ram_db.pool, valid_demand.id).await); + assert!(Not::not( + >::exists(&db.ram_db.pool, expired_demand.id).await + )); } #[cfg_attr(not(feature = "test-suite"), ignore)] @@ -111,22 +106,18 @@ async fn test_offer() { let offer_dao = db.as_dao::(); let validation_ts = (Utc::now() - Duration::days(100)).naive_utc(); offer_dao - .put(valid_offer.clone(), validation_ts.clone()) + .put(valid_offer.clone(), validation_ts) .await .unwrap(); offer_dao - .put(expired_offer.clone(), validation_ts.clone()) + .put(expired_offer.clone(), validation_ts) .await .unwrap(); clean(db.clone(), &db_config()).await; - assert_eq!( - >::exists(&db.ram_db.pool, valid_offer.id).await, - true - ); - assert_eq!( - >::exists(&db.ram_db.pool, expired_offer.id).await, - false - ); + assert!(>::exists(&db.ram_db.pool, valid_offer.id).await); + assert!(Not::not( + >::exists(&db.ram_db.pool, expired_offer.id).await + )); } #[cfg_attr(not(feature = "test-suite"), ignore)] @@ -143,14 +134,12 @@ async fn test_events() { .await .unwrap(); clean(db.clone(), &db_config()).await; - assert_eq!( - >::exists(&db.ram_db.pool, valid_event.id).await, - true - ); - assert_eq!( - >::exists(&db.ram_db.pool, expired_event.id).await, - false + assert!( + >::exists(&db.ram_db.pool, valid_event.id).await ); + assert!(Not::not( + >::exists(&db.ram_db.pool, expired_event.id).await + )); } #[cfg_attr(not(feature = "test-suite"), ignore)] @@ -192,26 +181,20 @@ async fn test_proposal() { expired_proposals.push(proposal.clone()); } clean(db.clone(), &db_config()).await; - assert_eq!( - >::exists(&db.ram_db.pool, valid_negotiation.id).await, - true + assert!( + >::exists(&db.ram_db.pool, valid_negotiation.id).await ); for proposal in valid_proposals.into_iter() { - assert_eq!( - >::exists(&db.ram_db.pool, proposal.id).await, - true - ); + assert!(>::exists(&db.ram_db.pool, proposal.id).await); } - assert_eq!( + assert!(Not::not( >::exists(&db.ram_db.pool, expired_negotiation.id) - .await, - false - ); + .await + )); for proposal in expired_proposals.into_iter() { - assert_eq!( - >::exists(&db.ram_db.pool, proposal.id).await, - false - ); + assert!(Not::not( + >::exists(&db.ram_db.pool, proposal.id).await + )); } } @@ -241,9 +224,8 @@ async fn test_proposal_lotsa_negotiations() { } clean(db.clone(), &db_config()).await; for n in expired_negotiations { - assert_eq!( - >::exists(&db.ram_db.pool, n.id).await, - false - ); + assert!(Not::not( + >::exists(&db.ram_db.pool, n.id).await + )); } } diff --git a/core/market/tests/test_initial_proposal.rs b/core/market/tests/test_initial_proposal.rs index 893a577e6e..2afbc2f345 100644 --- a/core/market/tests/test_initial_proposal.rs +++ b/core/market/tests/test_initial_proposal.rs @@ -264,7 +264,7 @@ async fn test_query_events_edge_cases() { .await .unwrap() .unwrap(); - if events.len() > 0 { + if !events.is_empty() { assert_eq!(events.len(), 1); break; } diff --git a/core/market/tests/test_negotiations.rs b/core/market/tests/test_negotiations.rs index 7e2f68c610..e251ea47f8 100644 --- a/core/market/tests/test_negotiations.rs +++ b/core/market/tests/test_negotiations.rs @@ -772,20 +772,20 @@ async fn test_reject_initial_offer() { req_mkt .requestor_engine - .reject_proposal(&demand_id, &proposal0id, &req_id, Some("dblah".into())) + .reject_proposal(&demand_id, proposal0id, &req_id, Some("dblah".into())) .await - .map_err(|e| panic!("Expected Ok(()), got: {}\nDEBUG: {:?}", e.to_string(), e)) + .map_err(|e| panic!("Expected Ok(()), got: {}\nDEBUG: {:?}", e, e)) .unwrap(); req_mkt .requestor_engine .query_events(&demand_id, 1.2, Some(5)) .await - .map_err(|e| panic!("Expected Ok([]), got: {}\nDEBUG: {:?}", e.to_string(), e)) + .map_err(|e| panic!("Expected Ok([]), got: {}\nDEBUG: {:?}", e, e)) .map(|events| assert_eq!(events.len(), 0)) .unwrap(); - let proposal0updated = req_mkt.get_proposal(&proposal0id).await.unwrap(); + let proposal0updated = req_mkt.get_proposal(proposal0id).await.unwrap(); assert_eq!(proposal0updated.body.state, ProposalState::Rejected); } @@ -822,7 +822,7 @@ async fn test_reject_demand() { let req_demand_proposal1_id = req_mkt .requestor_engine - .counter_proposal(&demand_id, &proposal0id, &demand, &req_id) + .counter_proposal(&demand_id, proposal0id, &demand, &req_id) .await .unwrap(); @@ -849,7 +849,7 @@ async fn test_reject_demand() { .requestor_engine .query_events(&demand_id, 1.2, Some(5)) .await - .map_err(|e| panic!("Expected Ok([ev]), got: {}\nDEBUG: {:?}", e.to_string(), e)) + .map_err(|e| panic!("Expected Ok([ev]), got: {}\nDEBUG: {:?}", e, e)) .map(|events| { assert_eq!(events.len(), 1); match &events[0] { @@ -939,11 +939,11 @@ async fn test_proposal_events_last() { assert_eq!(events.len(), 2); match events[0] { RequestorEvent::ProposalRejectedEvent { .. } => {} - _ => assert!(false, "Invalid first event_type: {:#?}", events[0]), + _ => panic!("Invalid first event_type: {:#?}", events[0]), } match events[events.len() - 1] { RequestorEvent::ProposalEvent { .. } => {} - _ => assert!(false, "Invalid last event_type: {:#?}", events[0]), + _ => panic!("Invalid last event_type: {:#?}", events[0]), } } diff --git a/core/market/tests/test_rest_api.rs b/core/market/tests/test_rest_api.rs index 7459579abe..37127ab1dd 100644 --- a/core/market/tests/test_rest_api.rs +++ b/core/market/tests/test_rest_api.rs @@ -75,12 +75,12 @@ async fn test_rest_get_offers() { assert_offers_broadcasted(&[&market_remote], &[subscription_id_local.clone()]).await; - let mut app = network.get_rest_app("Node-1").await; + let app = network.get_rest_app("Node-1").await; let req = actix_web::test::TestRequest::get() .uri("/market-api/v1/offers") .to_request(); - let resp = actix_web::test::call_service(&mut app, req).await; + let resp = actix_web::test::call_service(&app, req).await; assert_eq!(resp.status(), StatusCode::OK); let result: Vec = read_response_json(resp).await; @@ -104,12 +104,12 @@ async fn test_rest_get_demands() { .unwrap(); let demand_local = market_local.get_demand(&subscription_id).await.unwrap(); - let mut app = network.get_rest_app("Node-1").await; + let app = network.get_rest_app("Node-1").await; let req = actix_web::test::TestRequest::get() .uri("/market-api/v1/demands") .to_request(); - let resp = actix_web::test::call_service(&mut app, req).await; + let resp = actix_web::test::call_service(&app, req).await; assert_eq!(resp.status(), StatusCode::OK); let result: Vec = read_response_json(resp).await; assert_eq!(vec![demand_local.into_client_demand().unwrap()], result); @@ -123,14 +123,14 @@ async fn test_rest_invalid_subscription_id_should_return_400() { .await .add_market_instance("Node-1") .await; - let mut app = network.get_rest_app("Node-1").await; + let app = network.get_rest_app("Node-1").await; let req = actix_web::test::TestRequest::delete() .uri("/market-api/v1/offers/invalid-id") .to_request(); // when - let resp = actix_web::test::call_service(&mut app, req).await; + let resp = actix_web::test::call_service(&app, req).await; // then assert_eq!(resp.status(), StatusCode::BAD_REQUEST); @@ -154,7 +154,7 @@ async fn test_rest_subscribe_unsubscribe_offer() { .await .add_market_instance("Node-1") .await; - let mut app = network.get_rest_app("Node-1").await; + let app = network.get_rest_app("Node-1").await; let client_offer = sample_offer(); @@ -164,7 +164,7 @@ async fn test_rest_subscribe_unsubscribe_offer() { .to_request(); // when create offer - let resp = actix_web::test::call_service(&mut app, req).await; + let resp = actix_web::test::call_service(&app, req).await; // then assert_eq!(resp.status(), StatusCode::CREATED); @@ -188,7 +188,7 @@ async fn test_rest_subscribe_unsubscribe_offer() { .uri(&format!("/market-api/v1/offers/{}", subscription_id)) .to_request(); // when unsubscribe - let resp = actix_web::test::call_service(&mut app, req).await; + let resp = actix_web::test::call_service(&app, req).await; // then assert_eq!(resp.status(), StatusCode::NO_CONTENT); @@ -197,7 +197,7 @@ async fn test_rest_subscribe_unsubscribe_offer() { .uri(&format!("/market-api/v1/offers/{}", subscription_id)) .to_request(); // when unsubscribe again - let resp = actix_web::test::call_service(&mut app, req).await; + let resp = actix_web::test::call_service(&app, req).await; // then assert_eq!(resp.status(), StatusCode::GONE); let result: ErrorMessage = read_response_json(resp).await; @@ -216,7 +216,7 @@ async fn test_rest_subscribe_unsubscribe_demand() { .await .add_market_instance("Node-1") .await; - let mut app = network.get_rest_app("Node-1").await; + let app = network.get_rest_app("Node-1").await; let client_demand = sample_demand(); @@ -226,7 +226,7 @@ async fn test_rest_subscribe_unsubscribe_demand() { .to_request(); // when - let resp = actix_web::test::call_service(&mut app, req).await; + let resp = actix_web::test::call_service(&app, req).await; // then assert_eq!(resp.status(), StatusCode::CREATED); @@ -253,7 +253,7 @@ async fn test_rest_subscribe_unsubscribe_demand() { .uri(&format!("/market-api/v1/demands/{}", subscription_id)) .to_request(); // when - let resp = actix_web::test::call_service(&mut app, req).await; + let resp = actix_web::test::call_service(&app, req).await; // then assert_eq!(resp.status(), StatusCode::NO_CONTENT); @@ -262,7 +262,7 @@ async fn test_rest_subscribe_unsubscribe_demand() { .uri(&format!("/market-api/v1/demands/{}", subscription_id)) .to_request(); // when - let resp = actix_web::test::call_service(&mut app, req).await; + let resp = actix_web::test::call_service(&app, req).await; // then assert_eq!(resp.status(), StatusCode::NOT_FOUND); let result: ErrorMessage = read_response_json(resp).await; @@ -300,7 +300,7 @@ async fn test_rest_get_proposal() { .unwrap() .into_client() .unwrap(); - let mut app = network.get_rest_app("Provider").await; + let app = network.get_rest_app("Provider").await; let req_offers = actix_web::test::TestRequest::get() .uri( @@ -311,7 +311,7 @@ async fn test_rest_get_proposal() { .as_str(), ) .to_request(); - let resp_offers = actix_web::test::call_service(&mut app, req_offers).await; + let resp_offers = actix_web::test::call_service(&app, req_offers).await; assert_eq!(resp_offers.status(), StatusCode::OK); let result_offers: Proposal = read_response_json(resp_offers).await; assert_eq!(proposal, result_offers); @@ -325,7 +325,7 @@ async fn test_rest_get_proposal() { .as_str(), ) .to_request(); - let resp_demands = actix_web::test::call_service(&mut app, req_demands).await; + let resp_demands = actix_web::test::call_service(&app, req_demands).await; assert_eq!(resp_demands.status(), StatusCode::OK); let resp_demands: Proposal = read_response_json(resp_demands).await; assert_eq!(proposal, resp_demands); @@ -355,14 +355,14 @@ async fn test_rest_get_agreement() { .await .unwrap(); - let mut app = network.get_rest_app("Node-1").await; + let app = network.get_rest_app("Node-1").await; let req = actix_web::test::TestRequest::get() .uri(&format!( "/market-api/v1/agreements/{}", agreement_id.into_client() )) .to_request(); - let resp = actix_web::test::call_service(&mut app, req).await; + let resp = actix_web::test::call_service(&app, req).await; assert_eq!(resp.status(), StatusCode::OK); let agreement: Agreement = read_response_json(resp).await; @@ -407,7 +407,7 @@ async fn test_rest_query_agreement_events() { let after_timestamp = negotiation.confirm_timestamp; - let mut app = network.get_rest_app("Node-1").await; + let app = network.get_rest_app("Node-1").await; let url = format!( "/market-api/v1/agreementEvents?{}", QueryParamsBuilder::new() @@ -417,7 +417,7 @@ async fn test_rest_query_agreement_events() { .build() ); let req = actix_web::test::TestRequest::get().uri(&url).to_request(); - let resp = actix_web::test::call_service(&mut app, req).await; + let resp = actix_web::test::call_service(&app, req).await; assert_eq!(resp.status(), StatusCode::OK); let events: Vec = read_response_json(resp).await; @@ -463,8 +463,8 @@ async fn test_terminate_agreement() { .uri(&url) .set_json(&reason) .to_request(); - let mut app = network.get_rest_app(REQ_NAME).await; - let resp = actix_web::test::call_service(&mut app, req).await; + let app = network.get_rest_app(REQ_NAME).await; + let resp = actix_web::test::call_service(&app, req).await; assert_eq!(resp.status(), StatusCode::OK); @@ -523,8 +523,8 @@ async fn test_terminate_agreement_without_reason() { .uri(&url) .set_json(&reason) .to_request(); - let mut app = network.get_rest_app(REQ_NAME).await; - let resp = actix_web::test::call_service(&mut app, req).await; + let app = network.get_rest_app(REQ_NAME).await; + let resp = actix_web::test::call_service(&app, req).await; assert_eq!(resp.status(), StatusCode::OK); @@ -592,8 +592,8 @@ async fn test_rest_agreement_rejected() { .uri(&url) .set_json(&Some(gen_reason("Not-interested"))) .to_request(); - let mut app = network.get_rest_app(PROV_NAME).await; - let resp = actix_web::test::call_service(&mut app, req).await; + let app = network.get_rest_app(PROV_NAME).await; + let resp = actix_web::test::call_service(&app, req).await; assert_eq!(resp.status(), StatusCode::OK); @@ -654,8 +654,8 @@ async fn test_rest_agreement_cancelled() { .uri(&url) .set_json(&Some(gen_reason("Changed my mind"))) .to_request(); - let mut app = network.get_rest_app(REQ_NAME).await; - let resp = actix_web::test::call_service(&mut app, req).await; + let app = network.get_rest_app(REQ_NAME).await; + let resp = actix_web::test::call_service(&app, req).await; assert_eq!(resp.status(), StatusCode::OK); diff --git a/core/metrics/Cargo.toml b/core/metrics/Cargo.toml index 763ec7b380..d74f2fd8d4 100644 --- a/core/metrics/Cargo.toml +++ b/core/metrics/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ya-metrics" -version = "0.1.0" +version = "0.2.0" description="Yagna metrics service" keywords=["golem", "yagna", "metrics"] authors = ["Golem Factory "] @@ -10,10 +10,10 @@ repository = "https://github.com/golemfactory/yagna" license = "LGPL-3.0" [dependencies] -ya-core-model = { version = "^0.6", features = ["identity"] } +ya-core-model = { version = "^0.8", features = ["identity"] } ya-service-api = "0.1" -ya-service-api-interfaces = "0.1" -ya-service-bus = "0.4" +ya-service-api-interfaces = "0.2" +ya-service-bus = "0.6" awc = "3" actix-web = { version = "4", features = ["openssl"] } diff --git a/core/metrics/src/exporter.rs b/core/metrics/src/exporter.rs index b8596fd8ec..bbee25c361 100644 --- a/core/metrics/src/exporter.rs +++ b/core/metrics/src/exporter.rs @@ -29,6 +29,6 @@ where pub fn turn(&mut self) -> String { let mut observer = self.builder.build(); self.controller.observe(&mut observer); - return observer.drain(); + observer.drain() } } diff --git a/core/metrics/src/metrics.rs b/core/metrics/src/metrics.rs index fdc7cb05cf..67bd571222 100644 --- a/core/metrics/src/metrics.rs +++ b/core/metrics/src/metrics.rs @@ -16,7 +16,12 @@ impl Metrics { .build() .expect("Metrics initialization failure"); let root_sink = receiver.sink(); - let exporter = StringExporter::new(receiver.controller(), PrometheusBuilder::new()); + let exporter = StringExporter::new( + receiver.controller(), + PrometheusBuilder::new().set_quantiles(&[ + 0.0, 0.01, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99, 0.999, + ]), + ); receiver.install(); Arc::new(Mutex::new(Self { @@ -32,6 +37,6 @@ impl Metrics { } pub fn export(&mut self) -> String { - return self.exporter.turn(); + self.exporter.turn() } } diff --git a/core/metrics/src/pusher.rs b/core/metrics/src/pusher.rs index cbb975cf64..8c9fb1e64f 100644 --- a/core/metrics/src/pusher.rs +++ b/core/metrics/src/pusher.rs @@ -15,12 +15,12 @@ pub fn spawn(ctx: MetricsCtx) { log::debug!("Starting metrics pusher"); tokio::task::spawn_local(async move { - push_forever(ctx.push_host_url.unwrap().as_str()).await; + push_forever(ctx.push_host_url.unwrap().as_str(), &ctx.job).await; }); log::info!("Metrics pusher started"); } -pub async fn push_forever(host_url: &str) { +pub async fn push_forever(host_url: &str, job: &str) { let node_identity = match try_get_default_id().await { Ok(default_id) => default_id, Err(e) => { @@ -31,7 +31,7 @@ pub async fn push_forever(host_url: &str) { return; } }; - let push_url = match get_push_url(host_url, &node_identity) { + let push_url = match get_push_url(host_url, &node_identity, job) { Ok(url) => url, Err(e) => { log::warn!( @@ -91,7 +91,7 @@ async fn get_default_id() -> anyhow::Result { let default_id = bus::service(identity::BUS_ID) .call(identity::Get::ByDefault) .await?? - .ok_or(anyhow::anyhow!("Default identity not found"))?; + .ok_or_else(|| anyhow::anyhow!("Default identity not found"))?; Ok(default_id) } @@ -108,20 +108,20 @@ async fn try_get_default_id() -> anyhow::Result { } } } - Err(last_error.unwrap_or(anyhow::anyhow!("Undefined error"))) + Err(last_error.unwrap_or_else(|| anyhow::anyhow!("Undefined error"))) } -fn get_push_url(host_url: &str, id: &IdentityInfo) -> anyhow::Result { +fn get_push_url(host_url: &str, id: &IdentityInfo, job: &str) -> anyhow::Result { let base = url::Url::parse(host_url)?; let url = base - .join("/metrics/job/community.1/")? + .join(&format!("/metrics/job/{job}/"))? .join(&format!("instance/{}/", &id.node_id))? .join(&format!( "hostname/{}", id.alias .as_ref() .map(|alias| utf8_percent_encode(alias, NON_ALPHANUMERIC).to_string()) - .unwrap_or(id.node_id.to_string()) + .unwrap_or_else(|| id.node_id.to_string()) ))?; Ok(String::from(url.as_str())) } @@ -141,6 +141,7 @@ mod test { is_locked: false, is_default: false, }, + "community.1", ) .unwrap(); assert_eq!("http://a/metrics/job/community.1/instance/0x0000000000000000000000000000000000000000/hostname/ala%2Fma%2Fkota", url); @@ -156,6 +157,7 @@ mod test { is_locked: false, is_default: false, }, + "community.1", ) .unwrap(); assert_eq!("http://a/metrics/job/community.1/instance/0x0000000000000000000000000000000000000000/hostname/za%C5%BC%C3%B3%C5%82%C4%87%3Fg%C4%99%C5%9Bl%C4%85%21ja%C5%BA%C5%84%3D", url); diff --git a/core/metrics/src/service.rs b/core/metrics/src/service.rs index a130318f80..aacc85c0ad 100644 --- a/core/metrics/src/service.rs +++ b/core/metrics/src/service.rs @@ -25,6 +25,9 @@ pub struct MetricsPusherOpts { default_value = DEFAULT_YAGNA_METRICS_URL, )] pub metrics_push_url: Url, + /// Metrics job name, which allows to distinguish different groups of Nodes. + #[structopt(long, env = "YAGNA_METRICS_JOB_NAME", default_value = "community.1")] + pub metrics_job_name: String, } impl From<&MetricsPusherOpts> for MetricsCtx { @@ -32,6 +35,7 @@ impl From<&MetricsPusherOpts> for MetricsCtx { MetricsCtx { push_enabled: !opts.disable_metrics_push, push_host_url: Some(opts.metrics_push_url.clone()), + job: opts.metrics_job_name.clone(), } } } diff --git a/core/model/Cargo.toml b/core/model/Cargo.toml index 072bed2c49..a94b19d3cd 100644 --- a/core/model/Cargo.toml +++ b/core/model/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ya-core-model" -version = "0.6.0" +version = "0.8.0" authors = ["Golem Factory "] edition = "2018" homepage = "https://github.com/golemfactory/yagna" @@ -35,18 +35,18 @@ sgx = ['graphene-sgx'] version = [] [dependencies] -ya-client-model = "0.4" -ya-service-bus = "0.4" +ya-client-model = "0.5" +ya-service-bus = "0.6" bigdecimal = { version = "0.2", features = ["serde"], optional = true } bitflags = { version = "1.2", optional = true } chrono = { version = "0.4", features = ["serde"] } derive_more = "0.99.11" -graphene-sgx = { version = "0.4", optional = true } +graphene-sgx = { version = "0.3.3", optional = true } log = "0.4" serde = { version = "1.0", features = ["derive"] } serde_bytes = "0.11.3" structopt = "0.3" -strum = "0.20" -strum_macros = "0.20" +strum = "0.24" +strum_macros = "0.24" thiserror = "1.0.9" diff --git a/core/model/src/activity.rs b/core/model/src/activity.rs index 53035ce5b7..d5dcbd953a 100644 --- a/core/model/src/activity.rs +++ b/core/model/src/activity.rs @@ -5,7 +5,6 @@ use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; -use crate::Role; use ya_client_model::activity::{ ActivityState, ActivityUsage, ExeScriptCommand, ExeScriptCommandResult, ExeScriptCommandState, RuntimeEvent, @@ -53,14 +52,14 @@ impl RpcMessage for Create { type Error = RpcMessageError; } -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct CreateResponse { pub activity_id: String, pub credentials: Option, } -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(untagged)] pub enum CreateResponseCompat { ActivityId(String), @@ -145,7 +144,7 @@ impl RpcMessage for GetUsage { } /// Update remote network configuration -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub enum VpnControl { AddNodes { @@ -181,7 +180,7 @@ impl RpcMessage for VpnControl { } /// Network data -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct VpnPacket(pub Vec); @@ -247,7 +246,7 @@ impl RpcMessage for GetExecBatchResults { } /// Stream script execution events. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct StreamExecBatchResults { pub activity_id: String, pub batch_id: String, @@ -280,18 +279,19 @@ pub mod local { use super::*; use chrono::{DateTime, Utc}; use std::collections::BTreeMap; + use ya_client_model::market::Role; /// Local activity bus address. pub const BUS_ID: &str = "/local/activity"; /// Set state of the activity. - #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Stats { pub identity: NodeId, } - #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct StatsResult { pub total: BTreeMap, @@ -337,7 +337,7 @@ pub mod local { type Error = RpcMessageError; } - #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub enum Credentials { Sgx { diff --git a/core/model/src/appkey.rs b/core/model/src/appkey.rs index e8c47a919d..425a02ac10 100644 --- a/core/model/src/appkey.rs +++ b/core/model/src/appkey.rs @@ -4,7 +4,7 @@ use thiserror::Error; use ya_client_model::NodeId; use ya_service_bus::RpcMessage; -pub const BUS_ID: &'static str = "/local/appkey"; +pub const BUS_ID: &str = "/local/appkey"; pub const DEFAULT_ROLE: &str = "manager"; @@ -24,6 +24,13 @@ impl Error { message: e.to_string(), } } + + pub fn bad_request(e: impl std::fmt::Display) -> Self { + Self { + code: 400, + message: e.to_string(), + } + } } #[derive(Clone, Debug, Serialize, Deserialize)] diff --git a/core/model/src/driver.rs b/core/model/src/driver.rs index 27824faebc..a2f25ccbdc 100644 --- a/core/model/src/driver.rs +++ b/core/model/src/driver.rs @@ -260,7 +260,7 @@ impl Init { self.token.clone() } pub fn mode(&self) -> AccountMode { - self.mode.clone() + self.mode } } @@ -315,7 +315,7 @@ impl SchedulePayment { } pub fn due_date(&self) -> DateTime { - self.due_date.clone() + self.due_date } } @@ -452,6 +452,7 @@ pub struct Transfer { pub gasless: bool, } +#[allow(clippy::too_many_arguments)] impl Transfer { pub fn new( sender: String, diff --git a/core/model/src/identity.rs b/core/model/src/identity.rs index 0a83021e3b..edeab66713 100644 --- a/core/model/src/identity.rs +++ b/core/model/src/identity.rs @@ -3,7 +3,7 @@ use thiserror::Error; use ya_client_model::NodeId; use ya_service_bus::RpcMessage; -pub const BUS_ID: &'static str = "/local/identity"; +pub const BUS_ID: &str = "/local/identity"; #[derive(Clone, Debug, Serialize, Deserialize)] pub struct Ack {} @@ -20,6 +20,8 @@ pub enum Error { InternalErr(String), #[error("bad keystore format: {0}")] BadKeyStoreFormat(String), + #[error("invalid password")] + InvalidPassword, } impl Error { @@ -211,6 +213,18 @@ impl RpcMessage for Subscribe { type Error = Error; } +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Unsubscribe { + pub endpoint: String, +} + +impl RpcMessage for Unsubscribe { + const ID: &'static str = "Unsubscribe"; + type Item = Ack; + type Error = Error; +} + #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetPubKey(pub NodeId); diff --git a/core/model/src/lib.rs b/core/model/src/lib.rs index eee62f1e95..48d782597d 100644 --- a/core/model/src/lib.rs +++ b/core/model/src/lib.rs @@ -31,12 +31,4 @@ pub mod sgx; #[cfg(feature = "version")] pub mod version; -use derive_more::Display; -use serde::{Deserialize, Serialize}; pub use ya_client_model::NodeId; - -#[derive(Clone, Copy, Debug, Display, PartialEq, Serialize, Deserialize)] -pub enum Role { - Provider, - Requestor, -} diff --git a/core/model/src/market.rs b/core/model/src/market.rs index cf270539b0..5454ea8df6 100644 --- a/core/model/src/market.rs +++ b/core/model/src/market.rs @@ -1,8 +1,9 @@ //! Market service bus API. +use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; -use crate::Role; -pub use ya_client_model::market::Agreement; +use ya_client_model::market::{agreement::State, Role}; +pub use ya_client_model::market::{Agreement, AgreementListEntry}; use ya_service_bus::RpcMessage; /// Public Market bus address. @@ -41,6 +42,22 @@ impl RpcMessage for GetAgreement { type Error = RpcMessageError; } +/// Lists all agreements +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub struct ListAgreements { + pub state: Option, + pub before_date: Option>, + pub after_date: Option>, + pub app_session_id: Option, +} + +impl RpcMessage for ListAgreements { + const ID: &'static str = "ListAgreements"; + type Item = Vec; + type Error = RpcMessageError; +} + /// Error message for market service bus API. #[derive(thiserror::Error, Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] diff --git a/core/model/src/net.rs b/core/model/src/net.rs index f01e89775c..803ddf404f 100644 --- a/core/model/src/net.rs +++ b/core/model/src/net.rs @@ -7,6 +7,7 @@ use ya_service_bus::typed::Endpoint; pub const BUS_ID: &str = "/net"; pub const BUS_ID_UDP: &str = "/udp/net"; +pub const BUS_ID_TRANSFER: &str = "/transfer/net"; // TODO: replace with dedicated endpoint/service descriptor with enum for visibility pub const PUBLIC_PREFIX: &str = "/public"; @@ -181,6 +182,7 @@ pub mod local { pub seen: Duration, pub duration: Duration, pub ping: Duration, + pub metrics: StatusMetrics, } #[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq, Hash)] @@ -204,6 +206,28 @@ pub mod local { pub metrics: StatusMetrics, } + #[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq, Hash)] + #[serde(rename_all = "camelCase")] + pub struct FindNode { + pub node_id: String, + } + + impl RpcMessage for FindNode { + const ID: &'static str = "FindNode"; + type Item = FindNodeResponse; + type Error = StatusError; + } + + #[derive(Clone, Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct FindNodeResponse { + pub identities: Vec, + pub endpoints: Vec, + pub seen: u32, + pub slot: u32, + pub encryption: Vec, + } + /// Measures time between sending GSB message and getting response. /// This is different from session ping, because it takes into account /// Virtual TCP overhead. Moreover we can measure ping between Nodes @@ -302,6 +326,11 @@ pub fn net_service_udp(service: impl ToString) -> String { format!("{}/{}", BUS_ID_UDP, service.to_string()) } +#[inline] +pub fn net_transfer_service(service: impl ToString) -> String { + format!("{}/{}", BUS_ID_TRANSFER, service.to_string()) +} + fn extract_exported_part(local_service_addr: &str) -> &str { assert!(local_service_addr.starts_with(PUBLIC_PREFIX)); &local_service_addr[PUBLIC_PREFIX.len()..] @@ -310,6 +339,7 @@ fn extract_exported_part(local_service_addr: &str) -> &str { pub trait RemoteEndpoint { fn service(&self, bus_addr: &str) -> bus::Endpoint; fn service_udp(&self, bus_addr: &str) -> bus::Endpoint; + fn service_transfer(&self, bus_addr: &str) -> bus::Endpoint; } impl RemoteEndpoint for NodeId { @@ -328,6 +358,14 @@ impl RemoteEndpoint for NodeId { extract_exported_part(bus_addr) )) } + + fn service_transfer(&self, bus_addr: &str) -> Endpoint { + bus::service(format!( + "{}{}", + net_transfer_service(self), + extract_exported_part(bus_addr) + )) + } } impl RemoteEndpoint for NetDst { @@ -348,6 +386,15 @@ impl RemoteEndpoint for NetDst { extract_exported_part(bus_addr) )) } + + fn service_transfer(&self, bus_addr: &str) -> Endpoint { + bus::service(format!( + "/transfer/from/{}/to/{}{}", + self.src, + self.dst, + extract_exported_part(bus_addr) + )) + } } #[cfg(test)] @@ -402,4 +449,15 @@ mod tests { "/net/0xbabe000000000000000000000000000000000000".to_string() ); } + + #[test] + fn test_transfer_service() { + let node_id: NodeId = "0xbabe000000000000000000000000000000000000" + .parse() + .unwrap(); + assert_eq!( + node_id.service_transfer("/public/zima/x").addr(), + "/transfer/net/0xbabe000000000000000000000000000000000000/zima/x" + ); + } } diff --git a/core/model/src/payment.rs b/core/model/src/payment.rs index 7cc87ec909..f5c9cb0207 100644 --- a/core/model/src/payment.rs +++ b/core/model/src/payment.rs @@ -26,11 +26,11 @@ pub mod local { use std::time::Duration; use structopt::*; use strum::{EnumProperty, VariantNames}; - use strum_macros::{Display, EnumProperty, EnumString, EnumVariantNames, IntoStaticStr}; + use strum_macros::{Display, EnumString, EnumVariantNames, IntoStaticStr}; use ya_client_model::NodeId; - pub const BUS_ID: &'static str = "/local/payment"; + pub const BUS_ID: &str = "/local/payment"; pub const DEFAULT_PAYMENT_DRIVER: &str = "erc20"; #[derive(Clone, Debug, Serialize, Deserialize)] @@ -435,11 +435,12 @@ pub mod local { EnumString, EnumVariantNames, IntoStaticStr, - EnumProperty, - Display, + strum_macros::EnumProperty, + strum_macros::Display, Debug, Clone, PartialEq, + Eq, Serialize, Deserialize, )] @@ -468,6 +469,7 @@ pub mod local { Debug, Clone, PartialEq, + Eq, Serialize, Deserialize, )] @@ -529,7 +531,7 @@ pub mod public { use super::*; use ya_client_model::NodeId; - pub const BUS_ID: &'static str = "/public/payment"; + pub const BUS_ID: &str = "/public/payment"; #[derive(Clone, Debug, Serialize, Deserialize)] pub struct Ack {} @@ -613,7 +615,7 @@ pub mod public { type Error = AcceptRejectError; } - #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct CancelDebitNote { pub debit_note_id: String, @@ -672,7 +674,7 @@ pub mod public { type Error = AcceptRejectError; } - #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct CancelInvoice { pub invoice_id: String, diff --git a/core/model/src/sgx.rs b/core/model/src/sgx.rs index de37dd8972..33f78f8b86 100644 --- a/core/model/src/sgx.rs +++ b/core/model/src/sgx.rs @@ -13,7 +13,7 @@ pub enum Error { Attestation(String), } -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct VerifyAttestationEvidence { pub production: bool, diff --git a/core/model/src/version.rs b/core/model/src/version.rs index d9785b1691..f9be623b90 100644 --- a/core/model/src/version.rs +++ b/core/model/src/version.rs @@ -6,10 +6,10 @@ use serde::{Deserialize, Serialize}; use ya_client_model::ErrorMessage; use ya_service_bus::RpcMessage; -pub const BUS_ID: &'static str = "/local/version"; +pub const BUS_ID: &str = "/local/version"; /// Skip upgrading to the latest Yagna release. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Skip(); @@ -19,7 +19,7 @@ impl RpcMessage for Skip { type Error = ErrorMessage; } -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Get { pub check: bool, @@ -41,7 +41,7 @@ impl RpcMessage for Get { type Error = ErrorMessage; } -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, thiserror::Error)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, thiserror::Error)] #[serde(rename_all = "camelCase")] #[error("Version {version} '{name}' released {}", release_ts.format("%Y-%m-%d"))] pub struct Release { @@ -53,7 +53,7 @@ pub struct Release { pub update_ts: Option, } -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct VersionInfo { pub current: Release, diff --git a/core/net/Cargo.toml b/core/net/Cargo.toml index accf30fa9b..7892acdf5d 100644 --- a/core/net/Cargo.toml +++ b/core/net/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ya-net" -version = "0.2.0" +version = "0.3.0" authors = ["Golem Factory "] edition = "2018" @@ -11,19 +11,24 @@ service = [] hybrid-net = [] [dependencies] -ya-core-model = { version = "^0.6", features=["net", "identity"] } +ya-client-model = "0.5" +ya-core-model = { version = "^0.8", features=["net", "identity"] } -# ya-relay-client = "0.2" -ya-relay-client = { git = "https://github.com/golemfactory/ya-relay.git", rev = "0e6863c24767a246531d038455921f12c9e75e94" } +# ya-relay-client = "0.4" +# ya-relay-client = { path = "../../../ya-relay/client" } +ya-relay-client = { git = "https://github.com/golemfactory/ya-relay.git", rev = "ec174a250fc014fba084088a90f9b1c95d613ed3" } -ya-sb-proto = { version = "0.4" } +ya-sb-proto = { version = "0.6" } +ya-sb-util = { version = "0.4" } ya-service-api = "0.1" -ya-service-api-interfaces = "0.1" -ya-service-bus = "0.4" -ya-utils-networking = "0.1" +ya-service-api-interfaces = "0.2" +ya-service-bus = "0.6" +ya-utils-networking = "0.2" actix = "0.13" +actix-web = "4" anyhow = "1.0" +chrono = "0.4" futures = "0.3" humantime = "2.1" lazy_static = "1.4" @@ -31,7 +36,7 @@ log = "0.4" metrics="0.12" serde_json = "1.0" structopt = "0.3" -strum = { version = "0.22", features = ["derive"] } +strum = { version = "0.24", features = ["derive"] } thiserror = "1.0" tokio = { version = "1", features = ["time"] } tokio-stream = "0.1.8" @@ -44,9 +49,10 @@ prost = { version = "0.10" } rand = { version = "0.7"} [dev-dependencies] -ya-sb-proto = "0.4" -ya-sb-router = "0.4" +ya-sb-proto = "0.6" +ya-sb-router = "0.6" env_logger = "0.7" serde = "1.0" structopt = "0.3" +test-case = "2" diff --git a/core/net/src/central/api.rs b/core/net/src/central/api.rs index 26aa56294d..bc2409000e 100644 --- a/core/net/src/central/api.rs +++ b/core/net/src/central/api.rs @@ -46,7 +46,7 @@ where // will broadcast any Message related to this Topic. let subscribe_msg = M::into_subscribe_msg(broadcast_address); { - let mut subscriptions = SUBSCRIPTIONS.lock().unwrap(); + let mut subscriptions = SUBSCRIPTIONS.lock().await; subscriptions.insert(subscribe_msg.clone()); } diff --git a/core/net/src/central/cli.rs b/core/net/src/central/cli.rs index 9d0e4850e8..25aa551140 100644 --- a/core/net/src/central/cli.rs +++ b/core/net/src/central/cli.rs @@ -21,4 +21,8 @@ pub(crate) fn bind_service() { let _ = bus::bind(model::BUS_ID, move |_: model::GsbPing| { futures::future::err(err.clone()) }); + let err = error; + let _ = bus::bind(model::BUS_ID, move |_: model::FindNode| { + futures::future::err(err.clone()) + }); } diff --git a/core/net/src/central/handler.rs b/core/net/src/central/handler.rs index 0b55c8bfe2..b5d6890eb3 100644 --- a/core/net/src/central/handler.rs +++ b/core/net/src/central/handler.rs @@ -41,6 +41,7 @@ where caller: String, address: String, data: Vec, + _no_reply: bool, ) -> Self::Reply { (self.call_handler)(request_id, caller, address, data) } diff --git a/core/net/src/central/mod.rs b/core/net/src/central/mod.rs index 91482ea956..f1ec2a3c31 100644 --- a/core/net/src/central/mod.rs +++ b/core/net/src/central/mod.rs @@ -1,13 +1,16 @@ mod api; pub(crate) mod cli; mod handler; +mod rest_api; mod service; pub use api::*; +pub use rest_api::web_scope; pub use service::{bind_remote, Net}; use std::collections::HashSet; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; +use tokio::sync::Mutex; use ya_core_model::net::local::Subscribe; diff --git a/core/net/src/central/rest_api.rs b/core/net/src/central/rest_api.rs new file mode 100644 index 0000000000..f4ffcad6d8 --- /dev/null +++ b/core/net/src/central/rest_api.rs @@ -0,0 +1,20 @@ +use actix_web::{HttpResponse, Responder, Scope}; +use ya_client_model::net::{Status, NET_API_V2_NET_PATH}; + +use crate::error::Result; + +pub fn web_scope() -> Scope { + actix_web::web::scope(NET_API_V2_NET_PATH).service(get_info) +} + +#[actix_web::get("/status")] +async fn get_info() -> Result { + let (node_id, _) = crate::service::identities().await?; + let status = Status { + node_id, + listen_ip: None, + public_ip: None, + sessions: 1, + }; + Ok(HttpResponse::Ok().json(status)) +} diff --git a/core/net/src/central/service.rs b/core/net/src/central/service.rs index 0496871327..2959868e2b 100644 --- a/core/net/src/central/service.rs +++ b/core/net/src/central/service.rs @@ -46,7 +46,7 @@ pub async fn bind_remote( let bcast_service_id = as RpcMessage>::ID; // connect to hub with forwarding handler - let own_net_nodes: Vec<_> = nodes.iter().map(|id| net_service(id)).collect(); + let own_net_nodes: Vec<_> = nodes.iter().map(net_service).collect(); let forward_call = move |request_id: String, caller: String, addr: String, data: Vec| { let prefix = own_net_nodes @@ -65,11 +65,11 @@ pub async fn bind_remote( // actual forwarding to my local bus local_bus::call_stream(&local_addr, &caller, &data).right_stream() } else { - return stream::once(future::err(Error::GsbBadRequest(format!( + stream::once(future::err(Error::GsbBadRequest(format!( "wrong routing: {}; I'll accept only addrs starting with: {:?}", addr, own_net_nodes )))) - .left_stream(); + .left_stream() } }; @@ -107,9 +107,11 @@ pub async fn bind_remote( bind_net_handler(net::BUS_ID, central_bus.clone(), default_node_id); bind_net_handler(net::BUS_ID_UDP, central_bus.clone(), default_node_id); + bind_net_handler(net::BUS_ID_TRANSFER, central_bus.clone(), default_node_id); bind_from_handler("/from", central_bus.clone(), nodes.clone()); bind_from_handler("/udp/from", central_bus.clone(), nodes.clone()); + bind_from_handler("/transfer/from", central_bus.clone(), nodes.clone()); // Subscribe broadcast on remote { @@ -177,10 +179,17 @@ pub async fn bind_remote( Ok(done_rx) } -fn strip_udp(addr: &str) -> &str { +fn strip_prefixes_for_compatibility(addr: &str) -> &str { // Central NET doesn't support unreliable transport, so we just remove prefix // and use reliable protocol. - match addr.strip_prefix("/udp") { + let addr = match addr.strip_prefix("/udp") { + None => addr, + Some(wo_prefix) => wo_prefix, + }; + + // Central NET doesn't support transfer transport, so we just remove prefix + // and use tcp protocol. + match addr.strip_prefix("/transfer") { None => addr, Some(wo_prefix) => wo_prefix, } @@ -212,20 +221,20 @@ fn bind_net_handler( let default_caller_rpc = default_node_id.to_string(); let rpc = move |_caller: &str, addr: &str, msg: &[u8]| { let caller = default_caller_rpc.clone(); - let addr = strip_udp(addr); + let addr = strip_prefixes_for_compatibility(addr); log_message("rpc", &caller, addr); let addr = addr.to_string(); central_bus_rpc - .call(caller, addr.clone(), Vec::from(msg)) + .call(caller, addr.clone(), Vec::from(msg), false) .map_err(|e| Error::RemoteError(addr, e.to_string())) }; - let central_bus_stream = central_bus.clone(); + let central_bus_stream = central_bus; let default_caller_stream = default_node_id.to_string(); let stream = move |_caller: &str, addr: &str, msg: &[u8]| { let caller = default_caller_stream.clone(); - let addr = strip_udp(addr); + let addr = strip_prefixes_for_compatibility(addr); log_message("stream", &caller, addr); let addr = addr.to_string(); @@ -252,7 +261,7 @@ fn bind_from_handler( let nodes_rpc = nodes.clone(); let central_bus_rpc = central_bus.clone(); let rpc = move |_caller: &str, addr: &str, msg: &[u8]| { - let addr = strip_udp(addr); + let addr = strip_prefixes_for_compatibility(addr); let (from_node, to_addr) = match parse_from_addr(addr) { Ok(v) => v, @@ -268,15 +277,20 @@ fn bind_from_handler( } central_bus_rpc - .call(from_node.to_string(), to_addr.clone(), Vec::from(msg)) + .call( + from_node.to_string(), + to_addr.clone(), + Vec::from(msg), + false, + ) .map_err(|e| Error::RemoteError(to_addr, e.to_string())) .right_future() }; - let nodes_stream = nodes.clone(); - let central_bus_stream = central_bus.clone(); + let nodes_stream = nodes; + let central_bus_stream = central_bus; let stream = move |_caller: &str, addr: &str, msg: &[u8]| { - let addr = strip_udp(addr); + let addr = strip_prefixes_for_compatibility(addr); let (from_node, to_addr) = match parse_from_addr(addr) { Ok(v) => v, @@ -309,7 +323,7 @@ fn bind_from_handler( async fn unbind_remote(nodes: Vec) { let addrs = nodes .into_iter() - .map(|node_id| net_service(node_id)) + .map(net_service) .chain(std::iter::once(format!( "{}/{}", local_net::BUS_ID, @@ -331,7 +345,7 @@ async fn unbind_remote(nodes: Vec) { } async fn resubscribe() { - futures::stream::iter({ SUBSCRIPTIONS.lock().unwrap().clone() }.into_iter()) + futures::stream::iter({ SUBSCRIPTIONS.lock().await.clone() }.into_iter()) .for_each(|msg| { let topic = msg.topic().to_owned(); async move { @@ -371,7 +385,7 @@ where let reconnect_clone = reconnect.clone(); tokio::task::spawn_local(async move { - if let Ok(_) = dc_rx.await { + if dc_rx.await.is_ok() { metrics::counter!("net.disconnect", 1); reconnect_clone.borrow_mut().last_disconnect = Some(Instant::now()); log::warn!("Handlers disconnected"); @@ -461,14 +475,14 @@ impl Net { } pub(crate) fn parse_from_addr(from_addr: &str) -> anyhow::Result<(NodeId, String)> { - let mut it = from_addr.split("/").fuse(); + let mut it = from_addr.split('/').fuse(); if let (Some(""), Some("from"), Some(from_node_id), Some("to"), Some(to_node_id)) = (it.next(), it.next(), it.next(), it.next(), it.next()) { to_node_id.parse::()?; let prefix = 10 + from_node_id.len(); let service_id = &from_addr[prefix..]; - if let Some(_) = it.next() { + if it.next().is_some() { return Ok((from_node_id.parse()?, net_service(service_id))); } } diff --git a/core/net/src/cli.rs b/core/net/src/cli.rs index f303c507d7..6b12b93388 100644 --- a/core/net/src/cli.rs +++ b/core/net/src/cli.rs @@ -2,6 +2,7 @@ use std::cmp::Ordering; use std::fmt::Display; use std::time::Duration; +use chrono::{DateTime, NaiveDateTime, Utc}; use humantime::format_duration; use structopt::*; @@ -19,6 +20,11 @@ pub enum NetCommand { Sessions {}, /// List virtual sockets Sockets {}, + /// Find node + Find { + /// Node information to query for + node_id: String, + }, /// Ping connected nodes Ping {}, } @@ -50,7 +56,7 @@ impl NetCommand { let mut sessions: Vec = bus::service(model::BUS_ID) .send(model::Sessions {}) .await - .map_err(|e| anyhow::Error::msg(e))??; + .map_err(anyhow::Error::msg)??; sessions.sort_by_key(|s| s.node_id.unwrap_or_default().into_array()); @@ -62,6 +68,8 @@ impl NetCommand { "seen".into(), "time".into(), "ping".into(), + "in [MiB]".into(), + "out [MiB]".into(), ], values: sessions .into_iter() @@ -77,6 +85,8 @@ impl NetCommand { format_duration(seen).to_string(), format_duration(duration).to_string(), format_duration(ping).to_string(), + to_mib(s.metrics.tx_total, is_json), + to_mib(s.metrics.rx_total, is_json), ]} }) .collect(), @@ -87,7 +97,7 @@ impl NetCommand { let mut sockets: Vec = bus::service(model::BUS_ID) .send(model::Sockets {}) .await - .map_err(|e| anyhow::Error::msg(e))??; + .map_err(anyhow::Error::msg)??; sockets.sort_by(|l, r| match l.remote_addr.cmp(&r.remote_addr) { Ordering::Equal => l.remote_port.cmp(&r.remote_port), @@ -108,8 +118,8 @@ impl NetCommand { .into_iter() .map(|s| { serde_json::json! {[ - s.protocol.to_string(), - s.local_port.to_string(), + s.protocol, + s.local_port, s.remote_addr, s.remote_port, s.state, @@ -121,15 +131,32 @@ impl NetCommand { } .into()) } + NetCommand::Find { node_id } => { + let node: model::FindNodeResponse = bus::service(model::BUS_ID) + .send(model::FindNode { node_id }) + .await + .map_err(anyhow::Error::msg)??; + + let naive = NaiveDateTime::from_timestamp(node.seen.into(), 0); + let seen: DateTime = DateTime::from_utc(naive, Utc); + + CommandOutput::object(serde_json::json!({ + "identities": node.identities.into_iter().map(|n| n.to_string()).collect::>(), + "endpoints": node.endpoints.into_iter().map(|n| n.to_string()).collect::>(), + "seen": seen.to_string(), + "slot": node.slot, + "encryption": node.encryption, + })) + } NetCommand::Ping { .. } => { let pings = bus::service(model::BUS_ID) .send(model::GsbPing {}) .await - .map_err(|e| anyhow::Error::msg(e))??; + .map_err(anyhow::Error::msg)??; Ok(ResponseTable { columns: vec![ - "node".into(), + "nodeId".into(), "alias".into(), "p2p".into(), "ping (tcp)".into(), diff --git a/core/net/src/config.rs b/core/net/src/config.rs index bedccc4982..877819230d 100644 --- a/core/net/src/config.rs +++ b/core/net/src/config.rs @@ -20,12 +20,10 @@ pub struct Config { pub host: Option, #[structopt(env = "YA_NET_BIND_URL", default_value = "udp://0.0.0.0:11500")] pub bind_url: Url, - #[structopt(env = "YA_NET_BROADCAST_SIZE", default_value = "12")] + #[structopt(env = "YA_NET_BROADCAST_SIZE", default_value = "10")] pub broadcast_size: u32, #[structopt(env = "YA_NET_SESSION_EXPIRATION", parse(try_from_str = humantime::parse_duration), default_value = "15s")] pub session_expiration: Duration, - #[structopt(env = "YA_NET_VIRTUAL_TCP_BUFFER_SIZE_MULTIPLIER", default_value = "4")] - pub vtcp_buffer_size_multiplier: usize, } impl Config { @@ -43,6 +41,7 @@ impl Config { #[cfg(not(feature = "hybrid-net"))] impl Default for NetType { fn default() -> Self { + std::env::set_var("YA_NET_TYPE", "central"); NetType::Central } } diff --git a/core/net/src/error.rs b/core/net/src/error.rs new file mode 100644 index 0000000000..25da438c9d --- /dev/null +++ b/core/net/src/error.rs @@ -0,0 +1,29 @@ +use actix_web::{HttpResponse, ResponseError}; +use ya_client_model::ErrorMessage; + +pub type Result = std::result::Result; + +#[allow(dead_code)] +#[derive(thiserror::Error, Debug)] +pub enum NetError { + //TODO handle more specific cases + #[error("Error: {0}")] + Error(#[from] anyhow::Error), + #[error("Bad Request: {0}")] + BadRequest(String), +} + +impl ResponseError for NetError { + fn error_response(&self) -> HttpResponse { + match self { + NetError::Error(_) => { + log::error!("Network API server error: {}", self); + HttpResponse::InternalServerError().json(ErrorMessage::new(self)) + } + NetError::BadRequest(_) => { + log::error!("Network API server error: {}", self); + HttpResponse::BadRequest().json(ErrorMessage::new(self)) + } + } + } +} diff --git a/core/net/src/hybrid/api.rs b/core/net/src/hybrid/api.rs index 304b9f8efa..2afd52fa42 100644 --- a/core/net/src/hybrid/api.rs +++ b/core/net/src/hybrid/api.rs @@ -1,16 +1,8 @@ -use std::sync::Arc; - -use futures::lock::Mutex; -use futures::SinkExt; - +use ya_core_model::net; use ya_core_model::net::local::{ BindBroadcastError, BroadcastMessage, SendBroadcastMessage, ToEndpoint, }; -use ya_sb_proto::codec::GsbMessage; -use ya_service_bus::{serialization, Error, RpcMessage}; - -use crate::hybrid::codec::encode_message; -use crate::hybrid::service::{BCAST, BCAST_HANDLERS, BCAST_SENDER}; +use ya_service_bus::{typed as bus, Error, RpcEndpoint, RpcMessage}; pub async fn broadcast( caller: S, @@ -26,25 +18,10 @@ where M: BroadcastMessage + Send + Sync + Unpin + 'static, S: ToString + 'static, { - let mut sender = BCAST_SENDER - .read() - .await - .clone() - .ok_or_else(|| Error::Closed("network not initialized".to_string()))?; - - let request = GsbMessage::BroadcastRequest(ya_sb_proto::BroadcastRequest { - data: serialization::to_vec(&message)?, - caller: caller.to_string(), - topic: M::TOPIC.to_owned(), - }); - - let bytes = encode_message(request).map_err(|e| Error::EncodingProblem(e.to_string()))?; - sender - .send(bytes) + // TODO: We shouldn't use send_as. Put identity inside broadcasted message instead. + bus::service(net::local::BUS_ID) + .send_as(caller, SendBroadcastMessage::new(message)) .await - .map_err(|_| Error::Closed("broadcast channel is closed".to_string()))?; - - Ok(Ok(())) } pub async fn bind_broadcast_with_caller( @@ -61,37 +38,27 @@ where > + 'static, F: FnMut(String, SendBroadcastMessage) -> T + Send + 'static, { - let address = broadcast_address.to_string(); - let handler_rc = Arc::new(Mutex::new(handler)); + log::debug!("Creating broadcast topic {}.", M::TOPIC); + let address = broadcast_address.to_string(); let subscription = M::into_subscribe_msg(address.clone()); log::trace!( - "binding broadcast handler for topic: {}", + "Binding broadcast handler for topic: {}", subscription.topic() ); - let handler = move |caller: String, bytes: &[u8]| { - match serialization::from_slice::(bytes) { - Ok(m) => { - let m = SendBroadcastMessage::new(m); - let handler = handler_rc.clone(); - tokio::task::spawn_local(async move { - let mut h = handler.lock().await; - let _ = (*(h))(caller, m).await; - }); - } - Err(e) => { - log::debug!("broadcast msg {} deserialization error: {}", M::TOPIC, e); - } - }; - }; + bus::service(net::local::BUS_ID) + .send(subscription) + .await??; - BCAST.add(subscription).await; - BCAST_HANDLERS - .write() - .await - .insert(address, Arc::new(Mutex::new(Box::new(handler)))); + log::debug!( + "Binding handler '{broadcast_address}' for broadcast topic {}.", + M::TOPIC + ); + // We created endpoint address above. Now we must add handler, which will + // handle broadcasts forwarded to this address. + bus::bind_with_caller(broadcast_address, handler); Ok(()) } diff --git a/core/net/src/hybrid/cli.rs b/core/net/src/hybrid/cli.rs index 2a10934400..5da4dc3510 100644 --- a/core/net/src/hybrid/cli.rs +++ b/core/net/src/hybrid/cli.rs @@ -2,6 +2,7 @@ use anyhow::anyhow; use futures::future::join_all; use futures::TryFutureExt; use std::time::{Duration, Instant}; +use ya_client_model::NodeId; use ya_core_model::net as ya_net; use ya_core_model::net::local::{GsbPingResponse, StatusError}; @@ -41,16 +42,26 @@ pub(crate) fn bind_service() { let mut responses = Vec::new(); let now = Instant::now(); + let mut metrics = client.session_metrics().await?; + for session in client.sessions().await? { let node_id = client.remote_id(session.remote).await?; let kind = match node_id { Some(id) => { let is_p2p = client.is_p2p(id).await?; - is_p2p.then(|| "p2p").unwrap_or("relay") + if is_p2p { + "p2p" + } else { + "relay" + } } - None => "", + None => "server", }; + let mut metric = node_id + .and_then(|node_id| metrics.remove(&node_id)) + .unwrap_or_default(); + responses.push(model::SessionResponse { node_id, id: session.id.to_string(), @@ -59,6 +70,7 @@ pub(crate) fn bind_service() { seen: now - session.last_seen, duration: now - session.created, ping: session.last_ping, + metrics: to_status_metrics(&mut metric), }); } @@ -88,6 +100,14 @@ pub(crate) fn bind_service() { } .map_err(status_err) }); + let _ = bus::bind(model::BUS_ID, move |find: model::FindNode| { + async move { + let client = Net::client().await?; + let node_id: NodeId = find.node_id.parse()?; + client.find_node(node_id).await? + } + .map_err(status_err) + }); } fn to_status_metrics(metrics: &mut ChannelMetrics) -> model::StatusMetrics { @@ -118,10 +138,10 @@ pub async fn cli_ping() -> anyhow::Result> { let mut results = join_all( nodes .iter() - .map(|(id, alias)| { + .map(|(id, _)| { let target_id = *id; - async move { + let udp_future = async move { let udp_before = Instant::now(); ya_net::from(our_node_id) @@ -133,47 +153,46 @@ pub async fn cli_ping() -> anyhow::Result> { anyhow::Ok(udp_before.elapsed()) } - .map_err(|e| anyhow!("(Udp ping). {}", e)) - .and_then(move |udp_ping| { - async move { - let tcp_before = Instant::now(); - - ya_net::from(our_node_id) - .to(target_id) - .service(ya_net::DIAGNOSTIC) - .send(GsbRemotePing {}) - .timeout(Some(ping_timeout)) - .await???; - - let tcp_ping = tcp_before.elapsed(); - - anyhow::Ok(GsbPingResponse { - node_id: target_id, - node_alias: alias.clone(), - tcp_ping, - udp_ping, - is_p2p: false, // Updated later - }) - } - .map_err(|e| anyhow!("(Tcp ping). {}", e)) - }) + .map_err(|e| anyhow!("(Udp ping). {}", e)); + + let tcp_future = async move { + let tcp_before = Instant::now(); + + ya_net::from(our_node_id) + .to(target_id) + .service(ya_net::DIAGNOSTIC) + .send(GsbRemotePing {}) + .timeout(Some(ping_timeout)) + .await???; + + anyhow::Ok(tcp_before.elapsed()) + } + .map_err(|e| anyhow!("(Tcp ping). {}", e)); + + futures::future::join(udp_future, tcp_future) }) .collect::>(), ) .await .into_iter() .enumerate() - .map(|(idx, result)| match result { - Ok(ping) => ping, - Err(e) => { + .map(|(idx, results)| { + if let Err(e) = &results.0 { log::warn!("Failed to ping node: {} {}", nodes[idx].0, e); - GsbPingResponse { - node_id: nodes[idx].0, - node_alias: nodes[idx].1, - tcp_ping: ping_timeout, - udp_ping: ping_timeout, - is_p2p: false, // Updated later - } + } + if let Err(e) = &results.1 { + log::warn!("Failed to ping node: {} {}", nodes[idx].0, e); + } + + let udp_ping = results.0.unwrap_or(ping_timeout); + let tcp_ping = results.1.unwrap_or(ping_timeout); + + GsbPingResponse { + node_id: nodes[idx].0, + node_alias: nodes[idx].1, + tcp_ping, + udp_ping, + is_p2p: false, // Updated later } }) .collect::>(); diff --git a/core/net/src/hybrid/client.rs b/core/net/src/hybrid/client.rs index 35a0cdfcc2..de46d70477 100644 --- a/core/net/src/hybrid/client.rs +++ b/core/net/src/hybrid/client.rs @@ -1,10 +1,13 @@ -use std::net::SocketAddr; +use std::collections::HashMap; +use std::convert::TryFrom; +use std::net::{IpAddr, SocketAddr}; use std::sync::Arc; +use std::sync::RwLock; use actix::prelude::*; use anyhow::{anyhow, bail}; -use tokio::sync::RwLock; +use ya_core_model::net::local::FindNodeResponse; use ya_core_model::NodeId; use ya_relay_client::{ChannelMetrics, Client, SessionDesc, SocketDesc, SocketState}; @@ -16,10 +19,10 @@ lazy_static::lazy_static! { pub struct ClientProxy(Addr); impl ClientProxy { - pub async fn new() -> anyhow::Result { - let addr = match ADDRESS.read().await.clone() { + pub fn new() -> anyhow::Result { + let addr = match ADDRESS.read().unwrap().clone() { Some(addr) => addr, - None => bail!("network not initialized"), + None => bail!("Net client not initialized. ClientProxy has no address of ClientActor"), }; Ok(Self(addr)) @@ -45,23 +48,16 @@ pub(crate) struct ClientActor { } impl ClientActor { - pub fn new(client: Client) -> Self { - Self { client } + /// Creates and starts `ClientActor`, and then shares its address with `ClientProxy`. + pub fn init(client: Client) { + let client = Self { client }; + let addr = client.start(); + ADDRESS.write().unwrap().replace(addr); } } impl Actor for ClientActor { type Context = Context; - - fn started(&mut self, ctx: &mut Self::Context) { - let addr = ctx.address(); - ctx.wait( - async move { - ADDRESS.write().await.replace(addr); - } - .into_actor(self), - ); - } } macro_rules! proxy { @@ -141,6 +137,29 @@ proxy!( sockets, |client: Client| { futures::future::ready(client.sockets()) } ); +proxy!( + FindNode(NodeId) -> anyhow::Result, + find_node, + |client: Client, msg: FindNode| async move { + let res = client.find_node(msg.0).await?; + let identities = res.identities.into_iter() + .map(|i| NodeId::try_from(&i.node_id.to_vec())) + .collect::, _>>()?; + let endpoints = res.endpoints.into_iter() + .map(|e| { + e.address.parse().map(|ip: IpAddr| SocketAddr::new(ip, e.port as u16)) + }) + .collect::, _>>()?; + + Ok(FindNodeResponse { + identities, + endpoints, + seen: res.seen_ts, + slot: res.slot, + encryption: res.supported_encryptions, + }) + } +); proxy!( GetSessions -> Vec, sessions, @@ -171,3 +190,8 @@ proxy!( shutdown, |mut client: Client| async move { client.shutdown().await } ); +proxy!( + GetSessionMetrics -> HashMap, + session_metrics, + |client: Client| async move { client.session_metrics().await } +); diff --git a/core/net/src/hybrid/codec.rs b/core/net/src/hybrid/codec.rs index c6c84578c8..3a610ee1a7 100644 --- a/core/net/src/hybrid/codec.rs +++ b/core/net/src/hybrid/codec.rs @@ -73,12 +73,14 @@ pub(crate) fn encode_request( address: String, request_id: String, data: Vec, + no_reply: bool, ) -> anyhow::Result> { let message = GsbMessage::CallRequest(ya_sb_proto::CallRequest { caller: caller.to_string(), address, request_id, data, + no_reply, }); Ok(encode_message(message)?) } diff --git a/core/net/src/hybrid/mod.rs b/core/net/src/hybrid/mod.rs index 50a85c5eca..f8decd9c88 100644 --- a/core/net/src/hybrid/mod.rs +++ b/core/net/src/hybrid/mod.rs @@ -3,7 +3,9 @@ pub(crate) mod cli; mod client; mod codec; mod crypto; +mod rest_api; mod service; pub use api::*; +pub use rest_api::web_scope; pub use service::{start_network, Net}; diff --git a/core/net/src/hybrid/rest_api.rs b/core/net/src/hybrid/rest_api.rs new file mode 100644 index 0000000000..19f61ac1d2 --- /dev/null +++ b/core/net/src/hybrid/rest_api.rs @@ -0,0 +1,23 @@ +use actix_web::{web::Data, HttpResponse, Responder, Scope}; +use ya_client_model::net::{Status, NET_API_V2_NET_PATH}; + +use crate::error::Result; + +use super::client::ClientProxy; + +pub fn web_scope() -> Scope { + actix_web::web::scope(NET_API_V2_NET_PATH) + .app_data(Data::new(ClientProxy::new().unwrap())) + .service(get_info) +} + +#[actix_web::get("/status")] +async fn get_info(client: Data) -> Result { + let status = Status { + node_id: client.node_id().await?, + listen_ip: client.bind_addr().await?.map(|addr| addr.to_string()), + public_ip: client.public_addr().await?.map(|addr| addr.to_string()), + sessions: client.sessions().await?.len(), + }; + Ok(HttpResponse::Ok().json(status)) +} diff --git a/core/net/src/hybrid/service.rs b/core/net/src/hybrid/service.rs index 75d9e03793..35c35ad59a 100644 --- a/core/net/src/hybrid/service.rs +++ b/core/net/src/hybrid/service.rs @@ -9,10 +9,8 @@ use std::sync::atomic::Ordering::Relaxed; use std::sync::Arc; use std::task::Poll; -use actix::Actor; use anyhow::{anyhow, Context as AnyhowContext}; use futures::channel::{mpsc, oneshot}; -use futures::lock::Mutex; use futures::stream::LocalBoxStream; use futures::{FutureExt, SinkExt, Stream, StreamExt, TryStreamExt}; use metrics::counter; @@ -20,38 +18,38 @@ use tokio::sync::RwLock; use tokio_stream::wrappers::UnboundedReceiverStream; use url::Url; +use ya_core_model::net::local::{SendBroadcastMessage, SendBroadcastStub}; use ya_core_model::{identity, net, NodeId}; use ya_relay_client::codec::forward::{PrefixedSink, PrefixedStream, SinkKind}; use ya_relay_client::crypto::CryptoProvider; -use ya_relay_client::{Client, ClientBuilder, ForwardReceiver}; +use ya_relay_client::proto::Payload; +use ya_relay_client::{Client, ClientBuilder, ForwardReceiver, TransportType}; use ya_sb_proto::codec::GsbMessage; use ya_sb_proto::CallReplyCode; -use ya_service_bus::{typed, untyped as local_bus, Error, ResponseChunk, RpcEndpoint}; +use ya_sb_util::RevPrefixes; +use ya_service_bus::untyped::{Fn4HandlerExt, Fn4StreamHandlerExt}; +use ya_service_bus::{ + serialization, typed, untyped as local_bus, Error, ResponseChunk, RpcEndpoint, RpcMessage, +}; use ya_utils_networking::resolver; use crate::bcast::BCastService; use crate::config::Config; use crate::hybrid::client::{ClientActor, ClientProxy}; use crate::hybrid::codec; +use crate::hybrid::codec::encode_message; use crate::hybrid::crypto::IdentityCryptoProvider; const DEFAULT_NET_RELAY_HOST: &str = "127.0.0.1:7464"; -pub type BCastHandler = Box; - type BusSender = mpsc::Sender; type BusReceiver = mpsc::Receiver; -type NetSender = mpsc::Sender>; -type NetReceiver = mpsc::Receiver>; +type NetSender = mpsc::Sender; type NetSinkKind = SinkKind; -type NetSinkKey = (NodeId, bool); - -type ArcMap = Arc>>; +type NetSinkKey = (NodeId, TransportType); lazy_static::lazy_static! { pub(crate) static ref BCAST: BCastService = Default::default(); - pub(crate) static ref BCAST_HANDLERS: ArcMap>> = Default::default(); - pub(crate) static ref BCAST_SENDER: Arc>> = Default::default(); pub(crate) static ref SHUTDOWN_TX: Arc>>> = Default::default(); } @@ -64,16 +62,18 @@ pub struct Net; impl Net { #[inline] pub async fn client() -> anyhow::Result { - ClientProxy::new().await + ClientProxy::new() } pub async fn gsb(_: Context, config: Config) -> anyhow::Result<()> { + ya_service_bus::serialization::CONFIG.set_compress(true); + let (default_id, ids) = crate::service::identities().await?; let (started_tx, started_rx) = oneshot::channel(); let (shutdown_tx, shutdown_rx) = oneshot::channel(); log::info!( - "HYBRID_NET - Using default identity as network id: {:?}", + "Hybrid NET - Using default identity as network id: {:?}", default_id ); @@ -116,23 +116,19 @@ pub async fn start_network( let broadcast_size = config.broadcast_size; let crypto = IdentityCryptoProvider::new(default_id); - let client = build_client(config, crypto.clone()).await?; + CLIENT.with(|inner| { inner.borrow_mut().replace(client.clone()); }); - - ClientActor::new(client.clone()).start(); - - let (btx, brx) = mpsc::channel(1); - BCAST_SENDER.write().await.replace(btx); + ClientActor::init(client.clone()); let receiver = client.clone().forward_receiver().await.unwrap(); let mut services: HashSet<_> = Default::default(); ids.iter().for_each(|id| { - let service = net::net_service(id); - services.insert(format!("/udp{}", service)); - services.insert(service); + services.insert(net::net_service_udp(id)); + services.insert(net::net_service(id)); + services.insert(net::net_transfer_service(id)); }); let state = State::new(ids, services); @@ -143,8 +139,25 @@ pub async fn start_network( Err(err) => anyhow::bail!("invalid address: {}", err), } }; - bind_local_bus(net::BUS_ID_UDP, state.clone(), false, net_handler()); - bind_local_bus(net::BUS_ID, state.clone(), true, net_handler()); + + bind_local_bus( + net::BUS_ID_UDP, + state.clone(), + TransportType::Unreliable, + net_handler(), + ); + bind_local_bus( + net::BUS_ID, + state.clone(), + TransportType::Reliable, + net_handler(), + ); + bind_local_bus( + net::BUS_ID_TRANSFER, + state.clone(), + TransportType::Transfer, + net_handler(), + ); let from_handler = || { let state_from = state.clone(); @@ -152,17 +165,34 @@ pub async fn start_network( let (from, to, addr) = parse_from_to_addr(addr).map_err(|e| anyhow::anyhow!("invalid address: {}", e))?; if !state_from.inner.borrow().ids.contains(&from) { - anyhow::bail!("unknown identity: {:?}", from); + anyhow::bail!("Trying to send message from unknown identity: {}", from); } Ok((from, to, addr)) } }; - bind_local_bus("/from", state.clone(), true, from_handler()); - bind_local_bus("/udp/from", state.clone(), false, from_handler()); - tokio::task::spawn_local(broadcast_handler(brx, broadcast_size)); + bind_local_bus( + "/from", + state.clone(), + TransportType::Reliable, + from_handler(), + ); + bind_local_bus( + "/udp/from", + state.clone(), + TransportType::Unreliable, + from_handler(), + ); + bind_local_bus( + "/transfer/from", + state.clone(), + TransportType::Transfer, + from_handler(), + ); + tokio::task::spawn_local(forward_handler(receiver, state.clone())); + bind_broadcast_handlers(broadcast_size); bind_identity_event_handler(crypto).await; if let Some(address) = client.public_addr().await { @@ -184,13 +214,10 @@ async fn build_client( .map_err(|e| anyhow!("Resolving hybrid NET relay server failed. Error: {}", e))?; let url = Url::parse(&format!("udp://{addr}"))?; - log::debug!("Setting up hybrid net with url: {url}"); - ClientBuilder::from_url(url) .crypto(crypto) .listen(config.bind_url.clone()) .expire_session_after(config.session_expiration) - .virtual_tcp_buffer_size_multiplier(config.vtcp_buffer_size_multiplier) .connect() .build() .await @@ -204,7 +231,9 @@ async fn relay_addr(config: &Config) -> anyhow::Result { // FIXME: remove .unwrap_or_else(|_| DEFAULT_NET_RELAY_HOST.to_string()), }; - log::trace!("resolving host_port: {}", host_port); + + log::info!("Hybrid NET relay server configured on url: udp://{host_port}"); + let (host, port) = &host_port .split_once(':') .context("Please use host:port format")?; @@ -216,17 +245,15 @@ async fn relay_addr(config: &Config) -> anyhow::Result { Ok(socket) } -fn bind_local_bus(address: &'static str, state: State, reliable: bool, resolver: F) +fn bind_local_bus(address: &'static str, state: State, transport: TransportType, resolver: F) where F: Fn(&str, &str) -> anyhow::Result<(NodeId, NodeId, String)> + 'static, { let resolver = Rc::new(resolver); - let resolver_ = resolver.clone(); let state_ = state.clone(); - let rpc = move |caller: &str, addr: &str, msg: &[u8]| { - log::trace!("local bus: rpc call (egress): {}", addr); + let rpc = move |caller: &str, addr: &str, msg: &[u8], no_reply: bool| { let (caller_id, remote_id, address) = match (*resolver_)(caller, addr) { Ok(id) => id, Err(err) => { @@ -236,37 +263,52 @@ where }; log::trace!( - "local bus: rpc call (egress): {} ({} -> {})", + "local bus: rpc call (egress): {} ({} -> {}), no_reply: {no_reply}", address, caller_id, - remote_id + remote_id, ); - let mut rx = if state_.inner.borrow().ids.contains(&remote_id) { - let (tx, rx) = mpsc::channel(1); - forward_bus_to_local(&caller_id.to_string(), addr, msg, &state_, tx); - rx + let is_local_dest = state_.inner.borrow().ids.contains(&remote_id); + + let rx = if no_reply { + if is_local_dest { + push_bus_to_local(caller_id, addr, msg, &state_); + } else { + push_bus_to_net(caller_id, remote_id, address, msg, &state_, transport); + } + + None } else { - forward_bus_to_net(caller_id, remote_id, address, msg, &state_, reliable) + let rx = if is_local_dest { + let (tx, rx) = mpsc::channel(1); + forward_bus_to_local(caller_id, addr, msg, &state_, tx); + rx + } else { + forward_bus_to_net(caller_id, remote_id, address, msg, &state_, transport) + }; + + Some(rx) }; async move { - match rx.next().await.ok_or(Error::Cancelled) { - Ok(chunk) => match chunk { - ResponseChunk::Full(data) => codec::decode_reply(data), - ResponseChunk::Part(_) => { - Err(Error::GsbFailure("partial response".to_string())) - } + match rx { + None => Ok(Vec::new()), + Some(mut rx) => match rx.next().await.ok_or(Error::Cancelled) { + Ok(chunk) => match chunk { + ResponseChunk::Full(data) => codec::decode_reply(data), + ResponseChunk::Part(_) => { + Err(Error::GsbFailure("partial response".to_string())) + } + }, + Err(err) => Err(err), }, - Err(err) => Err(err), } } .right_future() }; - let stream = move |caller: &str, addr: &str, msg: &[u8]| { - log::trace!("local bus: stream call (egress): {}", addr); - + let stream = move |caller: &str, addr: &str, msg: &[u8], no_reply: bool| { let (caller_id, remote_id, address) = match (*resolver)(caller, addr) { Ok(id) => id, Err(err) => { @@ -286,12 +328,23 @@ where remote_id ); - let rx = if state.inner.borrow().ids.contains(&remote_id) { - let (tx, rx) = mpsc::channel(1); - forward_bus_to_local(&caller_id.to_string(), addr, msg, &state, tx); - rx + let is_local_dest = state.inner.borrow().ids.contains(&remote_id); + let rx = if no_reply { + if is_local_dest { + push_bus_to_local(caller_id, addr, msg, &state); + } else { + push_bus_to_net(caller_id, remote_id, address, msg, &state, transport); + } + futures::stream::empty().left_stream() } else { - forward_bus_to_net(caller_id, remote_id, address, msg, &state, reliable) + let rx = if is_local_dest { + let (tx, rx) = mpsc::channel(1); + forward_bus_to_local(caller_id, addr, msg, &state, tx); + rx + } else { + forward_bus_to_net(caller_id, remote_id, address, msg, &state, transport) + }; + rx.right_stream() }; let eos = Rc::new(AtomicBool::new(false)); @@ -317,6 +370,8 @@ where }; log::debug!("local bus: subscribing to {}", address); + let rpc = rpc.into_handler(); + let stream = stream.into_stream_handler(); local_bus::subscribe(address, rpc, stream); } @@ -354,19 +409,12 @@ async fn bind_identity_event_handler(crypto: IdentityCryptoProvider) { } /// Forward requests from and to the local bus -fn forward_bus_to_local(caller: &str, addr: &str, data: &[u8], state: &State, tx: BusSender) { - let address = match { - let inner = state.inner.borrow(); - inner - .services - .iter() - .find(|&id| addr.starts_with(id)) - // replaces /net//test/1 --> /public/test/1 - .map(|s| addr.replacen(s, net::PUBLIC_PREFIX, 1)) - } { +fn forward_bus_to_local(caller: NodeId, addr: &str, data: &[u8], state: &State, tx: BusSender) { + let caller = caller.to_string(); + let address = match state.get_public_service(addr) { Some(address) => address, None => { - let err = format!("unknown address: {}", addr); + let err = format!("Net: unknown address: {}", addr); handler_reply_bad_request("unknown", err, tx); return; } @@ -374,7 +422,7 @@ fn forward_bus_to_local(caller: &str, addr: &str, data: &[u8], state: &State, tx log::trace!("forwarding /net call to a local endpoint: {}", address); - let send = local_bus::call_stream(address.as_str(), caller, data); + let send = local_bus::call_stream(&address, &caller, data); tokio::task::spawn_local(async move { let _ = send .forward(tx.sink_map_err(|e| Error::GsbFailure(e.to_string()))) @@ -382,6 +430,24 @@ fn forward_bus_to_local(caller: &str, addr: &str, data: &[u8], state: &State, tx }); } +fn push_bus_to_local(caller: NodeId, addr: &str, data: &[u8], state: &State) { + let caller = caller.to_string(); + let address = match state.get_public_service(addr) { + Some(address) => address, + None => { + log::debug!("Net: unknown address: {}", addr); + return; + } + }; + + log::trace!("pushing /net message to a local endpoint: {}", address); + + let send = local_bus::push(&address, &caller, data); + tokio::task::spawn_local(async move { + let _ = send.await; + }); +} + /// Forward requests from local bus to the network fn forward_bus_to_net( caller_id: NodeId, @@ -389,29 +455,32 @@ fn forward_bus_to_net( address: impl ToString, msg: &[u8], state: &State, - reliable: bool, + transport: TransportType, ) -> BusReceiver { let address = address.to_string(); let state = state.clone(); let request_id = gen_id().to_string(); - log::trace!("forward net {}", address); - let (tx, rx) = mpsc::channel(1); - let msg = - match codec::encode_request(caller_id, address.clone(), request_id.clone(), msg.to_vec()) { - Ok(vec) => vec, - Err(err) => { - log::debug!("forward net: invalid request: {}", err); - handler_reply_bad_request(request_id, format!("invalid request: {}", err), tx); - return rx; - } - }; + let msg = match codec::encode_request( + caller_id, + address.clone(), + request_id.clone(), + msg.to_vec(), + false, + ) { + Ok(vec) => vec, + Err(err) => { + log::debug!("Forward bus->net ({caller_id} -> {remote_id}), address: {address}: invalid request: {err}"); + handler_reply_bad_request(request_id, format!("Net: invalid request: {err}"), tx); + return rx; + } + }; let request = Request { caller_id, remote_id, - address, + address: address.clone(), tx: tx.clone(), }; { @@ -420,20 +489,20 @@ fn forward_bus_to_net( } tokio::task::spawn_local(async move { - log::trace!( - "local bus handler -> send message to remote ({} B)", + log::debug!( + "Local bus handler ({caller_id} -> {remote_id}), address: {address}, id: {request_id} -> send message to remote ({} B)", msg.len() ); - match state.forward_sink(remote_id, reliable).await { + match state.forward_sink(remote_id, transport).await { Ok(mut sink) => { let _ = sink.send(msg).await.map_err(|_| { - let err = "error sending message: session closed".to_string(); + let err = "Net: error sending message: session closed".to_string(); handler_reply_service_err(request_id, err, tx); }); } Err(error) => { - let err = format!("error forwarding message: {}", error); + let err = format!("Net: error forwarding message: {}", error); handler_reply_service_err(request_id, err, tx); } }; @@ -442,28 +511,116 @@ fn forward_bus_to_net( rx } -/// Forward broadcast messages from the network to the local bus +fn push_bus_to_net( + caller_id: NodeId, + remote_id: NodeId, + address: impl ToString, + msg: &[u8], + state: &State, + transport: TransportType, +) { + let address = address.to_string(); + let state = state.clone(); + let request_id = gen_id().to_string(); + + let msg = match codec::encode_request( + caller_id, + address.clone(), + request_id.clone(), + msg.to_vec(), + true, + ) { + Ok(vec) => vec, + Err(err) => { + log::debug!("Push bus->net ({caller_id} -> {remote_id}), address: {address}: invalid request: {err}"); + return; + } + }; + + tokio::task::spawn_local(async move { + log::debug!( + "Local bus push handler ({caller_id} -> {remote_id}), address: {address}, id: {request_id} -> send message to remote ({} B)", + msg.len() + ); + + match state.forward_sink(remote_id, transport).await { + Ok(mut sink) => { + let _ = sink.send(msg).await.map_err(|_| { + log::debug!("Net: error sending message: session closed"); + }); + } + Err(error) => { + log::debug!("Net: error forwarding message: {}", error); + } + }; + }); +} + +/// Forward broadcast messages from the local bus to the network fn broadcast_handler( - rx: NetReceiver, + caller: &str, + _addr: &str, + msg: &[u8], broadcast_size: u32, -) -> impl Future + Unpin + 'static { - StreamExt::for_each(rx, move |payload| { - async move { - let client = CLIENT - .with(|c| c.borrow().clone()) - .ok_or_else(|| anyhow::anyhow!("network not initialized"))?; - client - .broadcast(payload, broadcast_size) - .await - .map_err(|e| anyhow!("Broadcast failed: {}", e)) +) -> impl Future, Error>> { + let message = msg.to_vec(); + let caller = caller.to_string(); + + async move { + let stub: SendBroadcastStub = serialization::from_slice(&message) + .map_err(|e| Error::GsbFailure(format!("Invalid broadcast message: {e}")))?; + + let request = GsbMessage::BroadcastRequest(ya_sb_proto::BroadcastRequest { + //data: serialization::to_vec(&message)?, + data: message, + caller, + topic: stub.topic, + }); + + let payload = encode_message(request).map_err(|e| Error::EncodingProblem(e.to_string()))?; + + let client = CLIENT + .with(|c| c.borrow().clone()) + .ok_or_else(|| Error::GsbFailure("Network not initialized".to_string()))?; + client + .broadcast(payload, broadcast_size) + .await + .map_err(|e| Error::GsbFailure(format!("Broadcast failed: {e}")))?; + + Ok(serialization::to_vec(&Ok::<(), ()>(())).unwrap()) + } + .then(|result| async move { + if let Err(e) = &result { + log::debug!("Unable to broadcast message: {e}") } - .then(|result: anyhow::Result<()>| async move { - if let Err(e) = result { - log::debug!("unable to broadcast message: {}", e) - } - }) + result }) - .boxed_local() +} + +fn bind_broadcast_handlers(broadcast_size: u32) { + let _ = typed::bind( + net::local::BUS_ID, + move |subscribe: net::local::Subscribe| { + let topic = subscribe.topic().to_owned(); + let bcast = BCAST.clone(); + + async move { + log::debug!("NET: Subscribe topic {}", topic); + let (_is_new, id) = bcast.add(subscribe).await; + log::debug!("NET: Created new topic: {}", topic); + Ok(id) + } + }, + ); + + let bcast_service_id = as RpcMessage>::ID; + let _ = local_bus::subscribe( + &format!("{}/{}", net::local::BUS_ID, bcast_service_id), + move |caller: &str, addr: &str, msg: &[u8]| { + broadcast_handler(caller, addr, msg, broadcast_size) + }, + (), + ); } /// Handle incoming forward messages @@ -471,11 +628,13 @@ fn forward_handler( receiver: ForwardReceiver, state: State, ) -> impl Future + Unpin + 'static { + // Takes stream of generic packets, reads sender NodeId and translates + // into stream designated to handle this specific Node. UnboundedReceiverStream::new(receiver) .for_each(move |fwd| { let state = state.clone(); async move { - let key = (fwd.node_id, fwd.reliable); + let key = (fwd.node_id, fwd.transport); let mut tx = match { let inner = state.inner.borrow(); inner.routes.get(&key).cloned() @@ -483,7 +642,7 @@ fn forward_handler( Some(cached) => cached, None => { let state = state.clone(); - let (tx, rx) = forward_channel(fwd.reliable); + let (tx, rx) = forward_channel(fwd.transport); { let mut inner = state.inner.borrow_mut(); inner.routes.insert(key, tx.clone()); @@ -491,30 +650,39 @@ fn forward_handler( tokio::task::spawn_local(inbound_handler( rx, fwd.node_id, - fwd.reliable, + fwd.transport, state, )); tx } }; - log::trace!("received forward packet ({} B)", fwd.payload.len()); + log::trace!( + "Net: received forward ({}) packet ({} B) from [{}]", + fwd.transport, + fwd.payload.len(), + fwd.node_id + ); - if tx.send(fwd.payload).await.is_err() { - log::debug!("net routing error: channel closed"); - state.remove(&key); - } + tokio::task::spawn_local(async move { + if tx.send(fwd.payload).await.is_err() { + log::debug!("Net routing error: channel closed for [{}]", fwd.node_id); + state.remove_sink(&key); + } + }); } }) .boxed_local() } -fn forward_channel<'a>(reliable: bool) -> (mpsc::Sender>, LocalBoxStream<'a, Vec>) { - let (tx, rx) = mpsc::channel(1); - let rx = if reliable { +fn forward_channel<'a>( + transport: TransportType, +) -> (mpsc::Sender, LocalBoxStream<'a, Payload>) { + let (tx, rx) = mpsc::channel(8); + let rx = if transport == TransportType::Reliable || transport == TransportType::Transfer { PrefixedStream::new(rx) - .inspect_err(|e| log::debug!("prefixed stream error: {}", e)) - .filter_map(|r| async move { r.ok().map(|b| b.to_vec()) }) + .inspect_err(|e| log::debug!("Prefixed stream error: {e}")) + .filter_map(|r| async move { r.ok().map(Payload::from) }) .boxed_local() } else { rx.boxed_local() @@ -524,19 +692,26 @@ fn forward_channel<'a>(reliable: bool) -> (mpsc::Sender>, LocalBoxStream /// Forward node GSB messages from the network to the local bus fn inbound_handler( - rx: impl Stream> + 'static, + rx: impl Stream + 'static, remote_id: NodeId, - reliable: bool, + transport: TransportType, state: State, ) -> impl Future + Unpin + 'static { StreamExt::for_each(rx, move |payload| { let state = state.clone(); - log::trace!("local bus handler -> inbound message"); + log::trace!( + "local bus handler -> inbound message ({} B) from [{remote_id}]", + payload.len() + ); async move { - match codec::decode_message(payload.as_slice()) { + match codec::decode_message(payload.as_ref()) { Ok(Some(GsbMessage::CallRequest(request @ ya_sb_proto::CallRequest { .. }))) => { - handle_request(request, remote_id, state, reliable) + if request.no_reply { + handle_push(request, remote_id, state) + } else { + handle_request(request, remote_id, state, transport) + } } Ok(Some(GsbMessage::CallReply(reply @ ya_sb_proto::CallReply { .. }))) => { handle_reply(reply, remote_id, state) @@ -545,10 +720,10 @@ fn inbound_handler( request @ ya_sb_proto::BroadcastRequest { .. }, ))) => handle_broadcast(request, remote_id), Ok(None) => { - log::trace!("received a partial message"); + log::trace!("Received a partial message from {remote_id}"); Ok(()) } - Err(err) => anyhow::bail!("received message error: {}", err), + Err(err) => anyhow::bail!("Received message error: {}", err), _ => anyhow::bail!("unexpected message type"), } } @@ -561,50 +736,79 @@ fn inbound_handler( .boxed_local() } +/// Forward messages from the network to the local bus +fn handle_push( + request: ya_sb_proto::CallRequest, + remote_id: NodeId, + state: State, +) -> anyhow::Result<()> { + let caller_id = NodeId::from_str(&request.caller).ok(); + + // FIXME: implement authorization with encryption + // if !caller_id.map(|id| id == remote_id).unwrap_or(false) { + // anyhow::bail!("Invalid caller id: {}", request.caller); + // } + + let address = request.address; + let request_id = request.request_id; + let caller_id = caller_id.unwrap(); + + log::debug!("Handle push request {request_id} to {address} from {remote_id}"); + + let fut = match state.get_public_service(address.as_str()) { + Some(address) => { + log::trace!("Handle push request: calling: {address}"); + local_bus::push(&address, &request.caller, &request.data) + } + None => { + log::trace!("Handle push request failed: unknown address: {address}"); + let err = Error::GsbBadRequest(format!("Unknown address: {address}")); + return Err(err.into()); + } + }; + + tokio::task::spawn_local(async move { + let _ = fut.await; + log::debug!("Handled push request: {request_id} from: {caller_id}"); + }); + + Ok(()) +} + /// Forward messages from the network to the local bus fn handle_request( request: ya_sb_proto::CallRequest, remote_id: NodeId, state: State, - reliable: bool, + transport: TransportType, ) -> anyhow::Result<()> { let caller_id = NodeId::from_str(&request.caller).ok(); - if !caller_id.map(|id| id == remote_id).unwrap_or(false) { - anyhow::bail!("invalid caller id: {}", request.caller); - } + + // FIXME: implement authorization with encryption + // if !caller_id.map(|id| id == remote_id).unwrap_or(false) { + // anyhow::bail!("Invalid caller id: {}", request.caller); + // } let address = request.address; let caller_id = caller_id.unwrap(); let request_id = request.request_id; let request_id_chain = request_id.clone(); let request_id_filter = request_id.clone(); + let request_id_sent = request_id.clone(); - log::trace!( - "handle request {} to {} from {}", - request_id, - address, - remote_id - ); + log::debug!("Handle request {request_id} to {address} from {remote_id}"); let eos = Rc::new(AtomicBool::new(false)); let eos_map = eos.clone(); - let stream = match { - let inner = state.inner.borrow(); - inner - .services - .iter() - .find(|&id| address.starts_with(id)) - // replaces /net//test/1 --> /public/test/1 - .map(|s| address.replacen(s, net::PUBLIC_PREFIX, 1)) - } { + let stream = match state.get_public_service(address.as_str()) { Some(address) => { - log::trace!("handle request: calling: {}", address); + log::trace!("Handle request: calling: {address}"); local_bus::call_stream(&address, &request.caller, &request.data).left_stream() } None => { - log::trace!("handle request failed: unknown address: {}", address); - let err = Error::GsbBadRequest(format!("unknown address: {}", address)); + log::trace!("Handle request failed: unknown address: {address}"); + let err = Error::GsbBadRequest(format!("Unknown address: {address}")); futures::stream::once(futures::future::err(err)).right_stream() } } @@ -635,15 +839,14 @@ fn handle_request( .filter_map(move |reply| { let filtered = match codec::encode_message(reply) { Ok(vec) => { - log::trace!( - "handle request {}: reply chunk ({} B)", - request_id_filter, + log::debug!( + "Handle request {request_id_filter}: reply chunk ({} B)", vec.len() ); Some(Ok::, mpsc::SendError>(vec)) } Err(e) => { - log::debug!("handle request: encode reply error: {}", e); + log::debug!("Handle request: encode reply error: {e}"); None } }; @@ -652,13 +855,20 @@ fn handle_request( tokio::task::spawn_local( async move { - let sink = state.forward_sink(caller_id, reliable).await?; - stream.forward(sink).await?; + let mut sink = state.forward_sink(caller_id, transport).await?; + let mut stream = Box::pin(stream); + + //stream.forward(sink).await?; + while let Some(item) = stream.next().await { + sink.send(item?).await.ok(); + log::debug!("Handled request: {request_id_sent} from: {caller_id}"); + } + Ok::<_, anyhow::Error>(()) } - .then(|result| async move { + .then(move |result| async move { if let Err(e) = result { - log::debug!("reply forward error: {}", e); + log::debug!("Replying to [{caller_id}] - forward error: {e}"); } }), ); @@ -675,9 +885,7 @@ fn handle_reply( let full = reply.reply_type == ya_sb_proto::CallReplyType::Full as i32; log::debug!( - "handle reply from node {} (full: {}, code: {}, id: {}) {} B", - remote_id, - full, + "Handle reply from node {remote_id} (full: {full}, code: {}, id: {}) {} B", reply.code, reply.request_id, reply.data.len(), @@ -687,21 +895,22 @@ fn handle_reply( let inner = state.inner.borrow(); inner.requests.get(&reply.request_id).cloned() } { - Some(request) if request.remote_id == remote_id => { + // FIXME: implement authorization with encryption + Some(request) => { if full { let mut inner = state.inner.borrow_mut(); inner.requests.remove(&reply.request_id); } request } - Some(_) => anyhow::bail!("invalid caller for request id: {}", reply.request_id), None => anyhow::bail!("invalid reply request id: {}", reply.request_id), }; + let request_id = reply.request_id.clone(); let data = if reply.code == CallReplyCode::CallReplyOk as i32 { reply.data } else { - codec::encode_reply(reply).context("unable to encode reply")? + codec::encode_reply(reply).context("Unable to encode reply {request_id}")? }; tokio::task::spawn_local(async move { @@ -711,7 +920,7 @@ fn handle_reply( ResponseChunk::Part(data) }; if request.tx.send(chunk).await.is_err() { - log::debug!("failed to forward reply: channel closed"); + log::debug!("Failed to forward reply {request_id}: channel closed"); } }); @@ -725,7 +934,7 @@ fn handle_broadcast( ) -> anyhow::Result<()> { let caller_id = NodeId::from_str(&request.caller).ok(); if !caller_id.map(|id| id == remote_id).unwrap_or(false) { - anyhow::bail!("invalid broadcast caller id: {}", request.caller); + anyhow::bail!("Invalid broadcast caller id: {}", request.caller); } log::trace!( @@ -737,18 +946,24 @@ fn handle_broadcast( let caller = caller_id.unwrap().to_string(); tokio::task::spawn_local(async move { - let data: Rc<[u8]> = request.data.into(); + let data = request.data; let topic = request.topic; - let handlers = BCAST_HANDLERS.read().await; - for handler in BCAST + for endpoint in BCAST .resolve(&topic) .await .into_iter() - .filter_map(|e| handlers.get(e.as_ref())) + .map(|endpoint| endpoint.as_ref().to_string()) { - let mut h = handler.lock().await; - (*(h))(caller.clone(), data.as_ref()); + let bcast_service_id = as RpcMessage>::ID; + let addr = format!("{}/{}", endpoint, bcast_service_id); + + log::trace!( + "Forwarding broadcast from [{caller}] (topic: {topic}) to endpoint: {addr})" + ); + if let Err(e) = local_bus::send(&addr, &caller, &data).await { + log::debug!("Forwarding broadcast from [{caller}] to local endpoint error: {e}"); + } } }); @@ -779,15 +994,21 @@ impl State { } } - async fn forward_sink(&self, remote_id: NodeId, reliable: bool) -> anyhow::Result { + async fn forward_sink( + &self, + remote_id: NodeId, + transport: TransportType, + ) -> anyhow::Result { let client = CLIENT .with(|c| c.borrow().clone()) .ok_or_else(|| anyhow::anyhow!("network not started"))?; - let forward: NetSinkKind = if reliable { - PrefixedSink::new(client.forward(remote_id).await?).into() - } else { - client.forward_unreliable(remote_id).await?.into() + let forward: NetSinkKind = match transport { + TransportType::Unreliable => client.forward_unreliable(remote_id).await?.into(), + TransportType::Reliable => PrefixedSink::new(client.forward(remote_id).await?).into(), + TransportType::Transfer => { + PrefixedSink::new(client.forward_transfer(remote_id).await?).into() + } }; // FIXME: yagna daemon doesn't handle connections; ya-relay-client does @@ -800,7 +1021,14 @@ impl State { Ok(forward) } - fn remove(&self, key: &NetSinkKey) { + fn get_public_service(&self, addr: &str) -> Option { + let inner = self.inner.borrow(); + RevPrefixes(addr) + .find_map(|s| inner.services.get(s)) + .map(|s| addr.replacen(s, net::PUBLIC_PREFIX, 1)) + } + + fn remove_sink(&self, key: &NetSinkKey) { let mut inner = self.inner.borrow_mut(); inner.routes.remove(key); } @@ -808,10 +1036,11 @@ impl State { #[derive(Clone)] struct Request { - #[allow(dead_code)] + #[allow(unused)] caller_id: NodeId, + #[allow(unused)] remote_id: NodeId, - #[allow(dead_code)] + #[allow(unused)] address: String, tx: S, } @@ -845,6 +1074,7 @@ fn parse_net_to_addr(addr: &str) -> anyhow::Result<(NodeId, String)> { let (prefix, to) = match (it.next(), it.next(), it.next()) { (Some("udp"), Some("net"), Some(to)) if it.peek().is_some() => ("/udp", to), (Some("net"), Some(to), Some(_)) => ("", to), + (Some("transfer"), Some("net"), Some(to)) if it.peek().is_some() => ("/transfer", to), _ => anyhow::bail!("invalid net-to destination: {}", addr), }; @@ -864,6 +1094,11 @@ fn parse_from_to_addr(addr: &str) -> anyhow::Result<(NodeId, NodeId, String)> { ("/udp", from, to) } (Some("from"), Some(from), Some("to"), Some(to), Some(_)) => ("", from, to), + (Some("transfer"), Some("from"), Some(from), Some("to"), Some(to)) + if it.peek().is_some() => + { + ("/transfer", from, to) + } _ => anyhow::bail!("invalid net-from-to destination: {}", addr), }; @@ -880,3 +1115,114 @@ fn gen_id() -> u64 { let mut rng = rand::thread_rng(); rng.gen::() & 0x001f_ffff_ffff_ffff_u64 } + +#[cfg(test)] +mod tests { + use super::*; + use test_case::test_case; + + #[test_case( + "/net/0x95369fc6fd02afeca110b9c32a21fb8ad899ee0a/vpn/VpnControl", + NodeId::from_str("0x95369fc6fd02afeca110b9c32a21fb8ad899ee0a").unwrap(); + "net-to destination address") + ] + #[test_case( + "/transfer/net/0x95369fc6fd02afeca110b9c32a21fb8ad899ee0a/vpn/VpnControl", + NodeId::from_str("0x95369fc6fd02afeca110b9c32a21fb8ad899ee0a").unwrap(); + "Address using heavy transfer channel") + ] + #[test_case( + "/udp/net/0x95369fc6fd02afeca110b9c32a21fb8ad899ee0a/vpn/VpnControl", + NodeId::from_str("0x95369fc6fd02afeca110b9c32a21fb8ad899ee0a").unwrap(); + "Use unreliable transport protocol") + ] + fn test_parse_net_to_addr_positive(addr: &str, id: NodeId) { + let (parsed_id, parsed_gsb) = parse_net_to_addr(addr).unwrap(); + assert_eq!(id, parsed_id); + assert_eq!(addr, parsed_gsb); + } + + #[test_case( + "/net/0x95369fc6fd02afeca110b9c32a21fb8ad899ee0a"; + "net-to destination address - empty path - no trailing slash") + ] + #[test_case( + "/transfer/net/0x95369fc6fd02afeca110b9c32a21fb8ad899ee0a"; + "Address using heavy transfer channel - empty path - no trailing slash") + ] + #[test_case( + "/udp/net/0x95369fc6fd02afeca110b9c32a21fb8ad899ee0a"; + "Use unreliable transport protocol - empty path - no trailing slash") + ] + #[test_case( + "/net/0x9xxxxxc6fd02afeca110b9c32a21fb8ad899ee0a/vpn/VpnControl"; + "net-to destination address - invalid NodeId") + ] + #[test_case( + "/transfer/net/0x95369fc6fdca110b9c32a21fb8ad899ee0a/vpn/VpnControl"; + "Address using heavy transfer channel - invalid NodeId") + ] + #[test_case( + "/udp/net/0x95369fc6fd02afec32a21fb8ad899ee0a/vpn/VpnControl"; + "Use unreliable transport protocol - invalid NodeId") + ] + fn test_parse_net_to_addr_negative_cases(addr: &str) { + assert!(parse_net_to_addr(addr).is_err()) + } + + #[test_case( + "/from/0x95369fc6fd02afeca110b9c32a21fb8ad899ee0a/to/0xa5ad3f81e283983b8e9705b2e31d0c138bb2b1b7/vpn/VpnControl", + NodeId::from_str("0x95369fc6fd02afeca110b9c32a21fb8ad899ee0a").unwrap(), + NodeId::from_str("0xa5ad3f81e283983b8e9705b2e31d0c138bb2b1b7").unwrap(), + "/net/0xa5ad3f81e283983b8e9705b2e31d0c138bb2b1b7/vpn/VpnControl"; + "from-to destination address") + ] + #[test_case( + "/transfer/from/0x95369fc6fd02afeca110b9c32a21fb8ad899ee0a/to/0xa5ad3f81e283983b8e9705b2e31d0c138bb2b1b7/vpn/VpnControl", + NodeId::from_str("0x95369fc6fd02afeca110b9c32a21fb8ad899ee0a").unwrap(), + NodeId::from_str("0xa5ad3f81e283983b8e9705b2e31d0c138bb2b1b7").unwrap(), + "/transfer/net/0xa5ad3f81e283983b8e9705b2e31d0c138bb2b1b7/vpn/VpnControl"; + "from-to heavy transfer channel") + ] + #[test_case( + "/udp/from/0x95369fc6fd02afeca110b9c32a21fb8ad899ee0a/to/0xa5ad3f81e283983b8e9705b2e31d0c138bb2b1b7/vpn/VpnControl", + NodeId::from_str("0x95369fc6fd02afeca110b9c32a21fb8ad899ee0a").unwrap(), + NodeId::from_str("0xa5ad3f81e283983b8e9705b2e31d0c138bb2b1b7").unwrap(), + "/udp/net/0xa5ad3f81e283983b8e9705b2e31d0c138bb2b1b7/vpn/VpnControl"; + "from-to unreliable transport protocol") + ] + fn test_parse_from_to_addr_positive(addr: &str, from: NodeId, to: NodeId, remote_addr: &str) { + let (parsed_from, parsed_to, parsed_gsb) = parse_from_to_addr(addr).unwrap(); + assert_eq!(parsed_from, from); + assert_eq!(parsed_to, to); + assert_eq!(remote_addr, parsed_gsb); + } + + #[test_case( + "/from/0x95369fc6fd02afeca110b9c32a21fb8ad899ee0a/to/0xa5ad3f81e283983b8e9705b2e31d0c138bb2b1b7"; + "from-to destination address - empty path - no trailing slash") + ] + #[test_case( + "/transfer/from/0x95369fc6fd02afeca110b9c32a21fb8ad899ee0a/to/0xa5ad3f81e283983b8e9705b2e31d0c138bb2b1b7"; + "from-to heavy transfer channel - empty path - no trailing slash") + ] + #[test_case( + "/udp/from/0x95369fc6fd02afeca110b9c32a21fb8ad899ee0a/to/0xa5ad3f81e283983b8e9705b2e31d0c138bb2b1b7"; + "from-to unreliable transport protocol - empty path - no trailing slash") + ] + #[test_case( + "/from/0x9xxxxxc6fd02afeca110b9c32a21fb8ad899ee0a/to/0xa5ad3f81e283983b8e9705b2e31d0c138bb2b1b7/vpn/VpnControl"; + "from-to destination address - invalid NodeId") + ] + #[test_case( + "/transfer/from/0x95369fc6fdca110b9c32a21fb8ad899ee0a/to/0xa5ad3f81e283983b8e9705b2e31d0c138bb2b1b7/vpn/VpnControl"; + "from-to heavy transfer channel - invalid NodeId") + ] + #[test_case( + "/udp/from/0x95369fc6fd02afec32a21fb8ad899ee0a/to/0xa5ad3f81e283983b8e9705b2e31d0c138bb2b1b7/vpn/VpnControl"; + "from-to unreliable transport protocol - invalid NodeId") + ] + fn test_parse_from_to_addr_negative_cases(addr: &str) { + assert!(parse_from_to_addr(addr).is_err()) + } +} diff --git a/core/net/src/lib.rs b/core/net/src/lib.rs index 97d94dd239..0bb3f8cb0e 100644 --- a/core/net/src/lib.rs +++ b/core/net/src/lib.rs @@ -12,3 +12,4 @@ mod service; mod cli; mod config; +mod error; diff --git a/core/net/src/service.rs b/core/net/src/service.rs index eb72983bf1..851070d99c 100644 --- a/core/net/src/service.rs +++ b/core/net/src/service.rs @@ -2,7 +2,7 @@ use std::sync::{Arc, RwLock}; use ya_core_model::net::local::{BindBroadcastError, BroadcastMessage, SendBroadcastMessage}; use ya_core_model::{identity, NodeId}; -use ya_service_api_interfaces::Service; +use ya_service_api_interfaces::{Provider, Service}; use ya_service_bus::{Error, RpcEndpoint, RpcMessage}; use crate::config::{Config, NetType}; @@ -60,6 +60,14 @@ impl Net { } } + pub fn rest>(_: &CONTEXT) -> actix_web::Scope { + let net_type = { *NET_TYPE.read().unwrap() }; + match net_type { + NetType::Central => crate::central::web_scope(), + NetType::Hybrid => crate::hybrid::web_scope(), + } + } + pub async fn shutdown() -> anyhow::Result<()> { let config = Config::from_env()?; diff --git a/core/payment-driver/base/Cargo.toml b/core/payment-driver/base/Cargo.toml index dd42f9cbc4..4a674eb4fa 100644 --- a/core/payment-driver/base/Cargo.toml +++ b/core/payment-driver/base/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ya-payment-driver" -version = "0.2.0" +version = "0.3.0" authors = ["Golem Factory "] edition = "2018" @@ -29,9 +29,9 @@ thiserror = "1.0" tokio = { version = "1", features = ["macros"] } ## yagna dependencies -ya-client-model = "0.4" -ya-core-model = { version = "^0.6", features = ["driver", "identity", "payment"] } -ya-persistence = "0.2" -ya-service-bus = "0.4" +ya-client-model = "0.5" +ya-core-model = { version = "^0.8", features = ["driver", "identity", "payment"] } +ya-persistence = "0.3" +ya-service-bus = "0.6" [dev-dependencies] diff --git a/core/payment-driver/base/src/db/models.rs b/core/payment-driver/base/src/db/models.rs index 99bc53e63f..64469a3415 100644 --- a/core/payment-driver/base/src/db/models.rs +++ b/core/payment-driver/base/src/db/models.rs @@ -53,7 +53,7 @@ impl TryFrom for TransactionStatus { } } -#[derive(Clone, Queryable, Debug, Identifiable, Insertable, PartialEq)] +#[derive(Clone, Queryable, Debug, Identifiable, Insertable, PartialEq, Eq)] #[primary_key(tx_id)] #[table_name = "transaction"] pub struct TransactionEntity { @@ -82,7 +82,7 @@ pub struct TransactionEntity { pub encoded: String, } -#[derive(Queryable, Clone, Debug, Identifiable, Insertable, PartialEq)] +#[derive(Queryable, Clone, Debug, Identifiable, Insertable, PartialEq, Eq)] #[primary_key(order_id)] #[table_name = "payment"] pub struct PaymentEntity { @@ -97,7 +97,7 @@ pub struct PaymentEntity { pub network: Network, } -#[derive(AsExpression, FromSqlRow, PartialEq, Debug, Clone, Copy, FromPrimitive)] +#[derive(AsExpression, FromSqlRow, PartialEq, Eq, Debug, Clone, Copy, FromPrimitive)] #[sql_type = "Integer"] pub enum Network { Mainnet = 1, //Main Ethereum chain @@ -123,10 +123,7 @@ impl FromStr for Network { "goerli" => Ok(Network::Goerli), "polygon" => Ok(Network::Polygon), "mumbai" => Ok(Network::Mumbai), - _ => Err(DbError::InvalidData(format!( - "Invalid network: {}", - s.to_string() - ))), + _ => Err(DbError::InvalidData(format!("Invalid network: {}", s))), } } } diff --git a/core/payment-driver/base/src/driver.rs b/core/payment-driver/base/src/driver.rs index fba340e40f..ceb9ff7207 100644 --- a/core/payment-driver/base/src/driver.rs +++ b/core/payment-driver/base/src/driver.rs @@ -61,7 +61,13 @@ pub trait PaymentDriver { fn get_networks(&self) -> HashMap; fn recv_init_required(&self) -> bool; + /// There is no guarentee that this method will be called only once + /// AccountMode in Init message should be incremental i.e. : + /// first init with mode: Send + /// second init with mode: Recv + /// should result in driver capable of both Sending & Receiving async fn init(&self, db: DbExecutor, caller: String, msg: Init) -> Result; + async fn fund(&self, db: DbExecutor, caller: String, msg: Fund) -> Result; diff --git a/core/payment-driver/dummy/Cargo.toml b/core/payment-driver/dummy/Cargo.toml index b8a98c77ab..eb9a88d38a 100644 --- a/core/payment-driver/dummy/Cargo.toml +++ b/core/payment-driver/dummy/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ya-dummy-driver" -version = "0.2.0" +version = "0.3.0" authors = ["Golem Factory "] edition = "2018" @@ -8,12 +8,12 @@ edition = "2018" default = [] [dependencies] -ya-core-model = { version = "^0.6", features = ["driver", "identity", "payment"] } -ya-client-model = { version = "0.4" } -ya-payment-driver = "0.2" -ya-persistence = "0.2" -ya-service-api-interfaces = "0.1" -ya-service-bus = "0.4" +ya-core-model = { version = "^0.8", features = ["driver", "identity", "payment"] } +ya-client-model = { version = "0.5" } +ya-payment-driver = "0.3" +ya-persistence = "0.3" +ya-service-api-interfaces = "0.2" +ya-service-bus = "0.6" anyhow = "1.0" bigdecimal = "0.2" diff --git a/core/payment-driver/dummy/src/lib.rs b/core/payment-driver/dummy/src/lib.rs index c753a82ba7..98c62858de 100644 --- a/core/payment-driver/dummy/src/lib.rs +++ b/core/payment-driver/dummy/src/lib.rs @@ -1,9 +1,9 @@ mod service; -pub const DRIVER_NAME: &'static str = "dummy"; -pub const NETWORK_NAME: &'static str = "dummy"; -pub const TOKEN_NAME: &'static str = "GLM"; -pub const PLATFORM_NAME: &'static str = "dummy-glm"; +pub const DRIVER_NAME: &str = "dummy"; +pub const NETWORK_NAME: &str = "dummy"; +pub const TOKEN_NAME: &str = "GLM"; +pub const PLATFORM_NAME: &str = "dummy-glm"; pub struct PaymentDriverService; diff --git a/core/payment-driver/dummy/src/service.rs b/core/payment-driver/dummy/src/service.rs index abceb65772..47f77f1b8c 100644 --- a/core/payment-driver/dummy/src/service.rs +++ b/core/payment-driver/dummy/src/service.rs @@ -131,7 +131,7 @@ async fn verify_payment( let confirmation = msg.confirmation(); let json_str = std::str::from_utf8(confirmation.confirmation.as_slice()).unwrap(); - let details = serde_json::from_str(&json_str).unwrap(); + let details = serde_json::from_str(json_str).unwrap(); Ok(details) } diff --git a/core/payment-driver/erc20/Cargo.toml b/core/payment-driver/erc20/Cargo.toml index a64ee7c226..514e5872f6 100644 --- a/core/payment-driver/erc20/Cargo.toml +++ b/core/payment-driver/erc20/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ya-erc20-driver" -version = "0.3.0" +version = "0.4.0" authors = ["Golem Factory "] edition = "2018" @@ -35,11 +35,11 @@ uuid = { version = "0.8", features = ["v4"] } web3 = { version = "0.16", default-features = false, features = [ "http-tls", "signing", "ws-tls-tokio" ] } ## yagna dependencies -ya-payment-driver = "0.2" -ya-client-model = "0.4" -ya-service-api-interfaces = "0.1" -ya-utils-futures = "0.1" -ya-utils-networking = "0.1" +ya-payment-driver = "0.3" +ya-client-model = "0.5" +ya-service-api-interfaces = "0.2" +ya-utils-futures = "0.2" +ya-utils-networking = "0.2" [dev-dependencies] actix-rt = "2.7" diff --git a/core/payment-driver/erc20/src/dao.rs b/core/payment-driver/erc20/src/dao.rs index 98f812fa11..911422eb50 100644 --- a/core/payment-driver/erc20/src/dao.rs +++ b/core/payment-driver/erc20/src/dao.rs @@ -71,7 +71,7 @@ impl Erc20Dao { gas: utils::u256_to_big_endian_hex(gas_amount), order_id: order_id.to_string(), payment_due_date: msg.due_date().naive_utc(), - sender: msg.sender().clone(), + sender: msg.sender(), recipient: recipient.clone(), status: PAYMENT_STATUS_NOT_YET, tx_id: None, @@ -140,7 +140,7 @@ impl Erc20Dao { .transaction() .confirm_tx( tx_id.to_string(), - TransactionStatus::Confirmed.into(), + TransactionStatus::Confirmed, None, Some(final_hash.to_string()), final_gas_price, @@ -163,7 +163,7 @@ impl Erc20Dao { .transaction() .confirm_tx( tx_id.to_string(), - TransactionStatus::ErrorOnChain.into(), + TransactionStatus::ErrorOnChain, Some(error.to_string()), Some(final_hash.to_string()), final_gas_price, @@ -180,7 +180,7 @@ impl Erc20Dao { .transaction() .confirm_tx( tx_id.to_string(), - TransactionStatus::ErrorNonceTooLow.into(), + TransactionStatus::ErrorNonceTooLow, Some(error.to_string()), None, None, @@ -276,7 +276,7 @@ impl Erc20Dao { .transaction() .update_error_sent( tx_id.to_string(), - TransactionStatus::ErrorSent.into(), + TransactionStatus::ErrorSent, new_resent_count, Some(error.to_string()), ) diff --git a/core/payment-driver/erc20/src/driver.rs b/core/payment-driver/erc20/src/driver.rs index 1ed9c955e3..43c01c4804 100644 --- a/core/payment-driver/erc20/src/driver.rs +++ b/core/payment-driver/erc20/src/driver.rs @@ -65,8 +65,8 @@ impl Erc20Driver { pub async fn load_active_accounts(&self) { log::debug!("load_active_accounts"); - let mut accounts = self.active_accounts.borrow_mut(); let unlocked_accounts = bus::list_unlocked_identities().await.unwrap(); + let mut accounts = self.active_accounts.borrow_mut(); for account in unlocked_accounts { log::debug!("account={}", account); accounts.add_account(account) @@ -255,9 +255,10 @@ impl PaymentDriverCron for Erc20Driver { log::trace!("Running ERC-20 send-out job..."); 'outer: for network_key in self.get_networks().keys() { - let network = Network::from_str(&network_key).unwrap(); + let network = Network::from_str(network_key).unwrap(); // Process payment rows - for node_id in self.active_accounts.borrow().list_accounts() { + let accounts = self.active_accounts.borrow().list_accounts(); + for node_id in accounts { if let Err(e) = cron::process_payments_for_account(&self.dao, &node_id, network).await { diff --git a/core/payment-driver/erc20/src/driver/cron.rs b/core/payment-driver/erc20/src/driver/cron.rs index 3219b03f03..1d5e98d177 100644 --- a/core/payment-driver/erc20/src/driver/cron.rs +++ b/core/payment-driver/erc20/src/driver/cron.rs @@ -50,7 +50,7 @@ lazy_static! { } pub async fn confirm_payments(dao: &Erc20Dao, name: &str, network_key: &str) { - let network = Network::from_str(&network_key).unwrap(); + let network = Network::from_str(network_key).unwrap(); let txs = dao.get_unconfirmed_txs(network).await; //log::debug!("confirm_payments {:?}", txs); let current_time = Utc::now().naive_utc(); @@ -81,7 +81,7 @@ pub async fn confirm_payments(dao: &Erc20Dao, name: &str, network_key: &str) { }; let mut tmp_onchain_txs_vec: Vec<&str> = vec![]; - for str in tmp_onchain_txs.split(";") { + for str in tmp_onchain_txs.split(';') { if str.len() > 2 { //todo make proper validation of transaction hash tmp_onchain_txs_vec.push(str); @@ -119,19 +119,19 @@ pub async fn confirm_payments(dao: &Erc20Dao, name: &str, network_key: &str) { } } } - if tx.status == TransactionStatus::ErrorSent as i32 { - if time_elapsed_from_last_action > *ERC20_WAIT_FOR_ERROR_SENT_TRANSACTION { - log::info!("Transaction not sent, retrying"); - log::warn!( - "Transaction not found on chain for {:?}", - time_elapsed_from_sent - ); - log::warn!("Time since last action {:?}", time_elapsed_from_last_action); - dao.retry_send_transaction(&tx.tx_id, false).await; - } + if tx.status == TransactionStatus::ErrorSent as i32 + && time_elapsed_from_last_action > *ERC20_WAIT_FOR_ERROR_SENT_TRANSACTION + { + log::info!("Transaction not sent, retrying"); + log::warn!( + "Transaction not found on chain for {:?}", + time_elapsed_from_sent + ); + log::warn!("Time since last action {:?}", time_elapsed_from_last_action); + dao.retry_send_transaction(&tx.tx_id, false).await; } - if tmp_onchain_txs_vec.len() == 0 { + if tmp_onchain_txs_vec.is_empty() { continue; } @@ -165,10 +165,7 @@ pub async fn confirm_payments(dao: &Erc20Dao, name: &str, network_key: &str) { } }; - let final_gas_price = match s.gas_price { - Some(gas_price) => Some(gas_price.to_string()), - None => None, - }; + let final_gas_price = s.gas_price.map(|gas_price| gas_price.to_string()); if !s.exists_on_chain { log::info!("Transaction not found on chain"); @@ -253,13 +250,13 @@ pub async fn confirm_payments(dao: &Erc20Dao, name: &str, network_key: &str) { continue; } }; - let details = match wallet::verify_tx(&newest_tx, network).await { + let details = match wallet::verify_tx(newest_tx, network).await { Ok(a) => a, Err(e) => { log::warn!("Failed to get transaction details from erc20, creating bespoke details. Error={}", e); let first_payment: PaymentEntity = - match dao.get_first_payment(&newest_tx).await { + match dao.get_first_payment(newest_tx).await { Some(p) => p, None => continue, }; @@ -271,7 +268,7 @@ pub async fn confirm_payments(dao: &Erc20Dao, name: &str, network_key: &str) { let mut details = utils::db_to_payment_details(&first_payment); details.amount = payments .into_iter() - .map(|payment| utils::db_amount_to_big_dec(payment.amount.clone())) + .map(|payment| utils::db_amount_to_big_dec(payment.amount)) .sum::(); details } @@ -329,7 +326,7 @@ pub async fn process_payments_for_account( ); let mut nonce = wallet::get_next_nonce( dao, - crate::erc20::utils::str_to_addr(&node_id).unwrap(), + crate::erc20::utils::str_to_addr(node_id).unwrap(), network, ) .await @@ -344,7 +341,7 @@ pub async fn process_payments_for_account( log::debug!("Payments: nonce={}, details={:?}", &nonce, payments); for payment in payments { - handle_payment(&dao, payment, &mut nonce).await; + handle_payment(dao, payment, &mut nonce).await; } } Ok(()) diff --git a/core/payment-driver/erc20/src/erc20/config.rs b/core/payment-driver/erc20/src/erc20/config.rs index cf24d031ae..dad1ba7e81 100644 --- a/core/payment-driver/erc20/src/erc20/config.rs +++ b/core/payment-driver/erc20/src/erc20/config.rs @@ -23,13 +23,13 @@ lazy_static! { pub static ref RINKEBY_CONFIG: EnvConfiguration = EnvConfiguration { glm_contract_address: utils::str_to_addr( &env::var("RINKEBY_TGLM_CONTRACT_ADDRESS") - .unwrap_or("0xd94e3DC39d4Cad1DAd634e7eb585A57A19dC7EFE".to_string()) + .unwrap_or_else(|_| "0xd94e3DC39d4Cad1DAd634e7eb585A57A19dC7EFE".to_string()) ) .unwrap(), glm_faucet_address: Some( utils::str_to_addr( &env::var("RINKEBY_TGLM_FAUCET_ADDRESS") - .unwrap_or("0x59259943616265A03d775145a2eC371732E2B06C".to_string()) + .unwrap_or_else(|_| "0x59259943616265A03d775145a2eC371732E2B06C".to_string()) ) .unwrap() ), @@ -43,7 +43,7 @@ lazy_static! { pub static ref MAINNET_CONFIG: EnvConfiguration = EnvConfiguration { glm_contract_address: utils::str_to_addr( &env::var("MAINNET_GLM_CONTRACT_ADDRESS") - .unwrap_or("0x7DD9c5Cba05E151C895FDe1CF355C9A1D5DA6429".to_string()) + .unwrap_or_else(|_| "0x7DD9c5Cba05E151C895FDe1CF355C9A1D5DA6429".to_string()) ) .unwrap(), glm_faucet_address: None, @@ -57,7 +57,7 @@ lazy_static! { pub static ref GOERLI_CONFIG: EnvConfiguration = EnvConfiguration { glm_contract_address: utils::str_to_addr( &env::var("GOERLI_TGLM_CONTRACT_ADDRESS") - .unwrap_or("0x33af15c79d64b85ba14aaffaa4577949104b22e8".to_string()) + .unwrap_or_else(|_| "0x33af15c79d64b85ba14aaffaa4577949104b22e8".to_string()) ) .unwrap(), glm_faucet_address: None, @@ -71,7 +71,7 @@ lazy_static! { pub static ref MUMBAI_CONFIG: EnvConfiguration = EnvConfiguration { glm_contract_address: utils::str_to_addr( &env::var("MUMBAI_TGLM_CONTRACT_ADDRESS") - .unwrap_or("0x2036807B0B3aaf5b1858EE822D0e111fDdac7018".to_string()) + .unwrap_or_else(|_| "0x2036807B0B3aaf5b1858EE822D0e111fDdac7018".to_string()) ) .unwrap(), glm_faucet_address: None, @@ -85,7 +85,7 @@ lazy_static! { pub static ref POLYGON_MAINNET_CONFIG: EnvConfiguration = EnvConfiguration { glm_contract_address: utils::str_to_addr( &env::var("POLYGON_GLM_CONTRACT_ADDRESS") - .unwrap_or("0x0b220b82f3ea3b7f6d9a1d8ab58930c064a2b5bf".to_string()) + .unwrap_or_else(|_| "0x0b220b82f3ea3b7f6d9a1d8ab58930c064a2b5bf".to_string()) ) .unwrap(), glm_faucet_address: None, diff --git a/core/payment-driver/erc20/src/erc20/eth_utils.rs b/core/payment-driver/erc20/src/erc20/eth_utils.rs index cfc16cfbe2..cbc8dce1cb 100644 --- a/core/payment-driver/erc20/src/erc20/eth_utils.rs +++ b/core/payment-driver/erc20/src/erc20/eth_utils.rs @@ -22,7 +22,7 @@ pub fn keccak256_hash(bytes: &[u8]) -> Vec { hasher.update(bytes); let mut resp: [u8; 32] = Default::default(); hasher.finalize(&mut resp); - resp.iter().cloned().collect() + resp.to_vec() } fn tx_encode(tx: &YagnaRawTransaction, s: &mut RlpStream) { @@ -51,7 +51,7 @@ pub fn encode_signed_tx( tx.begin_unbounded_list(); - tx_encode(&raw_tx, &mut tx); + tx_encode(raw_tx, &mut tx); tx.append(&sig_v); tx.append(&sig_r); tx.append(&sig_s); @@ -61,14 +61,14 @@ pub fn encode_signed_tx( tx.out().to_vec() } -fn prepare_signature(signature: Vec, chain_id: u64) -> (u64, Vec, Vec) { +fn prepare_signature(mut signature: Vec, chain_id: u64) -> (u64, Vec, Vec) { // TODO ugly solution assert_eq!(signature.len(), 65); let sig_v = signature[0]; let sig_v = sig_v as u64 + chain_id * 2 + 35; - let mut sig_r = signature.to_owned().split_off(1); + let mut sig_r = signature.split_off(1); let mut sig_s = sig_r.split_off(32); prepare_signature_part(&mut sig_r); diff --git a/core/payment-driver/erc20/src/erc20/ethereum.rs b/core/payment-driver/erc20/src/erc20/ethereum.rs index 6451940a4c..51215e89e0 100644 --- a/core/payment-driver/erc20/src/erc20/ethereum.rs +++ b/core/payment-driver/erc20/src/erc20/ethereum.rs @@ -1,3 +1,5 @@ +#![allow(clippy::too_many_arguments)] + use std::collections::HashMap; use std::sync::Arc; @@ -106,10 +108,10 @@ pub fn get_polygon_maximum_price() -> f64 { } pub fn get_polygon_max_gas_price_dynamic() -> f64 { - return std::env::var("POLYGON_MAX_GAS_PRICE_DYNAMIC") + std::env::var("POLYGON_MAX_GAS_PRICE_DYNAMIC") .ok() .and_then(|v| v.parse().ok()) - .unwrap_or(1000.0f64); + .unwrap_or(1000.0f64) } pub fn get_polygon_gas_price_method() -> PolygonGasPriceMethod { @@ -127,7 +129,7 @@ pub fn get_polygon_gas_price_method() -> PolygonGasPriceMethod { pub fn get_polygon_priority() -> PolygonPriority { match std::env::var("POLYGON_PRIORITY") - .unwrap_or("default".to_string()) + .unwrap_or_else(|_| "default".to_string()) .to_lowercase() .as_str() { @@ -222,7 +224,7 @@ where } pub async fn block_number(network: Network) -> Result { - with_clients(network, |client| block_number_with(client)).await + with_clients(network, block_number_with).await } async fn block_number_with(client: Web3) -> Result { @@ -292,7 +294,7 @@ pub async fn sign_raw_transfer_transaction( ) -> Result, GenericError> { let chain_id = network as u64; let node_id = NodeId::from(address.as_ref()); - let signature = bus::sign(node_id, eth_utils::get_tx_hash(&tx, chain_id)).await?; + let signature = bus::sign(node_id, eth_utils::get_tx_hash(tx, chain_id)).await?; Ok(signature) } @@ -359,9 +361,9 @@ async fn prepare_raw_transaction_with( }; let gas_limit = match network { - Network::Polygon => gas_limit_override.map_or(*GLM_POLYGON_GAS_LIMIT, |v| U256::from(v)), - Network::Mumbai => gas_limit_override.map_or(*GLM_POLYGON_GAS_LIMIT, |v| U256::from(v)), - _ => gas_limit_override.map_or(*GLM_TRANSFER_GAS, |v| U256::from(v)), + Network::Polygon => gas_limit_override.map_or(*GLM_POLYGON_GAS_LIMIT, U256::from), + Network::Mumbai => gas_limit_override.map_or(*GLM_POLYGON_GAS_LIMIT, U256::from), + _ => gas_limit_override.map_or(*GLM_TRANSFER_GAS, U256::from), }; let tx = YagnaRawTransaction { @@ -550,7 +552,7 @@ fn get_rpc_addr_from_env(network: Network) -> Vec { fn collect_rpc_addr_from(env: &str, default: &str) -> Vec { std::env::var(env) .ok() - .unwrap_or(default.to_string()) + .unwrap_or_else(|| default.to_string()) .split(',') .map(|path| path.to_string()) .collect() diff --git a/core/payment-driver/erc20/src/erc20/faucet.rs b/core/payment-driver/erc20/src/erc20/faucet.rs index 1c3a9211bf..b88b0cc0c7 100644 --- a/core/payment-driver/erc20/src/erc20/faucet.rs +++ b/core/payment-driver/erc20/src/erc20/faucet.rs @@ -78,7 +78,7 @@ pub async fn request_glm( return Ok(()); } let pending = dao.get_pending_faucet_txs(&str_addr, network).await; - for _tx in pending { + if !pending.is_empty() { log::info!("Already pending a mint transactin."); return Ok(()); } @@ -161,7 +161,7 @@ async fn resolve_faucet_url() -> Result { _ => { let faucet_host = resolver::resolve_yagna_srv_record(DEFAULT_FAUCET_SRV_PREFIX) .await - .unwrap_or(DEFAULT_ETH_FAUCET_HOST.to_string()); + .unwrap_or_else(|_| DEFAULT_ETH_FAUCET_HOST.to_string()); Ok(format!("http://{}:4000/donate", faucet_host)) } diff --git a/core/payment-driver/erc20/src/erc20/gasless_transfer.rs b/core/payment-driver/erc20/src/erc20/gasless_transfer.rs index 9baa9d0ee5..b8abbb05fc 100644 --- a/core/payment-driver/erc20/src/erc20/gasless_transfer.rs +++ b/core/payment-driver/erc20/src/erc20/gasless_transfer.rs @@ -155,7 +155,7 @@ pub async fn send_gasless_transfer( Err(GenericError::new(err.message)) } - status @ _ => { + status => { let resp_bytes = resp.body().await.map_err(GenericError::new)?; Err(GenericError::new(format!( @@ -167,5 +167,5 @@ pub async fn send_gasless_transfer( } fn resolve_gasless_url() -> String { - env::var(GASLESS_ADDR_ENVAR).unwrap_or(DEFAULT_GASLESS_HOST.to_string()) + env::var(GASLESS_ADDR_ENVAR).unwrap_or_else(|_| DEFAULT_GASLESS_HOST.to_string()) } diff --git a/core/payment-driver/erc20/src/erc20/transaction.rs b/core/payment-driver/erc20/src/erc20/transaction.rs index 05325410c2..f204918ce5 100644 --- a/core/payment-driver/erc20/src/erc20/transaction.rs +++ b/core/payment-driver/erc20/src/erc20/transaction.rs @@ -1,7 +1,7 @@ use ethereum_types::{H160, U256}; use serde::{Deserialize, Serialize}; -#[derive(Debug, Default, Clone, PartialEq, Deserialize, Serialize)] +#[derive(Debug, Default, Clone, PartialEq, Eq, Deserialize, Serialize)] pub struct YagnaRawTransaction { /// Nonce value pub nonce: U256, diff --git a/core/payment-driver/erc20/src/erc20/utils.rs b/core/payment-driver/erc20/src/erc20/utils.rs index 30e95401c1..1eea29112e 100644 --- a/core/payment-driver/erc20/src/erc20/utils.rs +++ b/core/payment-driver/erc20/src/erc20/utils.rs @@ -22,18 +22,18 @@ pub fn big_dec_to_u256(v: &BigDecimal) -> Result { let v = v * &(*PRECISION); let v = v .to_bigint() - .ok_or(GenericError::new("Failed to convert to bigint"))?; + .ok_or_else(|| GenericError::new("Failed to convert to bigint"))?; let v = &v.to_string(); - Ok(U256::from_dec_str(v).map_err(GenericError::new)?) + U256::from_dec_str(v).map_err(GenericError::new) } pub fn big_dec_gwei_to_u256(v: BigDecimal) -> Result { let v = v * &(*GWEI_PRECISION); let v = v .to_bigint() - .ok_or(GenericError::new("Failed to convert to bigint"))?; + .ok_or_else(|| GenericError::new("Failed to convert to bigint"))?; let v = &v.to_string(); - Ok(U256::from_dec_str(v).map_err(GenericError::new)?) + U256::from_dec_str(v).map_err(GenericError::new) } pub fn u256_to_big_dec(v: U256) -> Result { @@ -61,7 +61,7 @@ pub fn str_to_addr(addr: &str) -> Result { Ok(addr) => Ok(addr), Err(_e) => Err(GenericError::new(format!( "Unable to parse address {}", - addr.to_string() + addr ))), } } @@ -73,15 +73,16 @@ pub fn convert_float_gas_to_u256(gas_in_gwei: f64) -> U256 { } pub fn convert_u256_gas_to_float(gas_in_wei: U256) -> f64 { let gas_in_wei = gas_in_wei.as_u64() as f64; - let gas_in_gwei = gas_in_wei * 1.0E-9; - gas_in_gwei + + gas_in_wei * 1.0E-9 } pub fn gas_float_equals(gas_value1: f64, gas_value2: f64) -> bool { - if gas_value1 > 0.0 && gas_value2 > 0.0 { - if (gas_value1 - gas_value2).abs() / (gas_value1 + gas_value2) < 0.0001 { - return true; - } + if gas_value1 > 0.0 + && gas_value2 > 0.0 + && (gas_value1 - gas_value2).abs() / (gas_value1 + gas_value2) < 0.0001 + { + return true; } if gas_value1 == 0.0 && gas_value2 == 0.0 { return true; diff --git a/core/payment-driver/erc20/src/erc20/wallet.rs b/core/payment-driver/erc20/src/erc20/wallet.rs index 5d23d1ca56..472a501225 100644 --- a/core/payment-driver/erc20/src/erc20/wallet.rs +++ b/core/payment-driver/erc20/src/erc20/wallet.rs @@ -74,8 +74,8 @@ pub async fn init_wallet(msg: &Init) -> Result<(), GenericError> { log::debug!("init_wallet. msg={:?}", msg); let mode = msg.mode(); let address = msg.address(); - let network = msg.network().unwrap_or(RINKEBY_NETWORK.to_string()); - let network = Network::from_str(&network).map_err(|e| GenericError::new(e))?; + let network = msg.network().unwrap_or_else(|| RINKEBY_NETWORK.to_string()); + let network = Network::from_str(&network).map_err(GenericError::new)?; if mode.contains(AccountMode::SEND) { let h160_addr = str_to_addr(&address)?; diff --git a/core/payment-driver/erc20/src/lib.rs b/core/payment-driver/erc20/src/lib.rs index f5867fe385..4cbe2ce4cd 100644 --- a/core/payment-driver/erc20/src/lib.rs +++ b/core/payment-driver/erc20/src/lib.rs @@ -5,37 +5,37 @@ */ // Public -pub const DRIVER_NAME: &'static str = "erc20"; - -pub const RINKEBY_NETWORK: &'static str = "rinkeby"; -pub const RINKEBY_TOKEN: &'static str = "tGLM"; -pub const RINKEBY_PLATFORM: &'static str = "erc20-rinkeby-tglm"; -pub const RINKEBY_CURRENCY_SHORT: &'static str = "tETH"; -pub const RINKEBY_CURRENCY_LONG: &'static str = "Rinkeby Ether"; - -pub const GOERLI_NETWORK: &'static str = "goerli"; -pub const GOERLI_TOKEN: &'static str = "tGLM"; -pub const GOERLI_PLATFORM: &'static str = "erc20-goerli-tglm"; -pub const GOERLI_CURRENCY_SHORT: &'static str = "tETH"; -pub const GOERLI_CURRENCY_LONG: &'static str = "Goerli Ether"; - -pub const MUMBAI_NETWORK: &'static str = "mumbai"; -pub const MUMBAI_TOKEN: &'static str = "tGLM"; -pub const MUMBAI_PLATFORM: &'static str = "erc20-mumbai-tglm"; -pub const MUMBAI_CURRENCY_SHORT: &'static str = "tMATIC"; -pub const MUMBAI_CURRENCY_LONG: &'static str = "Test MATIC"; - -pub const MAINNET_NETWORK: &'static str = "mainnet"; -pub const MAINNET_TOKEN: &'static str = "GLM"; -pub const MAINNET_PLATFORM: &'static str = "erc20-mainnet-glm"; -pub const MAINNET_CURRENCY_SHORT: &'static str = "ETH"; -pub const MAINNET_CURRENCY_LONG: &'static str = "Ether"; - -pub const POLYGON_MAINNET_NETWORK: &'static str = "polygon"; -pub const POLYGON_MAINNET_TOKEN: &'static str = "GLM"; -pub const POLYGON_MAINNET_PLATFORM: &'static str = "erc20-polygon-glm"; -pub const POLYGON_MAINNET_CURRENCY_SHORT: &'static str = "MATIC"; -pub const POLYGON_MAINNET_CURRENCY_LONG: &'static str = "Polygon"; +pub const DRIVER_NAME: &str = "erc20"; + +pub const RINKEBY_NETWORK: &str = "rinkeby"; +pub const RINKEBY_TOKEN: &str = "tGLM"; +pub const RINKEBY_PLATFORM: &str = "erc20-rinkeby-tglm"; +pub const RINKEBY_CURRENCY_SHORT: &str = "tETH"; +pub const RINKEBY_CURRENCY_LONG: &str = "Rinkeby Ether"; + +pub const GOERLI_NETWORK: &str = "goerli"; +pub const GOERLI_TOKEN: &str = "tGLM"; +pub const GOERLI_PLATFORM: &str = "erc20-goerli-tglm"; +pub const GOERLI_CURRENCY_SHORT: &str = "tETH"; +pub const GOERLI_CURRENCY_LONG: &str = "Goerli Ether"; + +pub const MUMBAI_NETWORK: &str = "mumbai"; +pub const MUMBAI_TOKEN: &str = "tGLM"; +pub const MUMBAI_PLATFORM: &str = "erc20-mumbai-tglm"; +pub const MUMBAI_CURRENCY_SHORT: &str = "tMATIC"; +pub const MUMBAI_CURRENCY_LONG: &str = "Test MATIC"; + +pub const MAINNET_NETWORK: &str = "mainnet"; +pub const MAINNET_TOKEN: &str = "GLM"; +pub const MAINNET_PLATFORM: &str = "erc20-mainnet-glm"; +pub const MAINNET_CURRENCY_SHORT: &str = "ETH"; +pub const MAINNET_CURRENCY_LONG: &str = "Ether"; + +pub const POLYGON_MAINNET_NETWORK: &str = "polygon"; +pub const POLYGON_MAINNET_TOKEN: &str = "GLM"; +pub const POLYGON_MAINNET_PLATFORM: &str = "erc20-polygon-glm"; +pub const POLYGON_MAINNET_CURRENCY_SHORT: &str = "MATIC"; +pub const POLYGON_MAINNET_CURRENCY_LONG: &str = "Polygon"; pub use service::Erc20Service as PaymentDriverService; diff --git a/core/payment-driver/erc20/src/network.rs b/core/payment-driver/erc20/src/network.rs index 6a2616edbf..d54ac9f419 100644 --- a/core/payment-driver/erc20/src/network.rs +++ b/core/payment-driver/erc20/src/network.rs @@ -89,7 +89,7 @@ pub fn network_token_to_platform( } }; - let token = token.unwrap_or(network_config.default_token.clone()); + let token = token.unwrap_or_else(|| network_config.default_token.clone()); let platform = network_config.tokens.get(&token); let platform = match platform { Some(p) => p, @@ -136,7 +136,7 @@ pub fn get_network_token(network: DbNetwork, token: Option) -> String { // Fetch network config, safe as long as all DbNetwork entries are in SUPPORTED_NETWORKS let network_config = (*SUPPORTED_NETWORKS).get(&(network.to_string())).unwrap(); // TODO: Check if token in network.tokens - token.unwrap_or(network_config.default_token.clone()) + token.unwrap_or_else(|| network_config.default_token.clone()) } pub fn network_like_to_network(network_like: Option) -> DbNetwork { diff --git a/core/payment-driver/zksync/Cargo.toml b/core/payment-driver/zksync/Cargo.toml index f98f8fceee..196ef1f09a 100644 --- a/core/payment-driver/zksync/Cargo.toml +++ b/core/payment-driver/zksync/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ya-zksync-driver" -version = "0.2.0" +version = "0.3.0" authors = ["Golem Factory "] edition = "2018" @@ -32,11 +32,11 @@ zksync = { git = "https://github.com/matter-labs/zksync", rev = "0e28e238f71b3be zksync_eth_signer = { git = "https://github.com/matter-labs/zksync", rev = "0e28e238f71b3be128e4a760b0147e7c62d8dee5"} ## yagna dependencies -ya-payment-driver = "0.2" -ya-client-model = "0.4" -ya-service-api-interfaces = "0.1" -ya-utils-futures = "0.1" -ya-utils-networking = "0.1" +ya-payment-driver = "0.3" +ya-client-model = "0.5" +ya-service-api-interfaces = "0.2" +ya-utils-futures = "0.2" +ya-utils-networking = "0.2" [dev-dependencies] actix-rt = "2.7" diff --git a/core/payment-driver/zksync/examples/deposit.rs b/core/payment-driver/zksync/examples/deposit.rs index 3c8250e610..cfebbc237e 100644 --- a/core/payment-driver/zksync/examples/deposit.rs +++ b/core/payment-driver/zksync/examples/deposit.rs @@ -13,7 +13,7 @@ const PRIVATE_KEY: &str = "e0c704b6e925c3be222337f9c94610c46b7fec95c14b8f5b9800d #[actix_rt::main] async fn main() -> anyhow::Result<()> { - let log_level = std::env::var("RUST_LOG").unwrap_or("info".to_owned()); + let log_level = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_owned()); std::env::set_var("RUST_LOG", log_level); env_logger::init(); diff --git a/core/payment-driver/zksync/examples/simple_balance.rs b/core/payment-driver/zksync/examples/simple_balance.rs index cbd0069d8b..f6c5249b39 100644 --- a/core/payment-driver/zksync/examples/simple_balance.rs +++ b/core/payment-driver/zksync/examples/simple_balance.rs @@ -9,7 +9,7 @@ extern crate log; #[tokio::main] async fn main() { - let log_level = std::env::var("RUST_LOG").unwrap_or("info".to_owned()); + let log_level = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_owned()); std::env::set_var("RUST_LOG", log_level); env_logger::init(); info!("Simple balance check example."); @@ -26,13 +26,13 @@ async fn main() { let balance_com = acc_info .committed .balances - .get(&token as &str) + .get(token as &str) .map(|x| x.0.clone()) .unwrap_or_default(); let balance_ver = acc_info .verified .balances - .get(&token as &str) + .get(token as &str) .map(|x| x.0.clone()) .unwrap_or_default(); diff --git a/core/payment-driver/zksync/examples/withdrawal.rs b/core/payment-driver/zksync/examples/withdrawal.rs index ec33d83634..e98df33bd4 100644 --- a/core/payment-driver/zksync/examples/withdrawal.rs +++ b/core/payment-driver/zksync/examples/withdrawal.rs @@ -26,7 +26,7 @@ struct Args { #[actix_rt::main] async fn main() -> anyhow::Result<()> { - let log_level = std::env::var("RUST_LOG").unwrap_or("info".to_owned()); + let log_level = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_owned()); std::env::set_var("RUST_LOG", log_level); env_logger::init(); @@ -52,7 +52,7 @@ async fn main() -> anyhow::Result<()> { let cred = WalletCredentials::from_eth_signer(address, signer, Network::Rinkeby).await?; let wallet = Wallet::new(provider, cred).await?; - if wallet.is_signing_key_set().await? == false { + if !(wallet.is_signing_key_set().await?) { info!("Unlocking account"); let unlock = wallet .start_change_pubkey() diff --git a/core/payment-driver/zksync/src/dao.rs b/core/payment-driver/zksync/src/dao.rs index f3ff9b8a8e..96121c387d 100644 --- a/core/payment-driver/zksync/src/dao.rs +++ b/core/payment-driver/zksync/src/dao.rs @@ -73,7 +73,7 @@ impl ZksyncDao { gas: utils::u256_to_big_endian_hex(gas_amount), order_id: order_id.to_string(), payment_due_date: msg.due_date().naive_utc(), - sender: msg.sender().clone(), + sender: msg.sender(), recipient: recipient.clone(), status: PAYMENT_STATUS_NOT_YET, tx_id: None, @@ -136,7 +136,7 @@ impl ZksyncDao { pub async fn transaction_confirmed(&self, tx_id: &str) -> Vec { if let Err(e) = self .transaction() - .update_tx_status(tx_id.to_string(), TransactionStatus::Confirmed.into(), None) + .update_tx_status(tx_id.to_string(), TransactionStatus::Confirmed, None) .await { log::error!("Failed to update tx status for {:?} : {:?}", tx_id, e) @@ -184,7 +184,7 @@ impl ZksyncDao { .transaction() .update_tx_status( tx_id.to_string(), - TransactionStatus::ErrorOnChain.into(), + TransactionStatus::ErrorOnChain, Some(err.to_string()), ) .await diff --git a/core/payment-driver/zksync/src/driver.rs b/core/payment-driver/zksync/src/driver.rs index f347bf7daf..da1d84e670 100644 --- a/core/payment-driver/zksync/src/driver.rs +++ b/core/payment-driver/zksync/src/driver.rs @@ -85,8 +85,8 @@ impl ZksyncDriver { pub async fn load_active_accounts(&self) { log::debug!("load_active_accounts"); - let mut accounts = self.active_accounts.borrow_mut(); let unlocked_accounts = bus::list_unlocked_identities().await.unwrap(); + let mut accounts = self.active_accounts.borrow_mut(); for account in unlocked_accounts { log::debug!("account={}", account); accounts.add_account(account) @@ -104,7 +104,7 @@ impl ZksyncDriver { async fn process_payments_for_account(&self, node_id: &str) { log::trace!("Processing payments for node_id={}", node_id); for network_key in self.get_networks().keys() { - let network = DbNetwork::from_str(&network_key).unwrap(); + let network = DbNetwork::from_str(network_key).unwrap(); let payments: Vec = self.dao.get_pending_payments(node_id, network).await; let mut nonce = 0; @@ -263,7 +263,7 @@ impl PaymentDriver for ZksyncDriver { .await .map_err(GenericError::new)??; - let network = msg.network().unwrap_or(DEFAULT_NETWORK.to_string()); + let network = msg.network().unwrap_or_else(|| DEFAULT_NETWORK.to_string()); let token = get_network_token( DbNetwork::from_str(&network).map_err(GenericError::new)?, msg.token(), @@ -288,8 +288,9 @@ impl PaymentDriver for ZksyncDriver { msg: Fund, ) -> Result { let address = msg.address(); - let network = DbNetwork::from_str(&msg.network().unwrap_or(DEFAULT_NETWORK.to_string())) - .map_err(GenericError::new)?; + let network = + DbNetwork::from_str(&msg.network().unwrap_or_else(|| DEFAULT_NETWORK.to_string())) + .map_err(GenericError::new)?; match network { DbNetwork::Rinkeby => { log::info!( @@ -306,9 +307,11 @@ impl PaymentDriver for ZksyncDriver { &address )) } - DbNetwork::Goerli => Ok(format!("Goerli network is not supported by this driver.")), - DbNetwork::Mumbai => Ok(format!("Mumbai network is not supported by this driver.")), - DbNetwork::Polygon => Ok(format!("Polygon network is not supported by this driver.")), + DbNetwork::Goerli => Ok("Goerli network is not supported by this driver.".to_string()), + DbNetwork::Mumbai => Ok("Mumbai network is not supported by this driver.".to_string()), + DbNetwork::Polygon => { + Ok("Polygon network is not supported by this driver.".to_string()) + } DbNetwork::Mainnet => Ok(format!( r#"Using this driver is not recommended. Consider using the Polygon driver instead. @@ -435,7 +438,7 @@ impl PaymentDriverCron for ZksyncDriver { for network_key in self.get_networks().keys() { let network = - match DbNetwork::from_str(&network_key) { + match DbNetwork::from_str(network_key) { Ok(n) => n, Err(e) => { log::error!( @@ -459,7 +462,7 @@ impl PaymentDriverCron for ZksyncDriver { &network, &tx_hash ); - let tx_success = match wallet::check_tx(&tx_hash, network).await { + let tx_success = match wallet::check_tx(tx_hash, network).await { None => continue, // Check_tx returns None when the result is unknown Some(tx_success) => tx_success, }; @@ -511,7 +514,7 @@ impl PaymentDriverCron for ZksyncDriver { // TODO: Add token support let platform = network_token_to_platform(Some(network), None).unwrap(); // TODO: Catch error? - let details = match wallet::verify_tx(&tx_hash, network).await { + let details = match wallet::verify_tx(tx_hash, network).await { Ok(a) => a, Err(e) => { log::warn!("Failed to get transaction details from zksync, creating bespoke details. Error={}", e); @@ -521,14 +524,14 @@ impl PaymentDriverCron for ZksyncDriver { // - Date is always now // - Amount needs to be updated to total of all PaymentEntity's let first_payment: PaymentEntity = - match self.dao.get_first_payment(&tx_hash).await { + match self.dao.get_first_payment(tx_hash).await { Some(p) => p, None => continue, }; let mut details = utils::db_to_payment_details(&first_payment); details.amount = payments .into_iter() - .map(|payment| utils::db_amount_to_big_dec(payment.amount.clone())) + .map(|payment| utils::db_amount_to_big_dec(payment.amount)) .sum::(); details } @@ -563,7 +566,8 @@ impl PaymentDriverCron for ZksyncDriver { Some(guard) => guard, }; log::trace!("Running zkSync send-out job..."); - for node_id in self.active_accounts.borrow().list_accounts() { + let accounts = self.active_accounts.borrow().list_accounts(); + for node_id in accounts { self.process_payments_for_account(&node_id).await; } log::trace!("ZkSync send-out job complete."); diff --git a/core/payment-driver/zksync/src/lib.rs b/core/payment-driver/zksync/src/lib.rs index b03c2b7a02..2787a9cf97 100644 --- a/core/payment-driver/zksync/src/lib.rs +++ b/core/payment-driver/zksync/src/lib.rs @@ -5,15 +5,15 @@ */ // Public -pub const DRIVER_NAME: &'static str = "zksync"; +pub const DRIVER_NAME: &str = "zksync"; -pub const DEFAULT_NETWORK: &'static str = "rinkeby"; -pub const DEFAULT_TOKEN: &'static str = "tGLM"; -pub const DEFAULT_PLATFORM: &'static str = "zksync-rinkeby-tglm"; +pub const DEFAULT_NETWORK: &str = "rinkeby"; +pub const DEFAULT_TOKEN: &str = "tGLM"; +pub const DEFAULT_PLATFORM: &str = "zksync-rinkeby-tglm"; -pub const MAINNET_NETWORK: &'static str = "mainnet"; -pub const MAINNET_TOKEN: &'static str = "GLM"; -pub const MAINNET_PLATFORM: &'static str = "zksync-mainnet-glm"; +pub const MAINNET_NETWORK: &str = "mainnet"; +pub const MAINNET_TOKEN: &str = "GLM"; +pub const MAINNET_PLATFORM: &str = "zksync-mainnet-glm"; pub use service::ZksyncService as PaymentDriverService; diff --git a/core/payment-driver/zksync/src/network.rs b/core/payment-driver/zksync/src/network.rs index 7a04ca3eee..c92a9a2024 100644 --- a/core/payment-driver/zksync/src/network.rs +++ b/core/payment-driver/zksync/src/network.rs @@ -58,7 +58,7 @@ pub fn network_token_to_platform( } }; - let token = token.unwrap_or(network_config.default_token.clone()); + let token = token.unwrap_or_else(|| network_config.default_token.clone()); let platform = network_config.tokens.get(&token); let platform = match platform { Some(p) => p, @@ -76,5 +76,5 @@ pub fn get_network_token(network: DbNetwork, token: Option) -> String { // Fetch network config, safe as long as all DbNetwork entries are in SUPPORTED_NETWORKS let network_config = (*SUPPORTED_NETWORKS).get(&(network.to_string())).unwrap(); // TODO: Check if token in network.tokens - token.unwrap_or(network_config.default_token.clone()) + token.unwrap_or_else(|| network_config.default_token.clone()) } diff --git a/core/payment-driver/zksync/src/zksync/signer.rs b/core/payment-driver/zksync/src/zksync/signer.rs index 307db702e3..a45be24f80 100644 --- a/core/payment-driver/zksync/src/zksync/signer.rs +++ b/core/payment-driver/zksync/src/zksync/signer.rs @@ -90,10 +90,10 @@ fn convert_to_eth_byte_order(signature: Vec) -> Vec { let r = &signature[1..33]; let s = &signature[33..65]; let mut result = Vec::with_capacity(65); - result.extend_from_slice(&r); - result.extend_from_slice(&s); + result.extend_from_slice(r); + result.extend_from_slice(s); result.push(if v % 2 == 1 { 0x1c } else { 0x1b }); - result.into() + result } fn sign_tx( @@ -121,7 +121,7 @@ fn encode_signed_tx(raw_tx: &RawTransaction, signature: Vec, chain_id: u64) tx.begin_unbounded_list(); - tx_encode(&raw_tx, &mut tx); + tx_encode(raw_tx, &mut tx); tx.append(&sig_v); tx.append(&sig_r); tx.append(&sig_s); @@ -131,14 +131,14 @@ fn encode_signed_tx(raw_tx: &RawTransaction, signature: Vec, chain_id: u64) tx.out().to_vec() } -fn prepare_signature(signature: Vec, chain_id: u64) -> (u64, Vec, Vec) { +fn prepare_signature(mut signature: Vec, chain_id: u64) -> (u64, Vec, Vec) { // TODO ugly solution assert_eq!(signature.len(), 65); let sig_v = signature[0]; let sig_v = sig_v as u64 + chain_id * 2 + 35; - let mut sig_r = signature.to_owned().split_off(1); + let mut sig_r = signature.split_off(1); let mut sig_s = sig_r.split_off(32); prepare_signature_part(&mut sig_r); diff --git a/core/payment-driver/zksync/src/zksync/utils.rs b/core/payment-driver/zksync/src/zksync/utils.rs index 945e0247d7..5f927e6e13 100644 --- a/core/payment-driver/zksync/src/zksync/utils.rs +++ b/core/payment-driver/zksync/src/zksync/utils.rs @@ -20,10 +20,10 @@ pub fn big_dec_to_big_uint(v: BigDecimal) -> Result { let v = v * &(*PRECISION); let v = v .to_bigint() - .ok_or(GenericError::new("Failed to convert to bigint"))?; + .ok_or_else(|| GenericError::new("Failed to convert to bigint"))?; let v = v .to_biguint() - .ok_or(GenericError::new("Failed to convert to biguint"))?; + .ok_or_else(|| GenericError::new("Failed to convert to biguint"))?; Ok(v) } @@ -34,7 +34,7 @@ pub fn big_uint_to_big_dec(v: BigUint) -> BigDecimal { /// Find the closest **bigger** packable amount pub fn pack_up(amount: &BigUint) -> BigUint { - let mut packable_amount = closest_packable_token_amount(&amount); + let mut packable_amount = closest_packable_token_amount(amount); while (&packable_amount < amount) || !is_token_amount_packable(&packable_amount) { packable_amount = increase_least_significant_digit(&packable_amount); } @@ -43,8 +43,8 @@ pub fn pack_up(amount: &BigUint) -> BigUint { fn increase_least_significant_digit(amount: &BigUint) -> BigUint { let digits = amount.to_radix_le(10); - for i in 0..digits.len() { - if digits[i] != 0 { + for (i, digit) in digits.iter().enumerate() { + if *digit != 0 { return amount + BigUint::from(10u32).pow(i as u32); } } diff --git a/core/payment-driver/zksync/src/zksync/wallet.rs b/core/payment-driver/zksync/src/zksync/wallet.rs index 8d5b9753ac..3d4c8e2e08 100644 --- a/core/payment-driver/zksync/src/zksync/wallet.rs +++ b/core/payment-driver/zksync/src/zksync/wallet.rs @@ -48,7 +48,7 @@ pub async fn account_balance(address: &str, network: Network) -> Result Result<(), GenericError> { log::debug!("init_wallet. msg={:?}", msg); let mode = msg.mode(); let address = msg.address().clone(); - let network = msg.network().unwrap_or(DEFAULT_NETWORK.to_string()); - let network = Network::from_str(&network).map_err(|e| GenericError::new(e))?; + let network = msg.network().unwrap_or_else(|| DEFAULT_NETWORK.to_string()); + let network = Network::from_str(&network).map_err(GenericError::new)?; if mode.contains(AccountMode::SEND) { let wallet = get_wallet(&address, network).await?; @@ -87,8 +87,8 @@ pub async fn fund(address: &str, network: Network) -> Result<(), GenericError> { } pub async fn exit(msg: &Exit) -> Result { - let network = msg.network().unwrap_or(DEFAULT_NETWORK.to_string()); - let network = Network::from_str(&network).map_err(|e| GenericError::new(e))?; + let network = msg.network().unwrap_or_else(|| DEFAULT_NETWORK.to_string()); + let network = Network::from_str(&network).map_err(GenericError::new)?; let wallet = get_wallet(&msg.sender(), network).await?; let token = get_network_token(network, None); @@ -117,15 +117,15 @@ pub async fn exit(msg: &Exit) -> Result { Some(false) => Err(GenericError::new( tx_info .fail_reason - .unwrap_or("Unknown failure reason".to_string()), + .unwrap_or_else(|| "Unknown failure reason".to_string()), )), None => Err(GenericError::new("Transaction time-outed")), } } pub async fn enter(msg: Enter) -> Result { - let network = msg.network.unwrap_or(DEFAULT_NETWORK.to_string()); - let network = Network::from_str(&network).map_err(|e| GenericError::new(e))?; + let network = msg.network.unwrap_or_else(|| DEFAULT_NETWORK.to_string()); + let network = Network::from_str(&network).map_err(GenericError::new)?; let wallet = get_wallet(&msg.address, network).await?; let tx_hash = deposit(wallet, network, msg.amount).await?; @@ -135,7 +135,7 @@ pub async fn enter(msg: Enter) -> Result { pub async fn get_tx_fee(address: &str, network: Network) -> Result { let token = get_network_token(network, None); - let wallet = get_wallet(&address, network).await?; + let wallet = get_wallet(address, network).await?; let tx_fee = wallet .provider .get_tx_fee(TxFeeTypes::Transfer, wallet.address(), token.as_str()) @@ -278,18 +278,15 @@ pub async fn verify_tx(tx_hash: &str, network: Network) -> Result RpcProvider { - let zk_network = get_zk_network(network); - let provider: RpcProvider = - RpcProvider::from_addr_and_network(get_rpc_addr(network), zk_network); - provider.clone() + RpcProvider::from_addr_and_network(get_rpc_addr(network), get_zk_network(network)) } fn get_rpc_addr(network: Network) -> String { match network { Network::Mainnet => env::var("ZKSYNC_MAINNET_RPC_ADDRESS") - .unwrap_or("https://api.zksync.golem.network/jsrpc".to_string()), + .unwrap_or_else(|_| "https://api.zksync.golem.network/jsrpc".to_string()), Network::Rinkeby => env::var("ZKSYNC_RINKEBY_RPC_ADDRESS") - .unwrap_or("https://rinkeby-api.zksync.golem.network/jsrpc".to_string()), + .unwrap_or_else(|_| "https://rinkeby-api.zksync.golem.network/jsrpc".to_string()), Network::Goerli => panic!("Goerli not supported on zksync"), Network::Polygon => panic!("Polygon not supported on zksync"), Network::Mumbai => panic!("Mumbai not supported on zksync"), @@ -298,11 +295,10 @@ fn get_rpc_addr(network: Network) -> String { fn get_ethereum_node_addr_from_env(network: Network) -> String { match network { - Network::Mainnet => { - env::var("MAINNET_GETH_ADDR").unwrap_or("https://geth.golem.network:55555".to_string()) - } + Network::Mainnet => env::var("MAINNET_GETH_ADDR") + .unwrap_or_else(|_| "https://geth.golem.network:55555".to_string()), Network::Rinkeby => env::var("RINKEBY_GETH_ADDR") - .unwrap_or("http://geth.testnet.golem.network:55555".to_string()), + .unwrap_or_else(|_| "http://geth.testnet.golem.network:55555".to_string()), Network::Goerli => panic!("Goerli not supported on zksync"), Network::Polygon => panic!("Polygon mainnet not supported on zksync"), Network::Mumbai => panic!("Polygon mumbai not supported on zksync"), @@ -310,7 +306,8 @@ fn get_ethereum_node_addr_from_env(network: Network) -> String { } fn get_ethereum_confirmation_timeout() -> std::time::Duration { - let value = std::env::var("ZKSYNC_ETH_CONFIRMATION_TIMEOUT_SECONDS").unwrap_or("60".to_owned()); + let value = std::env::var("ZKSYNC_ETH_CONFIRMATION_TIMEOUT_SECONDS") + .unwrap_or_else(|_| "60".to_owned()); std::time::Duration::from_secs(value.parse::().unwrap()) } @@ -350,8 +347,8 @@ async fn unlock_wallet( { log::info!("Unlocking wallet... address = {}", wallet.signer.address); let token = get_network_token(network, None); - let balance = get_balance(&wallet, &token).await?; - let unlock_fee = get_unlock_fee(&wallet, &token).await?; + let balance = get_balance(wallet, &token).await?; + let unlock_fee = get_unlock_fee(wallet, &token).await?; if unlock_fee > balance { return Err(GenericError::new("Not enough balance to unlock account")); } @@ -379,7 +376,7 @@ async fn unlock_wallet( log::debug!("tx_info = {:?}", tx_info); match tx_info.success { Some(true) => log::info!("Wallet successfully unlocked. address = {}", wallet.signer.address), - Some(false) => return Err(GenericError::new(format!("Failed to unlock wallet. reason={}", tx_info.fail_reason.unwrap_or("Unknown reason".to_string())))), + Some(false) => return Err(GenericError::new(format!("Failed to unlock wallet. reason={}", tx_info.fail_reason.unwrap_or_else(|| "Unknown reason".to_string())))), None => return Err(GenericError::new(format!("Unknown result from zksync unlock, please check your wallet on zkscan and try again. {:?}", tx_info))), } } @@ -522,7 +519,7 @@ pub async fn deposit( let mut ethereum = wallet .ethereum(get_ethereum_node_addr_from_env(network)) .await - .map_err(|err| GenericError::new(err))?; + .map_err(GenericError::new)?; ethereum.set_confirmation_timeout(get_ethereum_confirmation_timeout()); if !ethereum @@ -533,22 +530,19 @@ pub async fn deposit( let tx = ethereum .limited_approve_erc20_token_deposits(token.as_str(), amount) .await - .map_err(|err| GenericError::new(err))?; + .map_err(GenericError::new)?; info!( "Approve erc20 token for ZkSync deposit. Tx: https://rinkeby.etherscan.io/tx/{:#x}", tx ); - ethereum - .wait_for_tx(tx) - .await - .map_err(|err| GenericError::new(err))?; + ethereum.wait_for_tx(tx).await.map_err(GenericError::new)?; } let deposit_tx_hash = ethereum .deposit(token.as_str(), amount, address) .await - .map_err(|err| GenericError::new(err))?; + .map_err(GenericError::new)?; info!( "Check out deposit transaction at https://rinkeby.etherscan.io/tx/{:#x}", deposit_tx_hash diff --git a/core/payment/Cargo.toml b/core/payment/Cargo.toml index cd0d2ec349..20f3857c83 100644 --- a/core/payment/Cargo.toml +++ b/core/payment/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ya-payment" -version = "0.2.0" +version = "0.3.0" authors = ["Golem Factory "] edition = "2018" @@ -8,16 +8,16 @@ edition = "2018" default = [] [dependencies] -ya-agreement-utils = { version = "^0.3" } -ya-client-model = { version = "0.4", features = ["with-diesel"] } -ya-core-model = { version = "^0.6", features = [ "activity", "driver", "identity", "market", "payment" ] } -ya-net = "0.2" -ya-metrics = "0.1.0" -ya-persistence = "0.2" +ya-agreement-utils = { version = "0.4" } +ya-client-model = { version = "0.5", features = ["with-diesel"] } +ya-core-model = { version = "^0.8", features = [ "activity", "driver", "identity", "market", "payment" ] } +ya-net = "0.3" +ya-metrics = "0.2" +ya-persistence = "0.3" ya-service-api = "0.1" -ya-service-api-interfaces = "0.1" -ya-service-api-web = "0.1" -ya-service-bus = "0.4" +ya-service-api-interfaces = "0.2" +ya-service-api-web = "0.2" +ya-service-bus = "0.6" actix-web = "4" anyhow = "1.0" @@ -46,12 +46,12 @@ uuid = { version = "0.8", features = ["v4"] } humantime="2.0.1" [dev-dependencies] -ya-client = "0.6" -ya-dummy-driver = "0.2" -ya-erc20-driver = "0.3" -ya-zksync-driver = "0.2" -ya-net = { version = "0.2", features = ["service"] } -ya-sb-router = "0.4" +ya-client = "0.7" +ya-dummy-driver = "0.3" +ya-erc20-driver = "0.4" +ya-zksync-driver = "0.3" +ya-net = { version = "0.3", features = ["service"] } +ya-sb-router = "0.6" actix-rt = "2.7" rand = "0.8" diff --git a/core/payment/examples/account_balance.rs b/core/payment/examples/account_balance.rs index 6e1aac43e8..29135f92aa 100644 --- a/core/payment/examples/account_balance.rs +++ b/core/payment/examples/account_balance.rs @@ -3,7 +3,7 @@ use ya_service_bus::typed as bus; #[actix_rt::main] async fn main() -> anyhow::Result<()> { - let log_level = std::env::var("RUST_LOG").unwrap_or("info".to_owned()); + let log_level = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_owned()); std::env::set_var("RUST_LOG", log_level); env_logger::init(); diff --git a/core/payment/examples/cancel_invoice.rs b/core/payment/examples/cancel_invoice.rs index 82be48c67e..468814f39e 100644 --- a/core/payment/examples/cancel_invoice.rs +++ b/core/payment/examples/cancel_invoice.rs @@ -53,7 +53,7 @@ async fn assert_requested_amount( #[actix_rt::main] async fn main() -> anyhow::Result<()> { - let log_level = std::env::var("RUST_LOG").unwrap_or("info".to_owned()); + let log_level = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_owned()); std::env::set_var("RUST_LOG", log_level); env_logger::init(); diff --git a/core/payment/examples/debit_note_flow.rs b/core/payment/examples/debit_note_flow.rs index 1e24d5acd0..b19cd1cc79 100644 --- a/core/payment/examples/debit_note_flow.rs +++ b/core/payment/examples/debit_note_flow.rs @@ -16,7 +16,8 @@ struct Args { #[actix_rt::main] async fn main() -> anyhow::Result<()> { - let log_level = std::env::var("RUST_LOG").unwrap_or("debit_note_flow=debug,info".to_owned()); + let log_level = + std::env::var("RUST_LOG").unwrap_or_else(|_| "debit_note_flow=debug,info".to_owned()); std::env::set_var("RUST_LOG", log_level); env_logger::init(); diff --git a/core/payment/examples/get_accounts.rs b/core/payment/examples/get_accounts.rs index cde9aadfbf..169271130c 100644 --- a/core/payment/examples/get_accounts.rs +++ b/core/payment/examples/get_accounts.rs @@ -14,7 +14,7 @@ struct Args { #[actix_rt::main] async fn main() -> anyhow::Result<()> { - let log_level = std::env::var("RUST_LOG").unwrap_or("info".to_owned()); + let log_level = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_owned()); std::env::set_var("RUST_LOG", log_level); env_logger::init(); diff --git a/core/payment/examples/invoice_flow.rs b/core/payment/examples/invoice_flow.rs index b032905676..a2adc6ba79 100644 --- a/core/payment/examples/invoice_flow.rs +++ b/core/payment/examples/invoice_flow.rs @@ -17,7 +17,8 @@ struct Args { #[actix_rt::main] async fn main() -> anyhow::Result<()> { - let log_level = std::env::var("RUST_LOG").unwrap_or("invoice_flow=debug,info".to_owned()); + let log_level = + std::env::var("RUST_LOG").unwrap_or_else(|_| "invoice_flow=debug,info".to_owned()); std::env::set_var("RUST_LOG", log_level); env_logger::init(); @@ -127,7 +128,7 @@ async fn main() -> anyhow::Result<()> { .await?; assert_eq!(payments.len(), 1); let payment = payments.pop().unwrap(); - assert!(&payment.amount >= &invoice.amount); + assert!(payment.amount >= invoice.amount); log::info!("Payment verified correctly."); log::info!("Verifying invoice status..."); diff --git a/core/payment/examples/market_decoration.rs b/core/payment/examples/market_decoration.rs index f0a3b9d3f1..550b43fb2f 100644 --- a/core/payment/examples/market_decoration.rs +++ b/core/payment/examples/market_decoration.rs @@ -6,7 +6,7 @@ use ya_client_model::payment::NewAllocation; #[actix_rt::main] async fn main() -> anyhow::Result<()> { - let log_level = std::env::var("RUST_LOG").unwrap_or("info".to_owned()); + let log_level = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_owned()); std::env::set_var("RUST_LOG", log_level); env_logger::init(); diff --git a/core/payment/examples/payment_api.rs b/core/payment/examples/payment_api.rs index a056edc85b..1d5ce8b786 100644 --- a/core/payment/examples/payment_api.rs +++ b/core/payment/examples/payment_api.rs @@ -1,3 +1,5 @@ +#![allow(clippy::type_complexity)] + use actix_web::web::Data; use actix_web::{middleware, App, HttpServer, Scope}; use chrono::Utc; @@ -5,7 +7,7 @@ use ethsign::keyfile::Bytes; use ethsign::{KeyFile, Protected, SecretKey}; use futures::Future; use rand::Rng; -use serde_json; + use std::convert::TryInto; use std::io::Write; use std::pin::Pin; @@ -216,7 +218,7 @@ async fn main() -> anyhow::Result<()> { let provider_id = format!("0x{}", hex::encode(provider_account.public().address())); let provider_addr = args .provider_addr - .unwrap_or(provider_id.clone()) + .unwrap_or_else(|| provider_id.clone()) .to_lowercase(); let requestor_pass: Protected = args.requestor_pass.clone().into(); @@ -224,7 +226,7 @@ async fn main() -> anyhow::Result<()> { let requestor_id = format!("0x{}", hex::encode(requestor_account.public().address())); let requestor_addr = args .requestor_addr - .unwrap_or(requestor_id.clone()) + .unwrap_or_else(|| requestor_id.clone()) .to_lowercase(); log::info!( @@ -340,7 +342,7 @@ async fn main() -> anyhow::Result<()> { let requestor_id = requestor_id.parse()?; log::info!("bind remote..."); - let _ = ya_net::hybrid::start_network( + ya_net::hybrid::start_network( Arc::new(Config::from_env()?), provider_id, vec![provider_id, requestor_id], diff --git a/core/payment/examples/release_allocation.rs b/core/payment/examples/release_allocation.rs index ce0b9b95b1..59cb3c7cb0 100644 --- a/core/payment/examples/release_allocation.rs +++ b/core/payment/examples/release_allocation.rs @@ -6,7 +6,7 @@ use ya_client_model::payment::{Acceptance, NewAllocation, NewInvoice}; #[actix_rt::main] async fn main() -> anyhow::Result<()> { - let log_level = std::env::var("RUST_LOG").unwrap_or("info".to_owned()); + let log_level = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_owned()); std::env::set_var("RUST_LOG", log_level); env_logger::init(); diff --git a/core/payment/examples/validate_allocation.rs b/core/payment/examples/validate_allocation.rs index 696750206e..4875a96ad5 100644 --- a/core/payment/examples/validate_allocation.rs +++ b/core/payment/examples/validate_allocation.rs @@ -32,7 +32,7 @@ async fn get_requestor_balance_and_platform() -> anyhow::Result<(BigDecimal, Str #[actix_rt::main] async fn main() -> anyhow::Result<()> { - let log_level = std::env::var("RUST_LOG").unwrap_or("info".to_owned()); + let log_level = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_owned()); std::env::set_var("RUST_LOG", log_level); env_logger::init(); diff --git a/core/payment/src/accounts.rs b/core/payment/src/accounts.rs index 57371994fd..e3c5c0ead7 100644 --- a/core/payment/src/accounts.rs +++ b/core/payment/src/accounts.rs @@ -75,7 +75,7 @@ pub async fn save_default_account(data_dir: &Path, drivers: Vec) -> anyh let default_node_id = bus::service(identity::BUS_ID) .call(identity::Get::ByDefault) .await?? - .ok_or(anyhow::anyhow!("Default identity not found"))? + .ok_or_else(|| anyhow::anyhow!("Default identity not found"))? .node_id; let default_accounts: Vec = drivers .into_iter() diff --git a/core/payment/src/api/accounts.rs b/core/payment/src/api/accounts.rs index 56b8e85a11..67abc1cb4f 100644 --- a/core/payment/src/api/accounts.rs +++ b/core/payment/src/api/accounts.rs @@ -45,7 +45,6 @@ async fn get_requestor_accounts(db: Data, id: Identity) -> HttpRespo }; let recv_accounts: Vec = all_accounts .into_iter() - .filter(|account| account.send) .filter(|account| account.address == node_id) // TODO: Implement proper account permission system .collect(); response::ok(recv_accounts) diff --git a/core/payment/src/api/allocations.rs b/core/payment/src/api/allocations.rs index bf18015b47..c23064298a 100644 --- a/core/payment/src/api/allocations.rs +++ b/core/payment/src/api/allocations.rs @@ -1,3 +1,4 @@ +use std::convert::TryInto; use std::time::Duration; // External crates use actix_web::web::{delete, get, post, put, Data, Json, Path, Query}; @@ -18,6 +19,7 @@ use ya_service_api_web::middleware::Identity; use ya_service_bus::{typed as bus, RpcEndpoint}; // Local uses +use crate::accounts::{init_account, Account}; use crate::dao::*; use crate::error::{DbError, Error}; use crate::utils::response; @@ -47,8 +49,40 @@ async fn create_allocation( let payment_platform = allocation .payment_platform .clone() - .unwrap_or(DEFAULT_PAYMENT_PLATFORM.to_string()); - let address = allocation.address.clone().unwrap_or(node_id.to_string()); + .unwrap_or_else(|| DEFAULT_PAYMENT_PLATFORM.to_string()); + let address = allocation + .address + .clone() + .unwrap_or_else(|| node_id.to_string()); + + // If the request contains information about the payment platform, initialize the account + // by setting the `send` field to `true`, as it is implied by the intent behing allocation of funds. + if let Some(platform) = &allocation.payment_platform { + // payment_platform is of the form driver-network-token + // eg. erc20-rinkeby-tglm + let [driver, network, _token]: [&str; 3] = + match platform.split('-').collect::>().try_into() { + Ok(arr) => arr, + Err(_e) => { + return response::bad_request( + &"paymentPlatform must be of the form driver-network-token", + ) + } + }; + + let acc = Account { + driver: driver.to_owned(), + address: address.clone(), + network: Some(network.to_owned()), + token: None, + send: true, + receive: false, + }; + + if let Err(e) = init_account(acc).await { + return response::server_error(&e); + } + } let validate_msg = ValidateAllocation { platform: payment_platform.clone(), @@ -57,7 +91,7 @@ async fn create_allocation( }; match async move { Ok(bus::service(LOCAL_SERVICE).send(validate_msg).await??) }.await { Ok(true) => {} - Ok(false) => return response::bad_request(&"Insufficient funds to make allocation. Release all existing allocations to unlock the funds via `yagna payment release-allocations`"), + Ok(false) => return response::bad_request(&"Insufficient funds to make allocation. Top up your account or release all existing allocations to unlock the funds via `yagna payment release-allocations`"), Err(Error::Rpc(RpcMessageError::ValidateAllocation( ValidateAllocationError::AccountNotRegistered, ))) => return response::bad_request(&"Account not registered"), @@ -73,24 +107,23 @@ async fn create_allocation( Ok(allocation_id) => match dao.get(allocation_id, node_id).await { Ok(AllocationStatus::Active(allocation)) => { let allocation_id = allocation.allocation_id.clone(); - let allocation_timeout = allocation.timeout.clone(); release_allocation_after( db.clone(), allocation_id, - allocation_timeout, + allocation.timeout, Some(node_id), ) .await; response::created(allocation) } - Ok(AllocationStatus::NotFound) => return response::server_error(&"Database error"), - Ok(AllocationStatus::Gone) => return response::server_error(&"Database error"), - Err(DbError::Query(e)) => return response::bad_request(&e), - Err(e) => return response::server_error(&e), + Ok(AllocationStatus::NotFound) => response::server_error(&"Database error"), + Ok(AllocationStatus::Gone) => response::server_error(&"Database error"), + Err(DbError::Query(e)) => response::bad_request(&e), + Err(e) => response::server_error(&e), }, - Err(e) => return response::server_error(&e), + Err(e) => response::server_error(&e), } } diff --git a/core/payment/src/api/debit_notes.rs b/core/payment/src/api/debit_notes.rs index c2fcf6d5f1..a74f1098bb 100644 --- a/core/payment/src/api/debit_notes.rs +++ b/core/payment/src/api/debit_notes.rs @@ -107,14 +107,14 @@ async fn get_debit_note_events( .headers() .get("X-Requestor-Events") .and_then(|v| v.to_str().ok()) - .map(|v| v.split(",").map(|s| Cow::Owned(s.to_owned())).collect()) + .map(|v| v.split(',').map(|s| Cow::Owned(s.to_owned())).collect()) .unwrap_or_else(|| vec!["RECEIVED".into(), "CANCELLED".into()]); let provider_events: Vec> = req .headers() .get("X-Provider-Events") .and_then(|v| v.to_str().ok()) - .map(|v| v.split(",").map(|s| Cow::Owned(s.to_owned())).collect()) + .map(|v| v.split(',').map(|s| Cow::Owned(s.to_owned())).collect()) .unwrap_or_else(|| { vec![ "ACCEPTED".into(), @@ -132,9 +132,9 @@ async fn get_debit_note_events( let dao: DebitNoteEventDao = db.as_dao(); let getter = || async { dao.get_for_node_id( - node_id.clone(), - after_timestamp.clone(), - max_events.clone(), + node_id, + after_timestamp, + max_events, app_session_id.clone(), requestor_events.clone(), provider_events.clone(), @@ -160,7 +160,7 @@ async fn issue_debit_note( let agreement = match get_agreement_for_activity( activity_id.clone(), - ya_core_model::Role::Provider, + ya_client_model::market::Role::Provider, ) .await { @@ -399,10 +399,8 @@ async fn accept_debit_note( } Ok(Err(Error::Rpc(RpcMessageError::AcceptReject(AcceptRejectError::BadRequest( e, - ))))) => { - return response::bad_request(&e); - } - Ok(Err(e)) => return response::server_error(&e), + ))))) => response::bad_request(&e), + Ok(Err(e)) => response::server_error(&e), Err(_) => response::timeout(&"Timeout accepting Debit Note on remote Node."), } } diff --git a/core/payment/src/api/invoices.rs b/core/payment/src/api/invoices.rs index 4ef7384419..4eecee23df 100644 --- a/core/payment/src/api/invoices.rs +++ b/core/payment/src/api/invoices.rs @@ -93,14 +93,14 @@ async fn get_invoice_events( .headers() .get("X-Requestor-Events") .and_then(|v| v.to_str().ok()) - .map(|v| v.split(",").map(|s| Cow::Owned(s.to_owned())).collect()) + .map(|v| v.split(',').map(|s| Cow::Owned(s.to_owned())).collect()) .unwrap_or_else(|| vec!["RECEIVED".into(), "CANCELLED".into()]); let provider_events: Vec> = req .headers() .get("X-Provider-Events") .and_then(|v| v.to_str().ok()) - .map(|v| v.split(",").map(|s| Cow::Owned(s.to_owned())).collect()) + .map(|v| v.split(',').map(|s| Cow::Owned(s.to_owned())).collect()) .unwrap_or_else(|| { vec![ "ACCEPTED".into(), @@ -118,9 +118,9 @@ async fn get_invoice_events( let dao: InvoiceEventDao = db.as_dao(); let getter = || async { dao.get_for_node_id( - node_id.clone(), - after_timestamp.clone(), - max_events.clone(), + node_id, + after_timestamp, + max_events, app_session_id.clone(), requestor_events.clone(), provider_events.clone(), @@ -141,7 +141,12 @@ async fn issue_invoice(db: Data, body: Json, id: Identit let agreement_id = invoice.agreement_id.clone(); let activity_ids = invoice.activity_ids.clone().unwrap_or_default(); - let agreement = match get_agreement(agreement_id.clone(), ya_core_model::Role::Provider).await { + let agreement = match get_agreement( + agreement_id.clone(), + ya_client_model::market::Role::Provider, + ) + .await + { Ok(Some(agreement)) => agreement, Ok(None) => { return response::bad_request(&format!("Agreement not found: {}", agreement_id)) @@ -150,7 +155,7 @@ async fn issue_invoice(db: Data, body: Json, id: Identit }; for activity_id in activity_ids.iter() { - match get_agreement_id(activity_id.clone(), ya_core_model::Role::Provider).await { + match get_agreement_id(activity_id.clone(), ya_client_model::market::Role::Provider).await { Ok(Some(id)) if id != agreement_id => { return response::bad_request(&format!( "Activity {} belongs to agreement {} not {}", @@ -460,8 +465,8 @@ async fn accept_invoice( } Ok(Err(Error::Rpc(RpcMessageError::AcceptReject(AcceptRejectError::BadRequest( e, - ))))) => return response::bad_request(&e), - Ok(Err(e)) => return response::server_error(&e), + ))))) => response::bad_request(&e), + Ok(Err(e)) => response::server_error(&e), Err(_) => response::timeout(&"Timeout accepting Invoice on remote Node."), } } diff --git a/core/payment/src/api/payments.rs b/core/payment/src/api/payments.rs index d8c58e967e..c4562a94b7 100644 --- a/core/payment/src/api/payments.rs +++ b/core/payment/src/api/payments.rs @@ -54,9 +54,9 @@ async fn get_payments( let dao: PaymentDao = db.as_dao(); let getter = || async { dao.get_for_node_id( - node_id.clone(), - after_timestamp.clone(), - max_events.clone(), + node_id, + after_timestamp, + max_events, app_session_id.clone(), network.clone(), driver.clone(), diff --git a/core/payment/src/cli.rs b/core/payment/src/cli.rs index 8c64f29286..a5c5b95469 100644 --- a/core/payment/src/cli.rs +++ b/core/payment/src/cli.rs @@ -118,15 +118,23 @@ pub enum InvoiceCommand { impl PaymentCli { pub async fn run_command(self, ctx: &CliCtx) -> anyhow::Result { match self { - PaymentCli::Fund { account } => CommandOutput::object( - wallet::fund( - resolve_address(account.address()).await?, - account.driver(), - Some(account.network()), - None, + PaymentCli::Fund { account } => { + let address = resolve_address(account.address()).await?; + + init_account(Account { + driver: account.driver(), + address: address.clone(), + network: Some(account.network()), + token: None, // Use default -- we don't yet support other tokens than GLM + send: true, + receive: false, + }) + .await?; + + CommandOutput::object( + wallet::fund(address, account.driver(), Some(account.network()), None).await?, ) - .await?, - ), + } PaymentCli::Init { account, sender, @@ -147,7 +155,7 @@ impl PaymentCli { let address = resolve_address(account.address()).await?; let timestamp = last .map(|d| Utc::now() - chrono::Duration::seconds(d.as_secs() as i64)) - .unwrap_or(DateTime::from(UNIX_EPOCH)) + .unwrap_or_else(|| DateTime::from(UNIX_EPOCH)) .timestamp(); let status = bus::service(pay::BUS_ID) .call(pay::GetStatus { @@ -164,7 +172,7 @@ impl PaymentCli { let gas_info = match status.gas { Some(details) => format!("{} {}", details.balance, details.currency_short_name), - None => format!("N/A"), + None => "N/A".to_string(), }; Ok(ResponseTable { diff --git a/core/payment/src/dao/activity.rs b/core/payment/src/dao/activity.rs index e16f4a15bd..1a37bfbd3a 100644 --- a/core/payment/src/dao/activity.rs +++ b/core/payment/src/dao/activity.rs @@ -62,7 +62,7 @@ pub fn increase_amount_scheduled( amount: &BigDecimal, conn: &ConnType, ) -> DbResult<()> { - assert!(amount > &BigDecimal::zero().into()); // TODO: Remove when payment service is production-ready. + assert!(amount > &BigDecimal::zero()); // TODO: Remove when payment service is production-ready. let activity: WriteObj = dsl::pay_activity .find((activity_id, owner_id)) .first(conn)?; @@ -107,7 +107,7 @@ pub fn increase_amount_paid( for debit_note_id in debit_note_ids { debit_note_event::create::<()>( debit_note_id, - owner_id.clone(), + *owner_id, DebitNoteEventType::DebitNoteSettledEvent, None, conn, @@ -185,7 +185,7 @@ impl<'a> ActivityDao<'a> { .select(dsl::id) .first(conn) .optional()?; - if let Some(_) = existing { + if existing.is_some() { return Ok(()); } diff --git a/core/payment/src/dao/agreement.rs b/core/payment/src/dao/agreement.rs index 4d25339193..12c584b68b 100644 --- a/core/payment/src/dao/agreement.rs +++ b/core/payment/src/dao/agreement.rs @@ -95,7 +95,7 @@ pub fn increase_amount_scheduled( amount: &BigDecimal, conn: &ConnType, ) -> DbResult<()> { - assert!(amount > &BigDecimal::zero().into()); // TODO: Remove when payment service is production-ready. + assert!(amount > &BigDecimal::zero()); // TODO: Remove when payment service is production-ready. let agreement: ReadObj = dsl::pay_agreement .find((agreement_id, owner_id)) .first(conn)?; @@ -155,7 +155,7 @@ pub fn increase_amount_paid( invoice::update_status(&invoice_id, owner_id, &DocumentStatus::Settled, conn)?; invoice_event::create::<()>( invoice_id, - owner_id.clone(), + *owner_id, InvoiceEventType::InvoiceSettledEvent, None, conn, @@ -199,7 +199,7 @@ impl<'a> AgreementDao<'a> { .select(dsl::id) .first(conn) .optional()?; - if let Some(_) = existing { + if existing.is_some() { return Ok(()); } diff --git a/core/payment/src/dao/allocation.rs b/core/payment/src/dao/allocation.rs index 33132d6961..693f332c82 100644 --- a/core/payment/src/dao/allocation.rs +++ b/core/payment/src/dao/allocation.rs @@ -138,10 +138,10 @@ impl<'c> AllocationDao<'c> { query = query.filter(dsl::timestamp.gt(after_timestamp)) } if let Some(payment_platform) = payment_platform { - query = query.filter(dsl::timestamp.gt(payment_platform)) + query = query.filter(dsl::payment_platform.eq(payment_platform)) } if let Some(address) = address { - query = query.filter(dsl::timestamp.gt(address)) + query = query.filter(dsl::address.eq(address)) } if let Some(max_items) = max_items { query = query.limit(max_items.into()) @@ -185,13 +185,13 @@ impl<'c> AllocationDao<'c> { .set(dsl::released.eq(true)) .execute(conn)?; - return match num_released { + match num_released { 1 => Ok(AllocationReleaseStatus::Released), _ => Err(DbError::Query(format!( "Update error occurred when releasing allocation {}", allocation_id ))), - }; + } }) .await } @@ -218,6 +218,7 @@ impl<'c> AllocationDao<'c> { } } +#[allow(clippy::large_enum_variant)] pub enum AllocationStatus { Active(Allocation), Gone, diff --git a/core/payment/src/dao/debit_note.rs b/core/payment/src/dao/debit_note.rs index b6eaef32f1..cb6b1cf0c8 100644 --- a/core/payment/src/dao/debit_note.rs +++ b/core/payment/src/dao/debit_note.rs @@ -128,7 +128,7 @@ impl<'c> DebitNoteDao<'c> { .optional()?; let debit_note = WriteObj::issued(debit_note, previous_debit_note_id, issuer_id); let debit_note_id = debit_note.id.clone(); - let owner_id = debit_note.owner_id.clone(); + let owner_id = debit_note.owner_id; activity::set_amount_due( &debit_note.activity_id, &debit_note.owner_id, @@ -160,7 +160,7 @@ impl<'c> DebitNoteDao<'c> { .optional()?; let debit_note = WriteObj::received(debit_note, previous_debit_note_id); let debit_note_id = debit_note.id.clone(); - let owner_id = debit_note.owner_id.clone(); + let owner_id = debit_note.owner_id; activity::set_amount_due( &debit_note.activity_id, &debit_note.owner_id, diff --git a/core/payment/src/dao/debit_note_event.rs b/core/payment/src/dao/debit_note_event.rs index b8b37eab5c..c96f754ad7 100644 --- a/core/payment/src/dao/debit_note_event.rs +++ b/core/payment/src/dao/debit_note_event.rs @@ -13,7 +13,7 @@ use ya_client_model::NodeId; use ya_persistence::executor::{ do_with_transaction, readonly_transaction, AsDao, ConnType, PoolType, }; -use ya_persistence::types::Role; +use ya_persistence::types::{AdaptTimestamp, Role}; pub fn create( debit_note_id: String, @@ -68,7 +68,7 @@ impl<'c> DebitNoteEventDao<'c> { .order_by(read_dsl::timestamp.asc()) .into_boxed(); if let Some(timestamp) = after_timestamp { - query = query.filter(read_dsl::timestamp.gt(timestamp)); + query = query.filter(read_dsl::timestamp.gt(timestamp.adapt())); } if let Some(app_session_id) = app_session_id { query = query.filter(read_dsl::app_session_id.eq(app_session_id)); diff --git a/core/payment/src/dao/invoice.rs b/core/payment/src/dao/invoice.rs index 27b496806d..e06f6754c5 100644 --- a/core/payment/src/dao/invoice.rs +++ b/core/payment/src/dao/invoice.rs @@ -74,7 +74,7 @@ pub fn update_status( impl<'c> InvoiceDao<'c> { async fn insert(&self, invoice: WriteObj, activity_ids: Vec) -> DbResult<()> { let invoice_id = invoice.id.clone(); - let owner_id = invoice.owner_id.clone(); + let owner_id = invoice.owner_id; let role = invoice.role.clone(); do_with_transaction(self.pool, move |conn| { if let Some(read_invoice) = query!() @@ -85,9 +85,10 @@ impl<'c> InvoiceDao<'c> { { return match equivalent(&read_invoice, &invoice) { true => Ok(()), - false => Err(DbError::Integrity(format!( + false => Err(DbError::Integrity( "Invoice with the same id and different content already exists." - ))), + .to_string(), + )), }; }; @@ -100,7 +101,7 @@ impl<'c> InvoiceDao<'c> { // Diesel cannot do batch insert into SQLite database activity_ids.into_iter().try_for_each(|activity_id| { let invoice_id = invoice_id.clone(); - let owner_id = owner_id.clone(); + let owner_id = owner_id; diesel::insert_into(activity_dsl::pay_invoice_x_activity) .values(InvoiceXActivity { invoice_id, @@ -125,8 +126,8 @@ impl<'c> InvoiceDao<'c> { } pub async fn create_new(&self, invoice: NewInvoice, issuer_id: NodeId) -> DbResult { - let activity_ids = invoice.activity_ids.clone().unwrap_or(vec![]); - let invoice = WriteObj::new_issued(invoice, issuer_id.clone()); + let activity_ids = invoice.activity_ids.clone().unwrap_or_default(); + let invoice = WriteObj::new_issued(invoice, issuer_id); let invoice_id = invoice.id.clone(); self.insert(invoice, activity_ids).await?; Ok(invoice_id) @@ -259,13 +260,7 @@ impl<'c> InvoiceDao<'c> { agreement::set_amount_accepted(&agreement_id, &owner_id, &amount, conn)?; for event in events { - invoice_event::create::<()>( - invoice_id.clone(), - owner_id.clone(), - event, - None, - conn, - )?; + invoice_event::create::<()>(invoice_id.clone(), owner_id, event, None, conn)?; } Ok(()) @@ -343,7 +338,7 @@ fn join_invoices_with_activities( invoices .into_iter() .map(|invoice| { - let activity_ids = activities_map.remove(&invoice.id).unwrap_or(vec![]); + let activity_ids = activities_map.remove(&invoice.id).unwrap_or_default(); invoice.into_api_model(activity_ids) }) .collect() diff --git a/core/payment/src/dao/invoice_event.rs b/core/payment/src/dao/invoice_event.rs index 5e67d36620..46e7b302ca 100644 --- a/core/payment/src/dao/invoice_event.rs +++ b/core/payment/src/dao/invoice_event.rs @@ -13,7 +13,7 @@ use ya_client_model::NodeId; use ya_persistence::executor::{ do_with_transaction, readonly_transaction, AsDao, ConnType, PoolType, }; -use ya_persistence::types::Role; +use ya_persistence::types::{AdaptTimestamp, Role}; pub fn create( invoice_id: String, @@ -68,7 +68,7 @@ impl<'c> InvoiceEventDao<'c> { .order_by(read_dsl::timestamp.asc()) .into_boxed(); if let Some(timestamp) = after_timestamp { - query = query.filter(read_dsl::timestamp.gt(timestamp)); + query = query.filter(read_dsl::timestamp.gt(timestamp.adapt())); } if let Some(app_session_id) = app_session_id { query = query.filter(read_dsl::app_session_id.eq(app_session_id)); diff --git a/core/payment/src/dao/payment.rs b/core/payment/src/dao/payment.rs index 9a012e304e..f03299dd15 100644 --- a/core/payment/src/dao/payment.rs +++ b/core/payment/src/dao/payment.rs @@ -28,7 +28,7 @@ pub struct PaymentDao<'c> { fn insert_activity_payments( activity_payments: Vec, - payment_id: &String, + payment_id: &str, owner_id: &NodeId, conn: &ConnType, ) -> DbResult<()> { @@ -37,13 +37,13 @@ fn insert_activity_payments( let amount = activity_payment.amount.into(); let allocation_id = activity_payment.allocation_id; - activity::increase_amount_paid(&activity_payment.activity_id, &owner_id, &amount, conn)?; + activity::increase_amount_paid(&activity_payment.activity_id, owner_id, &amount, conn)?; diesel::insert_into(activity_pay_dsl::pay_activity_payment) .values(DbActivityPayment { - payment_id: payment_id.clone(), + payment_id: payment_id.to_string(), activity_id: activity_payment.activity_id, - owner_id: owner_id.clone(), + owner_id: *owner_id, amount, allocation_id, }) @@ -56,7 +56,7 @@ fn insert_activity_payments( fn insert_agreement_payments( agreement_payments: Vec, - payment_id: &String, + payment_id: &str, owner_id: &NodeId, conn: &ConnType, ) -> DbResult<()> { @@ -65,13 +65,13 @@ fn insert_agreement_payments( let amount = agreement_payment.amount.into(); let allocation_id = agreement_payment.allocation_id; - agreement::increase_amount_paid(&agreement_payment.agreement_id, &owner_id, &amount, conn)?; + agreement::increase_amount_paid(&agreement_payment.agreement_id, owner_id, &amount, conn)?; diesel::insert_into(agreement_pay_dsl::pay_agreement_payment) .values(DbAgreementPayment { - payment_id: payment_id.clone(), + payment_id: payment_id.to_string(), agreement_id: agreement_payment.agreement_id, - owner_id: owner_id.clone(), + owner_id: *owner_id, amount, allocation_id, }) @@ -96,7 +96,7 @@ impl<'c> PaymentDao<'c> { agreement_payments: Vec, ) -> DbResult<()> { let payment_id = payment.id.clone(); - let owner_id = payment.owner_id.clone(); + let owner_id = payment.owner_id; let amount = payment.amount.clone(); do_with_transaction(self.pool, move |conn| { @@ -106,14 +106,15 @@ impl<'c> PaymentDao<'c> { .execute(conn)?; log::trace!("Payment inserted."); - insert_activity_payments(activity_payments, &payment_id, &owner_id, &conn)?; - insert_agreement_payments(agreement_payments, &payment_id, &owner_id, &conn)?; + insert_activity_payments(activity_payments, &payment_id, &owner_id, conn)?; + insert_agreement_payments(agreement_payments, &payment_id, &owner_id, conn)?; Ok(()) }) .await } + #[allow(clippy::too_many_arguments)] pub async fn create_new( &self, payer_id: NodeId, @@ -301,8 +302,12 @@ fn join_activity_and_agreement_payments( payments .into_iter() .map(|payment| { - let activity_payments = activity_payments_map.remove(&payment.id).unwrap_or(vec![]); - let agreement_payments = agreement_payments_map.remove(&payment.id).unwrap_or(vec![]); + let activity_payments = activity_payments_map + .remove(&payment.id) + .unwrap_or_default(); + let agreement_payments = agreement_payments_map + .remove(&payment.id) + .unwrap_or_default(); payment.into_api_model(activity_payments, agreement_payments) }) .collect() diff --git a/core/payment/src/models/agreement.rs b/core/payment/src/models/agreement.rs index 86c79c3130..ace1faa2d2 100644 --- a/core/payment/src/models/agreement.rs +++ b/core/payment/src/models/agreement.rs @@ -26,11 +26,11 @@ pub struct WriteObj { impl WriteObj { pub fn new(agreement: Agreement, role: Role) -> Self { - let provider_id = agreement.provider_id().clone(); - let requestor_id = agreement.requestor_id().clone(); + let provider_id = *agreement.provider_id(); + let requestor_id = *agreement.requestor_id(); let (owner_id, peer_id) = match &role { - Role::Provider => (provider_id.clone(), requestor_id.clone()), - Role::Requestor => (requestor_id.clone(), provider_id.clone()), + Role::Provider => (provider_id, requestor_id), + Role::Requestor => (requestor_id, provider_id), }; let demand_properties = expand(agreement.demand.properties); @@ -45,12 +45,12 @@ impl WriteObj { .pointer(format!("/golem/com/payment/platform/{}/address", payment_platform).as_str()) .as_typed(Value::as_str) .map(ToOwned::to_owned) - .unwrap_or(provider_id.to_string().to_lowercase()); + .unwrap_or_else(|_| provider_id.to_string().to_lowercase()); let payer_addr = demand_properties .pointer(format!("/golem/com/payment/platform/{}/address", payment_platform).as_str()) .as_typed(Value::as_str) .map(ToOwned::to_owned) - .unwrap_or(requestor_id.to_string().to_lowercase()); + .unwrap_or_else(|_| requestor_id.to_string().to_lowercase()); Self { id: agreement.agreement_id, diff --git a/core/payment/src/models/debit_note_event.rs b/core/payment/src/models/debit_note_event.rs index f7d4eb1875..7e03dd38bd 100644 --- a/core/payment/src/models/debit_note_event.rs +++ b/core/payment/src/models/debit_note_event.rs @@ -6,7 +6,7 @@ use serde::Serialize; use std::convert::TryFrom; use ya_client_model::payment::{DebitNoteEvent, DebitNoteEventType}; use ya_client_model::NodeId; -use ya_persistence::types::Role; +use ya_persistence::types::{AdaptTimestamp, Role, TimestampAdapter}; #[derive(Debug, Identifiable, Insertable)] #[table_name = "pay_debit_note_event"] @@ -16,6 +16,7 @@ pub struct WriteObj { pub owner_id: NodeId, pub event_type: String, pub details: Option, + pub timestamp: TimestampAdapter, } impl WriteObj { @@ -34,6 +35,7 @@ impl WriteObj { owner_id, event_type: event_type.to_string(), details, + timestamp: Utc::now().adapt(), }) } } diff --git a/core/payment/src/models/invoice_event.rs b/core/payment/src/models/invoice_event.rs index 83a12ca2d8..441dedc4d6 100644 --- a/core/payment/src/models/invoice_event.rs +++ b/core/payment/src/models/invoice_event.rs @@ -6,7 +6,7 @@ use serde::Serialize; use std::convert::TryFrom; use ya_client_model::payment::{InvoiceEvent, InvoiceEventType}; use ya_client_model::NodeId; -use ya_persistence::types::Role; +use ya_persistence::types::{AdaptTimestamp, Role, TimestampAdapter}; #[derive(Debug, Identifiable, Insertable)] #[table_name = "pay_invoice_event"] @@ -16,6 +16,7 @@ pub struct WriteObj { pub owner_id: NodeId, pub event_type: String, pub details: Option, + pub timestamp: TimestampAdapter, } impl WriteObj { @@ -29,11 +30,13 @@ impl WriteObj { Some(details) => Some(json_to_string(&details)?), None => None, }; + Ok(Self { invoice_id, owner_id, event_type: event_type.to_string(), details, + timestamp: Utc::now().adapt(), }) } } diff --git a/core/payment/src/models/payment.rs b/core/payment/src/models/payment.rs index 572175b8c3..50982a84d7 100644 --- a/core/payment/src/models/payment.rs +++ b/core/payment/src/models/payment.rs @@ -124,12 +124,12 @@ pub struct ActivityPayment { pub allocation_id: Option, } -impl Into for ActivityPayment { - fn into(self) -> api_model::ActivityPayment { - api_model::ActivityPayment { - activity_id: self.activity_id, - amount: self.amount.0, - allocation_id: self.allocation_id, +impl From for api_model::ActivityPayment { + fn from(ap: ActivityPayment) -> Self { + Self { + activity_id: ap.activity_id, + amount: ap.amount.0, + allocation_id: ap.allocation_id, } } } @@ -145,12 +145,12 @@ pub struct AgreementPayment { pub allocation_id: Option, } -impl Into for AgreementPayment { - fn into(self) -> api_model::AgreementPayment { - api_model::AgreementPayment { - agreement_id: self.agreement_id, - amount: self.amount.0, - allocation_id: self.allocation_id, +impl From for api_model::AgreementPayment { + fn from(ap: AgreementPayment) -> Self { + Self { + agreement_id: ap.agreement_id, + amount: ap.amount.0, + allocation_id: ap.allocation_id, } } } diff --git a/core/payment/src/processor.rs b/core/payment/src/processor.rs index 3536a82f74..3a1927e74b 100644 --- a/core/payment/src/processor.rs +++ b/core/payment/src/processor.rs @@ -41,18 +41,20 @@ async fn validate_orders( amount: &BigDecimal, ) -> Result<(), OrderValidationError> { if orders.is_empty() { - return Err(OrderValidationError::new("orders not found in the database").into()); + return Err(OrderValidationError::new( + "orders not found in the database", + )); } let mut total_amount = BigDecimal::zero(); for order in orders.iter() { - if &order.payment_platform != platform { + if order.payment_platform != platform { return OrderValidationError::platform(order, platform); } - if &order.payer_addr != &payer_addr { + if order.payer_addr != payer_addr { return OrderValidationError::payer_addr(order, payer_addr); } - if &order.payee_addr != &payee_addr { + if order.payee_addr != payee_addr { return OrderValidationError::payee_addr(order, payee_addr); } @@ -219,11 +221,11 @@ impl DriverRegistry { network: Option, ) -> Result<(String, Network), RegisterAccountError> { let driver_details = self.get_driver(&driver)?; - let network_name = network.unwrap_or(driver_details.default_network.to_owned()); + let network_name = network.unwrap_or_else(|| driver_details.default_network.to_owned()); match driver_details.networks.get(&network_name) { None => Err(RegisterAccountError::UnsupportedNetwork( network_name, - driver.into(), + driver, )), Some(network_details) => Ok((network_name, network_details.clone())), } @@ -236,7 +238,7 @@ impl DriverRegistry { token: Option, ) -> Result { let (network_name, network_details) = self.get_network(driver.clone(), network)?; - let token = token.unwrap_or(network_details.default_token.to_owned()); + let token = token.unwrap_or_else(|| network_details.default_token.to_owned()); match network_details.tokens.get(&token) { None => Err(RegisterAccountError::UnsupportedToken( token, @@ -383,7 +385,7 @@ impl PaymentProcessor { amount, allocation_id: Some(order.allocation_id.clone()), }), - _ => return NotifyPaymentError::invalid_order(&order), + _ => return NotifyPaymentError::invalid_order(order), } } @@ -455,7 +457,7 @@ impl PaymentProcessor { msg.payer_addr.clone(), msg.payee_addr.clone(), msg.payment_platform.clone(), - msg.due_date.clone(), + msg.due_date, )) .await??; @@ -496,14 +498,14 @@ impl PaymentProcessor { .await??; // Verify if amount declared in message matches actual amount transferred on blockchain - if &details.amount < &payment.amount { + if details.amount < payment.amount { return VerifyPaymentError::amount(&details.amount, &payment.amount); } // Verify if payment shares for agreements and activities sum up to the total amount let agreement_sum = payment.agreement_payments.iter().map(|p| &p.amount).sum(); let activity_sum = payment.activity_payments.iter().map(|p| &p.amount).sum(); - if &details.amount < &(&agreement_sum + &activity_sum) { + if details.amount < (&agreement_sum + &activity_sum) { return VerifyPaymentError::shares(&details.amount, &agreement_sum, &activity_sum); } @@ -533,7 +535,7 @@ impl PaymentProcessor { Some(agreement) if &agreement.payer_addr != payer_addr => { return VerifyPaymentError::agreement_payer(&agreement, payer_addr); } - Some(agreement) if &agreement.payment_platform != &payment.payment_platform => { + Some(agreement) if agreement.payment_platform != payment.payment_platform => { return VerifyPaymentError::agreement_platform( &agreement, &payment.payment_platform, diff --git a/core/payment/src/service.rs b/core/payment/src/service.rs index 429dd2c7cd..7309c22144 100644 --- a/core/payment/src/service.rs +++ b/core/payment/src/service.rs @@ -4,7 +4,6 @@ use futures::prelude::*; use metrics::counter; use std::collections::HashMap; use std::sync::Arc; -use ya_core_model as core; use ya_persistence::executor::DbExecutor; use ya_service_bus::typed::ServiceBinder; @@ -174,7 +173,7 @@ mod local { .get_network(driver.clone(), network) .await .map_err(GenericError::new)?; - let token = token.unwrap_or(network_details.default_token.clone()); + let token = token.unwrap_or_else(|| network_details.default_token.clone()); let after_timestamp = NaiveDateTime::from_timestamp(after_timestamp, 0); let platform = match network_details.tokens.get(&token) { Some(platform) => platform.clone(), @@ -188,33 +187,21 @@ mod local { let incoming_fut = async { db.as_dao::() - .incoming_transaction_summary( - platform.clone(), - address.clone(), - after_timestamp.clone(), - ) + .incoming_transaction_summary(platform.clone(), address.clone(), after_timestamp) .await } .map_err(GenericError::new); let outgoing_fut = async { db.as_dao::() - .outgoing_transaction_summary( - platform.clone(), - address.clone(), - after_timestamp.clone(), - ) + .outgoing_transaction_summary(platform.clone(), address.clone(), after_timestamp) .await } .map_err(GenericError::new); let reserved_fut = async { db.as_dao::() - .total_remaining_allocation( - platform.clone(), - address.clone(), - after_timestamp.clone(), - ) + .total_remaining_allocation(platform.clone(), address.clone(), after_timestamp) .await } .map_err(GenericError::new); @@ -329,7 +316,8 @@ mod local { _caller: String, msg: ReleaseAllocations, ) -> Result<(), GenericError> { - Ok(processor.lock().await.release_allocations(true).await) + processor.lock().await.release_allocations(true).await; + Ok(()) } async fn get_drivers( @@ -350,7 +338,8 @@ mod local { // It's crucial to drop the lock on processor (hence assigning the future to a variable). // Otherwise, we won't be able to handle calls to `notify_payment` sent by drivers during shutdown. let shutdown_future = processor.lock().await.shut_down(msg.timeout); - Ok(shutdown_future.await) + shutdown_future.await; + Ok(()) } } @@ -402,7 +391,12 @@ mod public { ); counter!("payment.debit_notes.requestor.received.call", 1); - let agreement = match get_agreement(agreement_id.clone(), core::Role::Requestor).await { + let agreement = match get_agreement( + agreement_id.clone(), + ya_client_model::market::Role::Requestor, + ) + .await + { Err(e) => { return Err(SendError::ServiceError(e.to_string())); } @@ -421,7 +415,7 @@ mod public { return Err(SendError::BadRequest("Invalid sender node ID".to_owned())); } - let node_id = agreement.requestor_id().clone(); + let node_id = *agreement.requestor_id(); match async move { db.as_dao::() .create_if_not_exists(agreement, node_id, Role::Requestor) @@ -444,8 +438,8 @@ mod public { .await { Ok(_) => Ok(Ack {}), - Err(DbError::Query(e)) => return Err(SendError::BadRequest(e.to_string())), - Err(e) => return Err(SendError::ServiceError(e.to_string())), + Err(DbError::Query(e)) => Err(SendError::BadRequest(e)), + Err(e) => Err(SendError::ServiceError(e.to_string())), } } @@ -467,7 +461,7 @@ mod public { let dao: DebitNoteDao = db.as_dao(); let debit_note: DebitNote = match dao.get(debit_note_id.clone(), node_id).await { - Ok(Some(debit_note)) => debit_note.into(), + Ok(Some(debit_note)) => debit_note, Ok(None) => return Err(AcceptRejectError::ObjectNotFound), Err(e) => return Err(AcceptRejectError::ServiceError(e.to_string())), }; @@ -501,7 +495,7 @@ mod public { counter!("payment.debit_notes.provider.accepted", 1); Ok(Ack {}) } - Err(DbError::Query(e)) => Err(AcceptRejectError::BadRequest(e.to_string())), + Err(DbError::Query(e)) => Err(AcceptRejectError::BadRequest(e)), Err(e) => Err(AcceptRejectError::ServiceError(e.to_string())), } } @@ -541,7 +535,12 @@ mod public { ); counter!("payment.invoices.requestor.received.call", 1); - let agreement = match get_agreement(agreement_id.clone(), core::Role::Requestor).await { + let agreement = match get_agreement( + agreement_id.clone(), + ya_client_model::market::Role::Requestor, + ) + .await + { Err(e) => { return Err(SendError::ServiceError(e.to_string())); } @@ -555,7 +554,12 @@ mod public { }; for activity_id in activity_ids.iter() { - match provider::get_agreement_id(activity_id.clone(), core::Role::Requestor).await { + match provider::get_agreement_id( + activity_id.clone(), + ya_client_model::market::Role::Requestor, + ) + .await + { Ok(Some(id)) if id != agreement_id => { return Err(SendError::BadRequest(format!( "Activity {} belongs to agreement {} not {}", @@ -579,7 +583,7 @@ mod public { return Err(SendError::BadRequest("Invalid sender node ID".to_owned())); } - let node_id = agreement.requestor_id().clone(); + let node_id = *agreement.requestor_id(); match async move { db.as_dao::() .create_if_not_exists(agreement, node_id, Role::Requestor) @@ -605,8 +609,8 @@ mod public { .await { Ok(_) => Ok(Ack {}), - Err(DbError::Query(e)) => return Err(SendError::BadRequest(e.to_string())), - Err(e) => return Err(SendError::ServiceError(e.to_string())), + Err(DbError::Query(e)) => Err(SendError::BadRequest(e)), + Err(e) => Err(SendError::ServiceError(e.to_string())), } } @@ -628,7 +632,7 @@ mod public { let dao: InvoiceDao = db.as_dao(); let invoice: Invoice = match dao.get(invoice_id.clone(), node_id).await { - Ok(Some(invoice)) => invoice.into(), + Ok(Some(invoice)) => invoice, Ok(None) => return Err(AcceptRejectError::ObjectNotFound), Err(e) => return Err(AcceptRejectError::ServiceError(e.to_string())), }; @@ -662,7 +666,7 @@ mod public { counter!("payment.invoices.provider.accepted", 1); Ok(Ack {}) } - Err(DbError::Query(e)) => Err(AcceptRejectError::BadRequest(e.to_string())), + Err(DbError::Query(e)) => Err(AcceptRejectError::BadRequest(e)), Err(e) => Err(AcceptRejectError::ServiceError(e.to_string())), } } @@ -691,7 +695,7 @@ mod public { let dao: InvoiceDao = db.as_dao(); let invoice: Invoice = match dao.get(invoice_id.clone(), msg.recipient_id).await { - Ok(Some(invoice)) => invoice.into(), + Ok(Some(invoice)) => invoice, Ok(None) => return Err(CancelError::ObjectNotFound), Err(e) => return Err(CancelError::ServiceError(e.to_string())), }; diff --git a/core/payment/src/utils.rs b/core/payment/src/utils.rs index 9484b661ba..924f1904c1 100644 --- a/core/payment/src/utils.rs +++ b/core/payment/src/utils.rs @@ -3,8 +3,8 @@ use actix_web::HttpResponse; use futures::Future; use serde::{Deserialize, Serialize}; use std::time::Duration; -use ya_client_model::market::Agreement; -use ya_core_model::{market, Role}; +use ya_client_model::market::{Agreement, Role}; +use ya_core_model::market; use ya_service_bus::{typed as bus, RpcEndpoint}; pub fn fake_get_agreement(agreement_id: String, agreement: Agreement) { @@ -43,8 +43,8 @@ pub async fn get_agreement(agreement_id: String, role: Role) -> Result>( log::trace!("Starting timeout for: {}s", timeout_secs); match tokio::time::timeout(Duration::from_secs_f64(timeout_secs), work).await { Ok(v) => v, - Err(_) => return HttpResponse::GatewayTimeout().finish(), + Err(_) => HttpResponse::GatewayTimeout().finish(), } } else { log::trace!("Executing /wo timeout."); @@ -155,7 +155,7 @@ pub async fn listen_for_events( let timeout_secs = timeout_secs.into(); match getter.get_events().await { Err(e) => return Err(e), - Ok(events) if events.len() > 0 || timeout_secs == 0.0 => return Ok(events), + Ok(events) if !events.is_empty() || timeout_secs == 0.0 => return Ok(events), _ => (), } @@ -164,7 +164,7 @@ pub async fn listen_for_events( loop { tokio::time::sleep(Duration::from_secs(1)).await; let events = getter.get_events().await?; - if events.len() > 0 { + if !events.is_empty() { break Ok(events); } } diff --git a/core/payment/src/wallet.rs b/core/payment/src/wallet.rs index b645313f51..19fb5a6e37 100644 --- a/core/payment/src/wallet.rs +++ b/core/payment/src/wallet.rs @@ -44,6 +44,7 @@ pub async fn exit( Ok(tx_id) } +#[allow(clippy::too_many_arguments)] pub async fn transfer( sender: String, to: String, diff --git a/core/persistence/Cargo.toml b/core/persistence/Cargo.toml index a7c380697d..6c804954c2 100644 --- a/core/persistence/Cargo.toml +++ b/core/persistence/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ya-persistence" -version = "0.2.0" +version = "0.3.0" authors = ["Golem Factory "] edition = "2018" @@ -9,11 +9,11 @@ default = [] service = ["ya-service-api", "ya-service-api-interfaces", "ya-utils-process", "structopt"] [dependencies] -ya-client-model = { version = "0.4", features = [ "with-diesel" ] } -ya-core-model = { version = "^0.6" } +ya-client-model = { version = "0.5", features = [ "with-diesel" ] } +ya-core-model = { version = "0.8" } ya-service-api = { version = "0.1", optional = true } -ya-service-api-interfaces = {version = "0.1", optional = true } -ya-utils-process = { version = "0.1", features = ["lock"], optional = true } +ya-service-api-interfaces = { version = "0.2", optional = true } +ya-utils-process = { version = "0.2", features = ["lock"], optional = true } anyhow = "1.0.26" bigdecimal = "0.2" @@ -30,4 +30,5 @@ tokio = { version = "1", features = [] } [dev-dependencies] tempdir = "0.3.7" +test-case = "2" tokio = { version = "1", features = ["macros", "rt"] } diff --git a/core/persistence/src/executor.rs b/core/persistence/src/executor.rs index 3972140210..9190a1a929 100644 --- a/core/persistence/src/executor.rs +++ b/core/persistence/src/executor.rs @@ -61,7 +61,7 @@ fn connection_customizer( let mut lock_cnt = self.0.write().unwrap(); *lock_cnt += 1; log::trace!("on_acquire connection [rw:{}]", *lock_cnt); - Ok(conn.batch_execute(CONNECTION_INIT).map_err(|e| { + conn.batch_execute(CONNECTION_INIT).map_err(|e| { log::error!( "error: {:?}, on: {}, [lock: {}]", e, @@ -69,7 +69,7 @@ fn connection_customizer( *lock_cnt ); diesel::r2d2::Error::QueryError(e) - })?) + }) } fn on_release(&self, _conn: SqliteConnection) { @@ -97,7 +97,7 @@ impl DbExecutor { let tx_lock: TxLock = Arc::new(RwLock::new(0)); let builder = Pool::builder().connection_customizer(Box::new(connection_customizer( - database_url.clone(), + database_url, tx_lock.clone(), ))); @@ -124,7 +124,7 @@ impl DbExecutor { pub fn from_env() -> Result { dotenv().ok(); - let database_url = env::var_os("DATABASE_URL").unwrap_or("".into()); + let database_url = env::var_os("DATABASE_URL").unwrap_or_else(|| "".into()); Self::new(database_url.to_string_lossy()) } @@ -225,7 +225,7 @@ where let pool = pool.clone(); match tokio::task::spawn_blocking(move || { let conn = pool.get()?; - let _ = pool.tx_lock.read().unwrap(); + let _guard = pool.tx_lock.read().unwrap(); f(&conn) }) .await @@ -250,6 +250,7 @@ where do_with_rw_connection(pool, move |conn| conn.immediate_transaction(|| f(conn))).await } +#[allow(clippy::let_and_return)] pub async fn readonly_transaction( pool: &PoolType, f: F, diff --git a/core/persistence/src/lib.rs b/core/persistence/src/lib.rs index fc86af2a07..5d08909503 100644 --- a/core/persistence/src/lib.rs +++ b/core/persistence/src/lib.rs @@ -4,6 +4,7 @@ extern crate diesel; pub mod executor; #[cfg(feature = "service")] pub mod service; +mod timestamp; pub mod types; pub use executor::Error; diff --git a/core/persistence/src/service.rs b/core/persistence/src/service.rs index cdda9443e1..071afda82c 100644 --- a/core/persistence/src/service.rs +++ b/core/persistence/src/service.rs @@ -1,3 +1,5 @@ +#![allow(clippy::ptr_arg)] + use std::path::{Path, PathBuf}; use structopt::StructOpt; diff --git a/core/persistence/src/timestamp.rs b/core/persistence/src/timestamp.rs new file mode 100644 index 0000000000..71396d396c --- /dev/null +++ b/core/persistence/src/timestamp.rs @@ -0,0 +1,82 @@ +use chrono::{DateTime, NaiveDateTime, Utc}; +use diesel::backend::Backend; +use diesel::deserialize::FromSql; +use diesel::serialize::{Output, ToSql}; +use diesel::sql_types::{Text, Timestamp}; +use diesel::sqlite::Sqlite; +use diesel::{deserialize, serialize}; +use std::io::Write; + +pub trait AdaptTimestamp { + fn adapt(self) -> TimestampAdapter; +} + +/// Sqlite Timestamp formatting omits sub-second parts if it is equal to zero. +/// This results in invalid comparison between `2022-07-29 12:33:14` and `2022-07-29 12:33:14.000`, +/// because sqlite compares text. +/// +/// This adapter enforces timestamp format in database so it is suitable for comparison. +/// +/// Check description of related issues: +/// https://github.com/golemfactory/yagna/issues/2145 +/// https://github.com/golemfactory/yagna/pull/2086 +#[derive(Clone, Debug, AsExpression)] +#[sql_type = "Timestamp"] +pub struct TimestampAdapter(pub NaiveDateTime); + +impl FromSql for TimestampAdapter { + fn from_sql(value: Option<&::RawValue>) -> deserialize::Result { + Ok(NaiveDateTime::from_sql(value)?.adapt()) + } +} + +impl ToSql for TimestampAdapter { + fn to_sql(&self, out: &mut Output) -> serialize::Result { + ToSql::::to_sql(&self.format(), out) + } +} + +impl TimestampAdapter { + pub fn format(&self) -> String { + self.0.format("%F %T.%6f").to_string() + } +} + +impl AdaptTimestamp for NaiveDateTime { + fn adapt(self) -> TimestampAdapter { + TimestampAdapter(self) + } +} + +impl AdaptTimestamp for DateTime { + fn adapt(self) -> TimestampAdapter { + TimestampAdapter(self.naive_utc()) + } +} + +#[cfg(test)] +mod tests { + use crate::types::AdaptTimestamp; + use chrono::{NaiveDate, NaiveDateTime, NaiveTime}; + use test_case::test_case; + + #[test_case(NaiveDateTime::new( + NaiveDate::from_ymd(2022, 7, 29), + NaiveTime::from_hms_micro(12, 33, 14, 0), + ) => "2022-07-29 12:33:14.000000".to_string(); "0 microseconds should be always printed")] + #[test_case(NaiveDateTime::new( + NaiveDate::from_ymd(2022, 7, 29), + NaiveTime::from_hms(12, 33, 14), + ) => "2022-07-29 12:33:14.000000".to_string(); "0 microseconds should be always printed even if creating with from_hms")] + #[test_case(NaiveDateTime::new( + NaiveDate::from_ymd(2022, 7, 29), + NaiveTime::from_hms_micro(12, 33, 14, 123456), + ) => "2022-07-29 12:33:14.123456".to_string(); "non zero microseconds should be printed")] + #[test_case(NaiveDateTime::new( + NaiveDate::from_ymd(2022, 7, 29), + NaiveTime::from_hms_nano(12, 33, 14, 123456789), + ) => "2022-07-29 12:33:14.123456".to_string(); "nanoseconds should be truncated")] + fn test_timestamp_adapter_formatting(timestamp: NaiveDateTime) -> String { + timestamp.adapt().format() + } +} diff --git a/core/persistence/src/types.rs b/core/persistence/src/types.rs index 0d53494659..d96a6ef0de 100644 --- a/core/persistence/src/types.rs +++ b/core/persistence/src/types.rs @@ -8,6 +8,8 @@ use std::io::Write; use std::ops::{Add, Sub}; use std::str::FromStr; +pub use crate::timestamp::{AdaptTimestamp, TimestampAdapter}; + #[derive(Debug, Clone, AsExpression, FromSqlRow, Default, PartialEq, PartialOrd, Eq, Ord)] #[sql_type = "Text"] pub struct BigDecimalField(pub BigDecimal); diff --git a/core/serv-api/Cargo.toml b/core/serv-api/Cargo.toml index 65c4a83b91..39e2de008c 100644 --- a/core/serv-api/Cargo.toml +++ b/core/serv-api/Cargo.toml @@ -8,9 +8,6 @@ edition = "2018" [dependencies] anyhow = "1.0" lazy_static = "1.4" -log = "0.4" -prettytable-rs = "0.8" serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" -serde_yaml = "0.8" +ya-utils-cli = "0.1" url = "2.1.1" diff --git a/core/serv-api/cache/src/lib.rs b/core/serv-api/cache/src/lib.rs index a4e3c45f3e..0896bc81c4 100644 --- a/core/serv-api/cache/src/lib.rs +++ b/core/serv-api/cache/src/lib.rs @@ -5,6 +5,7 @@ use std::hash::Hash; use std::pin::Pin; use std::time::{Duration, SystemTime}; +#[allow(clippy::type_complexity)] pub trait ValueResolver { type Key: Clone; type Value: Clone; @@ -114,7 +115,7 @@ where } self.ord.push(Reverse(KeyTimeEntry { - time: now.clone(), + time: now, key: key.clone(), })); self.map.insert(key, (now, value)); @@ -143,12 +144,6 @@ impl PartialOrd for KeyTimeEntry { impl Ord for KeyTimeEntry { fn cmp(&self, other: &Self) -> Ordering { - if self.time > other.time { - Ordering::Greater - } else if self.time == other.time { - Ordering::Equal - } else { - Ordering::Less - } + self.time.cmp(&other.time) } } diff --git a/core/serv-api/derive/Cargo.toml b/core/serv-api/derive/Cargo.toml index 347dac90d9..6d7c048940 100644 --- a/core/serv-api/derive/Cargo.toml +++ b/core/serv-api/derive/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ya-service-api-derive" -version = "0.1.0" +version = "0.2.0" authors = ["Golem Factory "] edition = "2018" @@ -12,12 +12,12 @@ lazy_static = "1.4" proc-macro-hack = "0.5" proc-macro2 = "1.0" quote = "1.0" -strum = "0.19" -strum_macros = "0.19" +strum = "0.24" +strum_macros = "0.24" syn = { version = "1.0", features = ["full"] } [dev-dependencies] -ya-service-api-interfaces = "0.1" +ya-service-api-interfaces = "0.2" actix-rt = "2.7" actix-service = "2" diff --git a/core/serv-api/derive/src/component.rs b/core/serv-api/derive/src/component.rs index aea06092ee..29e8962410 100644 --- a/core/serv-api/derive/src/component.rs +++ b/core/serv-api/derive/src/component.rs @@ -40,7 +40,7 @@ impl TryFrom<&MetaList> for Component { Some(nested_meta) => match Keyword::try_from(nested_meta)? { Keyword::Flatten => Ok(Component::Cli { flatten: true }), }, - None => return Err(error), + None => Err(error), }, _ => Err(error), } @@ -75,13 +75,9 @@ impl TryFrom<&NestedMeta> for Keyword { type Error = Error; fn try_from(nested_meta: &NestedMeta) -> Result { - let span = nested_meta.span().into(); match nested_meta { - NestedMeta::Meta(meta) => match meta { - Meta::Path(path) => Self::try_from(path), - _ => Err(Error::new(span, "Invalid format")), - }, - _ => Err(Error::new(span, "Invalid format")), + NestedMeta::Meta(Meta::Path(path)) => Self::try_from(path), + _ => Err(Error::new(nested_meta.span(), "Invalid format")), } } } diff --git a/core/serv-api/derive/src/lib.rs b/core/serv-api/derive/src/lib.rs index ed6b87b8ec..7a448a58c4 100644 --- a/core/serv-api/derive/src/lib.rs +++ b/core/serv-api/derive/src/lib.rs @@ -88,7 +88,7 @@ fn define_errors(errors: Vec) -> proc_macro2::TokenStream { fn define_cli_services( vis: &syn::Visibility, ident: &syn::Ident, - services: &Vec, + services: &[Service], ) -> proc_macro2::TokenStream { let mut variants = proc_macro2::TokenStream::new(); let mut variants_match = proc_macro2::TokenStream::new(); @@ -150,10 +150,7 @@ fn define_cli_services( } } -fn define_gsb_services( - services: &Vec, - context_path: &syn::Meta, -) -> proc_macro2::TokenStream { +fn define_gsb_services(services: &[Service], context_path: &syn::Meta) -> proc_macro2::TokenStream { let mut inner = proc_macro2::TokenStream::new(); for service in services.iter() { if !service.supports(Component::Gsb) { @@ -176,7 +173,7 @@ fn define_gsb_services( } fn define_rest_services( - services: &Vec, + services: &[Service], context_path: &syn::Meta, ) -> proc_macro2::TokenStream { let mut inner = proc_macro2::TokenStream::new(); diff --git a/core/serv-api/derive/src/service.rs b/core/serv-api/derive/src/service.rs index 688dcd7fe6..04931f4589 100644 --- a/core/serv-api/derive/src/service.rs +++ b/core/serv-api/derive/src/service.rs @@ -23,7 +23,7 @@ impl Service { self.components.contains(&component) } - fn parse_attrs(attrs: &Vec) -> Result> { + fn parse_attrs(attrs: &[Attribute]) -> Result> { let mut components = HashSet::new(); for attr in attrs.iter() { @@ -49,10 +49,7 @@ impl Service { let nested = syn::parse_macro_input::parse::(stream.into())?; match nested.first() { - Some(nested_meta) => match nested_meta { - NestedMeta::Meta(meta) => Ok(meta.clone()), - _ => Err(Error::new(attr.span(), "Invalid format")), - }, + Some(NestedMeta::Meta(meta)) => Ok(meta.clone()), _ => Err(Error::new(attr.span(), "Invalid format")), } } @@ -100,12 +97,11 @@ impl TryFrom<&Variant> for Service { type Error = Error; fn try_from(variant: &Variant) -> Result { - let span = variant.ident.span().into(); let name = variant.ident.clone(); let components = Self::parse_attrs(&variant.attrs)?; let path = match &variant.fields { Fields::Unnamed(fields) => Self::parse_fields(fields)?, - _ => return Err(Error::new(span, "Invalid format")), + _ => return Err(Error::new(variant.ident.span(), "Invalid format")), }; Ok(Self { @@ -121,7 +117,7 @@ impl std::fmt::Debug for Service { f.write_str(&format!( "Service < name: {}, path: {}, components: {:?} >", self.name, - self.path.to_token_stream().to_string(), + self.path.to_token_stream(), self.components )) } diff --git a/core/serv-api/derive/tests/service-macro.rs b/core/serv-api/derive/tests/service-macro.rs index 119d1364c8..6db236c41f 100644 --- a/core/serv-api/derive/tests/service-macro.rs +++ b/core/serv-api/derive/tests/service-macro.rs @@ -46,7 +46,7 @@ pub mod gsb_rest { pub fn rest>(ctx: &Context) -> actix_web::Scope { inc(ctx, "GsbRest-rest"); actix_web::Scope::new("/gsb-rest-api") - .service(web::resource("/test").to(|| HttpResponse::Ok())) + .service(web::resource("/test").to(HttpResponse::Ok)) } } } @@ -140,7 +140,7 @@ pub mod rest_cli { pub fn rest>(ctx: &Context) -> actix_web::Scope { inc(ctx, "RestCli-rest"); actix_web::Scope::new("/rest-cli-api") - .service(web::resource("/tester").to(|| HttpResponse::Ok())) + .service(web::resource("/tester").to(HttpResponse::Ok)) } } } @@ -207,7 +207,7 @@ fn test_cli_help() { Services::clap().write_long_help(&mut out).unwrap(); assert_eq!( String::from_utf8_lossy(&out), - "ya-service-api-derive 0.1.0 + "ya-service-api-derive 0.2.0 gsb_cli command help USAGE: diff --git a/core/serv-api/interfaces/Cargo.toml b/core/serv-api/interfaces/Cargo.toml index 848fd64813..bb1c470232 100644 --- a/core/serv-api/interfaces/Cargo.toml +++ b/core/serv-api/interfaces/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ya-service-api-interfaces" -version = "0.1.0" +version = "0.2.0" authors = ["Golem Factory "] edition = "2018" diff --git a/core/serv-api/src/lib.rs b/core/serv-api/src/lib.rs index e23aae967b..d368f15373 100644 --- a/core/serv-api/src/lib.rs +++ b/core/serv-api/src/lib.rs @@ -1,13 +1,12 @@ -use anyhow::Result; -use prettytable::{color, format, format::TableFormat, Attr, Cell, Row, Table}; -use serde::Serialize; -use std::collections::HashMap; use std::path::PathBuf; +pub use ya_utils_cli::{CommandOutput, ResponseTable}; + #[derive(Clone, Debug, Default)] pub struct MetricsCtx { pub push_enabled: bool, pub push_host_url: Option, + pub job: String, } #[derive(Clone, Debug, Default)] @@ -21,263 +20,7 @@ pub struct CliCtx { } impl CliCtx { - pub fn output(&self, output: CommandOutput) { + pub fn output(&self, output: CommandOutput) -> Result<(), anyhow::Error> { output.print(self.json_output) } } - -pub enum CommandOutput { - NoOutput, - Object(serde_json::Value), - Table { - columns: Vec, - values: Vec, - summary: Vec, - header: Option, - }, - FormattedObject(Box), -} - -impl CommandOutput { - pub fn object(value: T) -> Result { - Ok(CommandOutput::Object(serde_json::to_value(value)?)) - } - - pub fn print(&self, json_output: bool) { - match self { - CommandOutput::NoOutput => { - if json_output { - println!("null"); - } - } - CommandOutput::Table { - columns, - values, - summary, - header, - } => { - if json_output { - let columns_size_eq_values_size = values.iter().all(|row| match row { - serde_json::Value::Array(values) => values.len() == columns.len(), - _ => false, - }); - if columns_size_eq_values_size { - let kvs: Vec> = values - .iter() - .map(|row| match row { - serde_json::Value::Array(row_values) - if columns.len() == row_values.len() => - { - columns - .into_iter() - .enumerate() - .map(|(idx, key)| (key, &row_values[idx])) - .collect() - } - _ => unreachable!(), - }) - .collect(); - println!( - "{}", - serde_json::to_string_pretty(&serde_json::json!(kvs)).unwrap() - ) - } else { - println!( - "{}", - serde_json::to_string_pretty(&serde_json::json!({ - "headers": columns, - "values": values - })) - .unwrap() - ) - } - } else { - if let Some(txt) = header { - println!("{}", txt); - } - print_table(columns, values, summary); - } - } - CommandOutput::Object(v) => { - if json_output { - println!("{}", serde_json::to_string_pretty(&v).unwrap()) - } else { - match v { - serde_json::Value::String(s) => { - println!("{}", s); - } - v => println!("{}", serde_yaml::to_string(&v).unwrap()), - } - } - } - CommandOutput::FormattedObject(formatted_object) => { - if json_output { - println!( - "{}", - serde_json::to_string_pretty(&formatted_object.to_json().unwrap()).unwrap() - ) - } else { - formatted_object.print().unwrap() - } - } - } - } -} - -fn print_table( - columns: &Vec, - values: &Vec, - summary: &Vec, -) { - let mut table = Table::new(); - //table.set_format(*format::consts::FORMAT_NO_LINESEP_WITH_TITLE); - table.set_format(*FORMAT_BASIC); - - table.set_titles(Row::new( - columns - .iter() - .map(|c| { - Cell::new(c) - .with_style(Attr::Bold) - .with_style(Attr::ForegroundColor(color::GREEN)) - }) - .collect(), - )); - if values.is_empty() { - let _ = table.add_row(columns.iter().map(|_| Cell::new("")).collect()); - } - for row in values { - if let Some(row_items) = row.as_array() { - use serde_json::Value; - - let row_strings = row_items - .iter() - .map(|v| match v { - Value::String(s) => s.to_string(), - Value::Null => "".into(), - v => v.to_string(), - }) - .collect(); - table.add_row(row_strings); - } - } - if !summary.is_empty() { - table.add_row(Row::empty()); - table.add_empty_row(); - let l = summary.len(); - for (idx, row) in summary.into_iter().enumerate() { - if let Some(row_items) = row.as_array() { - use serde_json::Value; - - let row_strings = Row::new( - row_items - .iter() - .map(|v| { - let c = Cell::new(&match v { - Value::String(s) => s.to_string(), - Value::Null => "".into(), - v => v.to_string(), - }); - - if idx == l - 1 { - c.with_style(Attr::Bold) - } else { - c - } - }) - .collect(), - ); - table.add_row(row_strings); - } - } - } - let _ = table.printstd(); -} - -impl From<()> for CommandOutput { - fn from(_: ()) -> Self { - CommandOutput::NoOutput - } -} - -impl From for CommandOutput { - fn from(table: ResponseTable) -> Self { - CommandOutput::Table { - columns: table.columns, - values: table.values, - summary: Vec::new(), - header: None, - } - } -} - -pub trait FormattedObject { - fn to_json(&self) -> Result; - - fn print(&self) -> Result<()>; -} - -pub struct ResponseTable { - pub columns: Vec, - pub values: Vec, -} - -impl ResponseTable { - pub fn sort_by(mut self, arg_key: &Option>) -> Self { - let key = match arg_key { - None => return self, - Some(k) => k.as_ref(), - }; - let idx = - match self - .columns - .iter() - .enumerate() - .find_map(|(idx, v)| if v == key { Some(idx) } else { None }) - { - None => return self, - Some(idx) => idx, - }; - self.values - .sort_by_key(|v| Some(v.as_array()?.get(idx)?.to_string())); - self - } - - pub fn with_summary(self, summary: Vec) -> CommandOutput { - CommandOutput::Table { - columns: self.columns, - values: self.values, - summary, - header: None, - } - } - - pub fn with_header(self, header: String) -> CommandOutput { - CommandOutput::Table { - columns: self.columns, - values: self.values, - summary: Vec::new(), - header: Some(header), - } - } -} - -lazy_static::lazy_static! { - pub static ref FORMAT_BASIC: TableFormat = format::FormatBuilder::new() - .column_separator('│') - .borders('│') - .separators( - &[format::LinePosition::Top], - format::LineSeparator::new('─', '┬', '┌', '┐') - ) - .separators( - &[format::LinePosition::Title], - format::LineSeparator::new('─', '┼', '├', '┤') - ) - .separators( - &[format::LinePosition::Bottom], - format::LineSeparator::new('─', '┴', '└', '┘') - ) - .padding(2, 2) - .build(); -} diff --git a/core/serv-api/web/Cargo.toml b/core/serv-api/web/Cargo.toml index c68804c68b..2e836ce75c 100644 --- a/core/serv-api/web/Cargo.toml +++ b/core/serv-api/web/Cargo.toml @@ -1,16 +1,16 @@ [package] name = "ya-service-api-web" -version = "0.1.1" +version = "0.2.0" description = "Web interface for modules of Yagna service and CLI" authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-client = "0.6" -ya-core-model = { version = "^0.6", features = ["appkey"] } +ya-client = "0.7" +ya-core-model = { version = "^0.8", features = ["appkey"] } ya-service-api = "0.1" ya-service-api-cache = "0.1" -ya-service-bus = "0.4" +ya-service-bus = "0.6" actix-service = "2" actix-web = "4" @@ -21,11 +21,11 @@ serde = { version = "1.0", features = ["derive"] } url = "2.1.1" [dev-dependencies] -ya-identity = "0.2" -ya-persistence = "0.2" -ya-sb-router = "0.4" -ya-service-api-derive = "0.1" -ya-service-api-interfaces = "0.1" +ya-identity = "0.3" +ya-persistence = "0.3" +ya-sb-router = "0.6" +ya-service-api-derive = "0.2" +ya-service-api-interfaces = "0.2" actix-rt = "2.7" anyhow = "1.0" diff --git a/core/serv-api/web/examples/auth_middleware.rs b/core/serv-api/web/examples/auth_middleware.rs index e201d7e937..5e48000752 100644 --- a/core/serv-api/web/examples/auth_middleware.rs +++ b/core/serv-api/web/examples/auth_middleware.rs @@ -74,7 +74,7 @@ async fn main() -> anyhow::Result<()> { .await .map_err(map_err)? .map_err(map_err)? - .ok_or(anyhow::Error::msg("Identity not found"))? + .ok_or_else(|| anyhow::Error::msg("Identity not found"))? .node_id; let create = model::Create { diff --git a/core/serv-api/web/src/lib.rs b/core/serv-api/web/src/lib.rs index 1c70449565..f434b8a684 100644 --- a/core/serv-api/web/src/lib.rs +++ b/core/serv-api/web/src/lib.rs @@ -10,11 +10,11 @@ pub fn rest_api_addr() -> String { pub fn rest_api_host_port(api_url: url::Url) -> String { let host = api_url .host() - .expect(&format!("invalid API URL - no host: {}", api_url)) + .unwrap_or_else(|| panic!("invalid API URL - no host: {}", api_url)) .to_string(); let port = api_url .port_or_known_default() - .expect(&format!("invalid API URL - no port: {}", api_url)); + .unwrap_or_else(|| panic!("invalid API URL - no port: {}", api_url)); format!("{}:{}", host, port) } diff --git a/core/serv-api/web/src/middleware/auth/dummy.rs b/core/serv-api/web/src/middleware/auth/dummy.rs index 9f3e9b8acb..253250dd2e 100644 --- a/core/serv-api/web/src/middleware/auth/dummy.rs +++ b/core/serv-api/web/src/middleware/auth/dummy.rs @@ -17,7 +17,7 @@ impl DummyAuth { } } -impl<'s, S, B> Transform for DummyAuth +impl Transform for DummyAuth where S: Service, Error = Error> + 'static, S::Future: 'static, diff --git a/core/serv-api/web/src/middleware/auth/mod.rs b/core/serv-api/web/src/middleware/auth/mod.rs index 52792da2e6..d3b7a88d6e 100644 --- a/core/serv-api/web/src/middleware/auth/mod.rs +++ b/core/serv-api/web/src/middleware/auth/mod.rs @@ -31,7 +31,7 @@ impl Default for Auth { } } -impl<'s, S, B> Transform for Auth +impl Transform for Auth where S: Service, Error = Error> + 'static, S::Future: 'static, @@ -125,5 +125,5 @@ fn parse_auth(msg: &T) -> Result { .headers() .get(actix_web::http::header::AUTHORIZATION) .ok_or(ParseError::Header)?; - Ok(S::parse(header).map_err(|_| ParseError::Header)?) + S::parse(header).map_err(|_| ParseError::Header) } diff --git a/core/serv-api/web/src/middleware/auth/resolver.rs b/core/serv-api/web/src/middleware/auth/resolver.rs index 44d0261321..51712ff245 100644 --- a/core/serv-api/web/src/middleware/auth/resolver.rs +++ b/core/serv-api/web/src/middleware/auth/resolver.rs @@ -6,14 +6,9 @@ use ya_core_model::appkey::{self, AppKey, Get}; use ya_service_api_cache::ValueResolver; use ya_service_bus::actix_rpc; +#[derive(Default)] pub struct AppKeyResolver; -impl Default for AppKeyResolver { - fn default() -> Self { - AppKeyResolver {} - } -} - impl ValueResolver for AppKeyResolver { type Key = String; type Value = AppKey; diff --git a/core/serv/src/extension.rs b/core/serv/src/extension.rs index 9e758f43d7..d556cefe8d 100644 --- a/core/serv/src/extension.rs +++ b/core/serv/src/extension.rs @@ -20,17 +20,17 @@ use ya_core_model::appkey::AppKey; use ya_service_api::{CliCtx, CommandOutput}; use ya_service_bus::typed as bus; -const APP_NAME: &'static str = structopt::clap::crate_name!(); -const DIR_NAME: &'static str = "extensions"; +const APP_NAME: &str = structopt::clap::crate_name!(); +const DIR_NAME: &str = "extensions"; -pub const VAR_YAGNA_EXTENSIONS_DIR: &'static str = "YAGNA_EXTENSIONS_DIR"; +pub const VAR_YAGNA_EXTENSIONS_DIR: &str = "YAGNA_EXTENSIONS_DIR"; -const VAR_YAGNA_DATA_DIR: &'static str = "YAGNA_DATA_DIR"; -const VAR_YAGNA_NODE_ID: &'static str = "YAGNA_NODE_ID"; -const VAR_YAGNA_APP_KEY: &'static str = "YAGNA_APP_KEY"; -const VAR_YAGNA_API_URL: &'static str = "YAGNA_API_URL"; -const VAR_YAGNA_GSB_URL: &'static str = "YAGNA_GSB_URL"; -const VAR_YAGNA_JSON_OUTPUT: &'static str = "YAGNA_JSON_OUTPUT"; +const VAR_YAGNA_DATA_DIR: &str = "YAGNA_DATA_DIR"; +const VAR_YAGNA_NODE_ID: &str = "YAGNA_NODE_ID"; +const VAR_YAGNA_APP_KEY: &str = "YAGNA_APP_KEY"; +const VAR_YAGNA_API_URL: &str = "YAGNA_API_URL"; +const VAR_YAGNA_GSB_URL: &str = "YAGNA_GSB_URL"; +const VAR_YAGNA_JSON_OUTPUT: &str = "YAGNA_JSON_OUTPUT"; pub async fn run( cli_ctx: &CliCtx, @@ -88,7 +88,7 @@ async fn resolve_identity_and_key() -> anyhow::Result<(NodeId, Option)> .await? .context("Failed to call the identity service")?; - let node_id = match identities.into_iter().filter(|i| i.is_default).next() { + let node_id = match identities.into_iter().find(|i| i.is_default) { Some(i) => i.node_id, None => bail!("Default identity not found"), }; @@ -132,6 +132,7 @@ async fn monitor(extension: Extension, ctx: ExtensionCtx) { let _ = futures::future::select(interrupted, restart_loop).await; } +#[allow(clippy::large_enum_variant)] #[derive(Debug, Clone)] pub enum ExtensionCtx { Cli { @@ -150,10 +151,7 @@ pub enum ExtensionCtx { impl ExtensionCtx { pub fn is_autostart(&self) -> bool { - match self { - Self::Autostart { .. } => true, - _ => false, - } + matches!(self, Self::Autostart { .. }) } fn set_env(&self, command: &mut Command) -> anyhow::Result<()> { @@ -304,7 +302,7 @@ impl Extension { entry .file_name() .to_str() - .map(|name| (name.to_string(), entry.path().to_path_buf())) + .map(|name| (name.to_string(), entry.path())) }) .filter_map(|(name, path)| { if name.starts_with(&prefix) && name.ends_with(suffix) { @@ -317,7 +315,7 @@ impl Extension { .fold(Default::default(), |mut coll, (name, path)| { if is_executable(&path) { let conf = Self::conf_path(&path) - .and_then(|p| ExtensionConf::read(p)) + .and_then(ExtensionConf::read) .unwrap_or_default(); let ext = Self { name, path, conf }; if !coll.contains(&ext) { diff --git a/core/serv/src/main.rs b/core/serv/src/main.rs index 30bfe66107..7d92ef5c9b 100644 --- a/core/serv/src/main.rs +++ b/core/serv/src/main.rs @@ -112,7 +112,7 @@ impl CliArgs { pub async fn run_command(self) -> Result<()> { let ctx: CliCtx = (&self).try_into()?; - ctx.output(self.command.run_command(&ctx).await?); + ctx.output(self.command.run_command(&ctx).await?)?; Ok(()) } } @@ -179,20 +179,15 @@ impl Provider for ServiceContext { } impl Provider for ServiceContext { - fn component(&self) -> () { - () - } + fn component(&self) {} } impl ServiceContext { - fn make_entry(path: &PathBuf, name: &str) -> Result<(TypeId, DbExecutor)> { + fn make_entry(path: &Path, name: &str) -> Result<(TypeId, DbExecutor)> { Ok((TypeId::of::(), DbExecutor::from_data_dir(path, name)?)) } - fn make_mixed_entry( - path: &PathBuf, - name: &str, - ) -> Result<(TypeId, DbMixedExecutor)> { + fn make_mixed_entry(path: &Path, name: &str) -> Result<(TypeId, DbMixedExecutor)> { let disk_db = DbExecutor::from_data_dir(path, name)?; let ram_db = DbExecutor::in_memory(name)?; @@ -246,11 +241,12 @@ enum Services { Metrics(MetricsService), #[enable(gsb, rest, cli)] Version(VersionService), - #[enable(gsb, cli)] + #[enable(gsb, rest, cli)] Net(NetService), + //TODO enable VpnService::rest for v2 / or create common scope for v1 and v2 #[enable(rest)] Vpn(VpnService), - #[enable(gsb, rest)] + #[enable(gsb, rest, cli)] Market(MarketService), #[enable(gsb, rest, cli)] Activity(ActivityService), @@ -320,7 +316,7 @@ impl CliCommand { pub async fn run_command(self, ctx: &CliCtx) -> Result { match self { CliCommand::Commands(command) => { - start_logger("warn", None, &vec![], false)?; + start_logger("warn", None, &[], false)?; command.run_command(ctx).await } CliCommand::Complete(complete) => complete.run_command(ctx), @@ -374,14 +370,14 @@ impl ExtensionCommand { } fn map>(extensions: I) -> Result { - Ok(CommandOutput::object( + CommandOutput::object( extensions .map(|mut ext| { let name = std::mem::take(&mut ext.name); (name, ext) }) .collect::>(), - )?) + ) } fn table>(extensions: I) -> Result { @@ -407,6 +403,7 @@ impl ExtensionCommand { } } +#[allow(clippy::large_enum_variant)] #[derive(StructOpt, Debug)] enum ServiceCommand { /// Runs server in foreground @@ -489,7 +486,7 @@ impl ServiceCommand { env::set_var( "RUST_LOG", env::var("RUST_LOG") - .unwrap_or(format!("info,actix_web::middleware::logger=warn",)), + .unwrap_or_else(|_| "info,actix_web::middleware::logger=warn".to_string()), ); //this force_debug flag sets default log level to debug @@ -558,7 +555,6 @@ impl ServiceCommand { .wrap(auth::Auth::default()) .route("/me", web::get().to(me)) .service(forward_gsb); - let rest = Services::rest(app, &context); log::info!("Http server thread started on: {}", rest_address); rest @@ -568,7 +564,7 @@ impl ServiceCommand { .bind(api_host_port.clone()) .context(format!("Failed to bind http server on {:?}", api_host_port))?; - let _ = extension::autostart(&ctx.data_dir, &api_url, &ctx.gsb_url) + let _ = extension::autostart(&ctx.data_dir, api_url, &ctx.gsb_url) .await .map_err(|e| log::warn!("Failed to autostart extensions: {e}")); @@ -620,11 +616,11 @@ https://handbook.golem.network/see-also/terms let stdin = std::io::stdin(); let mut stdout = std::io::stdout(); - stdout.write(header.as_bytes())?; + let _ = stdout.write(header.as_bytes())?; stdout.flush()?; loop { - stdout.write("Do you accept the terms and conditions? [yes/no]: ".as_bytes())?; + let _ = stdout.write("Do you accept the terms and conditions? [yes/no]: ".as_bytes())?; stdout.flush()?; let mut buffer = String::new(); diff --git a/core/sgx/Cargo.toml b/core/sgx/Cargo.toml index 8097dda14e..7218e98780 100644 --- a/core/sgx/Cargo.toml +++ b/core/sgx/Cargo.toml @@ -1,12 +1,12 @@ [package] name = "ya-sgx" -version = "0.1.0" +version = "0.2.0" authors = ["Golem Factory "] edition = "2018" [dependencies] anyhow = "1.0" -graphene-sgx = { version = "0.4", features = [ "ias" ] } -ya-client-model = "0.4" -ya-core-model = { version = "^0.6", features = ["sgx"] } -ya-service-bus = "0.4" +graphene-sgx = { version = "0.3.3", features = [ "ias" ] } +ya-client-model = "0.5" +ya-core-model = { version = "^0.8", features = ["sgx"] } +ya-service-bus = "0.6" diff --git a/core/sgx/src/service.rs b/core/sgx/src/service.rs index c83eb514ee..29529146b4 100644 --- a/core/sgx/src/service.rs +++ b/core/sgx/src/service.rs @@ -6,7 +6,7 @@ use ya_core_model::sgx::{AttestationResponse, VerifyAttestationEvidence, BUS_ID} use ya_service_bus::typed as bus; pub fn bind_gsb() { - let _ = bus::bind(&BUS_ID, verify_attestation_evidence); + let _ = bus::bind(BUS_ID, verify_attestation_evidence); } async fn verify_attestation_evidence( diff --git a/core/version/Cargo.toml b/core/version/Cargo.toml index 5f05130536..10097b38a2 100644 --- a/core/version/Cargo.toml +++ b/core/version/Cargo.toml @@ -1,18 +1,18 @@ [package] name = "ya-version" -version = "0.1.0" +version = "0.2.0" description = "Version handling" authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-client = "0.6" +ya-client = "0.7" ya-compile-time-utils = "0.2" -ya-core-model = { version = "^0.6", features = ["version"] } -ya-persistence = "0.2" +ya-core-model = { version = "^0.8", features = ["version"] } +ya-persistence = "0.3" ya-service-api = "0.1" -ya-service-api-interfaces = "0.1" -ya-service-bus = "0.4" +ya-service-api-interfaces = "0.2" +ya-service-bus = "0.6" actix-web = "4" anyhow = "1.0" diff --git a/core/version/src/db/dao.rs b/core/version/src/db/dao.rs index c0f2095e01..e4d24415d5 100644 --- a/core/version/src/db/dao.rs +++ b/core/version/src/db/dao.rs @@ -36,7 +36,7 @@ impl<'c> ReleaseDAO<'c> { } pub async fn current_release(&self) -> anyhow::Result> { - readonly_transaction(self.pool, move |conn| get_current_release(conn)).await + readonly_transaction(self.pool, get_current_release).await } pub async fn pending_release(&self) -> anyhow::Result> { diff --git a/core/version/src/db/model.rs b/core/version/src/db/model.rs index 88996729a0..527c8101ca 100644 --- a/core/version/src/db/model.rs +++ b/core/version/src/db/model.rs @@ -5,7 +5,7 @@ use std::convert::TryFrom; use crate::db::schema::version_release; use ya_compile_time_utils::tag2semver; -pub(crate) const DEFAULT_RELEASE_TS: &'static str = "2015-10-13T15:43:00GMT+2"; +pub(crate) const DEFAULT_RELEASE_TS: &str = "2015-10-13T15:43:00GMT+2"; #[derive(Clone, Debug, Identifiable, Insertable, Queryable, Serialize, Deserialize)] #[primary_key(version)] @@ -29,7 +29,7 @@ impl DBRelease { ya_compile_time_utils::build_date(), ya_compile_time_utils::build_number_str() .map(|bn| format!(" build #{}", bn)) - .unwrap_or("".into()) + .unwrap_or_else(|| "".into()) ), seen: true, release_ts: parse_release_ts(DEFAULT_RELEASE_TS)?, @@ -67,7 +67,7 @@ impl TryFrom for DBRelease { } fn parse_release_ts(ts: &str) -> anyhow::Result { - Ok(NaiveDateTime::parse_from_str(&ts, "%Y-%m-%dT%H:%M:%S%Z")?) + Ok(NaiveDateTime::parse_from_str(ts, "%Y-%m-%dT%H:%M:%S%Z")?) } #[cfg(test)] diff --git a/core/version/src/github.rs b/core/version/src/github.rs index 25764115af..ff1b3964a4 100644 --- a/core/version/src/github.rs +++ b/core/version/src/github.rs @@ -10,8 +10,8 @@ use crate::db::dao::ReleaseDAO; use crate::db::model::DBRelease; use crate::service::cli::ReleaseMessage; -const REPO_OWNER: &'static str = "golemfactory"; -const REPO_NAME: &'static str = "yagna"; +const REPO_OWNER: &str = "golemfactory"; +const REPO_NAME: &str = "yagna"; pub async fn check_latest_release(db: &DbExecutor) -> anyhow::Result { log::debug!("Checking latest Yagna release"); diff --git a/core/version/src/notifier.rs b/core/version/src/notifier.rs index 53adcc573c..497972d87e 100644 --- a/core/version/src/notifier.rs +++ b/core/version/src/notifier.rs @@ -8,9 +8,9 @@ use crate::github::check_running_release; use crate::service::cli::ReleaseMessage; pub async fn on_start(db: &DbExecutor) -> anyhow::Result<()> { - check_running_release(&db).await?; + check_running_release(db).await?; - if let Err(e) = github::check_latest_release(&db).await { + if let Err(e) = github::check_latest_release(db).await { log::error!("Failed to check for new Yagna release: {}", e); }; @@ -42,7 +42,7 @@ pub(crate) async fn pinger(db: DbExecutor) -> ! { match release_dao.pending_release().await { Ok(Some(release)) => { if !release.seen { - log::warn!("{}", ReleaseMessage::Available(&release.into())) + log::warn!("{}", ReleaseMessage::Available(&release)) } } Ok(None) => log::trace!("Your Yagna is up to date"), diff --git a/core/version/src/service/cli.rs b/core/version/src/service/cli.rs index a46bb9a400..a4aa2b2f9c 100644 --- a/core/version/src/service/cli.rs +++ b/core/version/src/service/cli.rs @@ -4,10 +4,8 @@ use ya_core_model::version; use ya_service_api::{CliCtx, CommandOutput}; use ya_service_bus::{typed as bus, RpcEndpoint}; -const PROVIDER_UPDATE_CMD: &'static str = - "curl -sSf https://join.golem.network/as-provider | bash -"; -const REQUESTOR_UPDATE_CMD: &'static str = - "curl -sSf https://join.golem.network/as-requestor | bash -"; +const PROVIDER_UPDATE_CMD: &str = "curl -sSf https://join.golem.network/as-provider | bash -"; +const REQUESTOR_UPDATE_CMD: &str = "curl -sSf https://join.golem.network/as-requestor | bash -"; #[derive(thiserror::Error, Debug, Clone)] pub(crate) enum ReleaseMessage<'a> { diff --git a/core/version/src/service/gsb.rs b/core/version/src/service/gsb.rs index 2ac50f4f01..04ffb493b9 100644 --- a/core/version/src/service/gsb.rs +++ b/core/version/src/service/gsb.rs @@ -27,7 +27,6 @@ async fn skip_version_gsb( ) -> RpcMessageResult { match db.as_dao::().skip_pending_release().await { Ok(r) => Ok(r.map(|r| { - let r = r.into(); log::info!("{}", ReleaseMessage::Skipped(&r)); counter!("version.skip", 1); r diff --git a/core/vpn/Cargo.toml b/core/vpn/Cargo.toml index 14ae047043..9cc7484bd7 100644 --- a/core/vpn/Cargo.toml +++ b/core/vpn/Cargo.toml @@ -1,19 +1,19 @@ [package] name = "ya-vpn" -version = "0.1.0" +version = "0.2.0" authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-core-model = { version = "^0.6", features = ["activity", "market"] } -ya-client-model = { version = "0.4", features = ["sgx"] } -ya-net = "0.2" -ya-persistence = "0.2" +ya-core-model = { version = "^0.8", features = ["activity", "market"] } +ya-client-model = { version = "0.5", features = ["sgx"] } +ya-net = "0.3" +ya-persistence = "0.3" ya-service-api = "0.1" -ya-service-api-interfaces = "0.1" -ya-service-api-web = "0.1" -ya-service-bus = "0.4" -ya-utils-networking = { version = "0.1", default-features = false, features = ["vpn"]} +ya-service-api-interfaces = "0.2" +ya-service-api-web = "0.2" +ya-service-bus = "0.6" +ya-utils-networking = { version = "0.2", default-features = false, features = ["vpn"]} actix = "0.13" actix-web = "4" @@ -37,7 +37,7 @@ tokio = { version = "1", features = ["time"] } uuid = { version = "0.8", features = ["v4"] } [dev-dependencies] -ya-client = "0.6" +ya-client = "0.7" actix-rt = "2.7" sha3 = "0.8.2" diff --git a/core/vpn/examples/ws.rs b/core/vpn/examples/ws.rs index 653bc5d54d..b19be96b06 100644 --- a/core/vpn/examples/ws.rs +++ b/core/vpn/examples/ws.rs @@ -10,7 +10,7 @@ use structopt::StructOpt; use tokio::fs::OpenOptions; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use url::Url; -use ya_client::net::NetRequestorApi; +use ya_client::net::NetVpnApi; use ya_client::web::WebClient; use ya_client_model::net::{Address, NewNetwork, Node}; @@ -64,7 +64,7 @@ async fn main() -> anyhow::Result<()> { Some(_) => cli.api_url, None => std::env::var("YAGNA_API_URL").ok(), } - .unwrap_or("http://127.0.0.1:7464".to_string()); + .unwrap_or_else(|| "http://127.0.0.1:7464".to_string()); let app_key = match &cli.app_key { Some(app_key) => Some(app_key.clone()), None => std::env::var("YAGNA_APPKEY").ok(), @@ -108,7 +108,7 @@ async fn main() -> anyhow::Result<()> { .api_url(Url::parse(&api_url)?) .auth_token(&app_key) .build(); - let api: NetRequestorApi = client.interface()?; + let api: NetVpnApi = client.interface()?; if cli.skip_create { println!("Re-using network: {}", net_id); @@ -125,14 +125,14 @@ async fn main() -> anyhow::Result<()> { let net_id = &network.id; api.add_address( - &net_id, + net_id, &Address { ip: net_requestor_address, }, ) .await?; api.add_node( - &net_id, + net_id, &Node { id: cli.id.clone(), ip: cli.host.clone(), @@ -165,7 +165,7 @@ async fn main() -> anyhow::Result<()> { } vec }; - if vec.len() == 0 { + if vec.is_empty() { break; } remaining -= vec.len() as u64; diff --git a/core/vpn/src/device.rs b/core/vpn/src/device.rs index d57248cc8e..345d7ef787 100644 --- a/core/vpn/src/device.rs +++ b/core/vpn/src/device.rs @@ -70,8 +70,7 @@ impl<'a> phy::TxToken for TxToken<'a> { where F: FnOnce(&mut [u8]) -> smoltcp::Result, { - let mut buffer = Vec::with_capacity(len); - buffer.resize(len, 0); + let mut buffer = vec![0; len]; let result = f(&mut buffer); if result.is_ok() { self.queue.push_back(buffer); diff --git a/core/vpn/src/interface.rs b/core/vpn/src/interface.rs index d4741007ec..1c20bf43d0 100644 --- a/core/vpn/src/interface.rs +++ b/core/vpn/src/interface.rs @@ -43,8 +43,7 @@ pub fn add_iface_route(iface: &mut CaptureInterface, net_ip: IpCidr, route: Rout map.insert(net_ip, route); } ManagedMap::Borrowed(ref map) => { - let mut map: BTreeMap = - map.iter().filter_map(|e| (*e).clone()).collect(); + let mut map: BTreeMap = map.iter().filter_map(|e| (*e)).collect(); map.insert(net_ip, route); *routes = map.into(); } diff --git a/core/vpn/src/network.rs b/core/vpn/src/network.rs index 247f08088e..14a751698f 100644 --- a/core/vpn/src/network.rs +++ b/core/vpn/src/network.rs @@ -22,7 +22,7 @@ use crate::Result; use ya_core_model::activity::{VpnControl, VpnPacket}; use ya_core_model::NodeId; use ya_service_bus::typed::{self, Endpoint}; -use ya_service_bus::{actix_rpc, RpcEndpoint, RpcEnvelope}; +use ya_service_bus::{actix_rpc, RpcEndpoint, RpcEnvelope, RpcRawCall}; use ya_utils_networking::vpn::common::{to_ip, to_net}; use ya_utils_networking::vpn::*; @@ -74,12 +74,12 @@ impl VpnSupervisor { self.blueprints .get(network_id) .cloned() - .ok_or_else(|| Error::NetNotFound) + .ok_or(Error::NetNotFound) } pub async fn create_network( &mut self, - node_id: &NodeId, + node_id: NodeId, network: ya_client_model::net::NewNetwork, ) -> Result { let net = to_net(&network.ip, network.mask.as_ref())?; @@ -88,7 +88,7 @@ impl VpnSupervisor { let net_gw = match network .gateway .as_ref() - .map(|g| IpAddr::from_str(&g)) + .map(|g| IpAddr::from_str(g)) .transpose()? { Some(gw) => gw, @@ -102,8 +102,8 @@ impl VpnSupervisor { let actor = self .arbiter .spawn_ext(async move { - let stack = Stack::new(net_ip, net_route(net_gw.clone())?); - let vpn = Vpn::new(stack, vpn_net); + let stack = Stack::new(net_ip, net_route(net_gw)?); + let vpn = Vpn::new(node_id, stack, vpn_net); Ok::<_, Error>(vpn.start()) }) .await?; @@ -118,7 +118,7 @@ impl VpnSupervisor { self.networks.insert(net_id.clone(), actor); self.blueprints.insert(net_id.clone(), network.clone()); self.ownership - .entry(node_id.clone()) + .entry(node_id) .or_insert_with(Default::default) .insert(net_id); @@ -131,10 +131,7 @@ impl VpnSupervisor { network_id: &str, ) -> Result>> { self.owner(node_id, network_id)?; - let vpn = self - .networks - .remove(network_id) - .ok_or_else(|| Error::NetNotFound)?; + let vpn = self.networks.remove(network_id).ok_or(Error::NetNotFound)?; self.blueprints.remove(network_id); self.forward(vpn, Shutdown {}) } @@ -174,28 +171,34 @@ impl VpnSupervisor { self.networks .get(network_id) .cloned() - .ok_or_else(|| Error::NetNotFound) + .ok_or(Error::NetNotFound) } fn owner(&self, node_id: &NodeId, network_id: &str) -> Result<()> { self.ownership .get(node_id) .map(|s| s.contains(network_id)) - .ok_or_else(|| Error::NetNotFound)? - .then(|| ()) - .ok_or_else(|| Error::Forbidden) + .ok_or(Error::NetNotFound)? + .then_some(()) + .ok_or(Error::Forbidden) } } pub struct Vpn { + node_id: String, vpn: Network>, stack: Stack<'static>, connections: HashMap, } impl Vpn { - pub fn new(stack: Stack<'static>, vpn: Network>) -> Self { + pub fn new( + node_id: NodeId, + stack: Stack<'static>, + vpn: Network>, + ) -> Self { Self { + node_id: node_id.to_string(), vpn, stack, connections: Default::default(), @@ -258,7 +261,7 @@ impl Vpn { let addr_ = addr.clone(); tokio::task::spawn_local(async move { - if let Err(_) = user_tx.send(data).await { + if (user_tx.send(data).await).is_err() { addr_.do_send(Disconnect::new(handle, DisconnectReason::SinkClosed)); } }); @@ -268,7 +271,7 @@ impl Vpn { processed } - fn process_egress<'a>(&mut self) -> bool { + fn process_egress(&mut self) -> bool { let mut processed = false; let vpn_id = self.vpn.id().clone(); @@ -312,8 +315,10 @@ impl Vpn { }; let id = vpn_id.clone(); + let fut = endpoint.udp.push_raw_as(&self.node_id, frame.into()); + tokio::task::spawn_local(async move { - if let Err(err) = endpoint.udp.send(VpnPacket(frame.into())).await { + if let Err(err) = fut.await { let addr = endpoint.tcp.addr(); log::warn!("VPN {}: send error to endpoint '{}': {}", id, addr, err); } @@ -329,8 +334,11 @@ impl Actor for Vpn { fn started(&mut self, ctx: &mut Self::Context) { let id = self.vpn.id(); - let vpn_url = gsb_local_url(&id); - actix_rpc::bind::(&vpn_url, ctx.address().recipient()); + let vpn_url = gsb_local_url(id); + let addr = ctx.address(); + + actix_rpc::bind(&vpn_url, addr.clone().recipient()); + actix_rpc::bind_raw(&format!("{vpn_url}/raw"), addr.recipient()); ctx.run_interval(STACK_POLL_INTERVAL, |this, ctx| { this.poll(ctx.address()); @@ -350,6 +358,7 @@ impl Actor for Vpn { async move { let _ = typed::unbind(&vpn_url).await; + let _ = typed::unbind(&format!("{vpn_url}/raw")).await; log::info!("VPN {} stopped", id); } .into_actor(self) @@ -401,7 +410,7 @@ impl Handler for Vpn { .vpn .nodes() .iter() - .map(|(id, ips)| { + .flat_map(|(id, ips)| { ips.iter() .map(|ip| ya_client_model::net::Node { id: id.clone(), @@ -409,7 +418,6 @@ impl Handler for Vpn { }) .collect::>() }) - .flatten() .collect()) } } @@ -596,6 +604,16 @@ impl Handler> for Vpn { } } +impl Handler for Vpn { + type Result = std::result::Result, ya_service_bus::Error>; + + fn handle(&mut self, msg: RpcRawCall, ctx: &mut Self::Context) -> Self::Result { + self.stack.receive_phy(msg.body); + self.poll(ctx.address()); + Ok(Vec::new()) + } +} + impl Handler for Vpn { type Result = ::Result; @@ -726,7 +744,7 @@ fn gsb_local_url(net_id: &str) -> String { fn gsb_remote_url(node_id: &str, net_id: &str) -> network::DuoEndpoint { network::DuoEndpoint { tcp: typed::service(format!("/net/{}/vpn/{}", node_id, net_id)), - udp: typed::service(format!("/udp/net/{}/vpn/{}", node_id, net_id)), + udp: typed::service(format!("/udp/net/{}/vpn/{}/raw", node_id, net_id)), } } @@ -775,7 +793,7 @@ mod tests { let mut supervisor = VpnSupervisor::default(); let network = supervisor .create_network( - &node_id, + node_id, NewNetwork { ip: "10.0.0.0".to_string(), mask: None, diff --git a/core/vpn/src/requestor.rs b/core/vpn/src/requestor.rs index f9299a162c..76d7bd2834 100644 --- a/core/vpn/src/requestor.rs +++ b/core/vpn/src/requestor.rs @@ -1,7 +1,8 @@ +#![allow(clippy::let_unit_value)] + use crate::message::*; use crate::network::VpnSupervisor; use actix::prelude::*; -use actix_web::web::Data; use actix_web::{web, HttpRequest, HttpResponse, Responder, ResponseError}; use actix_web_actors::ws; use futures::channel::mpsc; @@ -20,9 +21,24 @@ const CLIENT_TIMEOUT: Duration = Duration::from_secs(10); type Result = std::result::Result; type WsResult = std::result::Result; +const API_ROOT_PATH: &str = "/net-api"; + pub fn web_scope(vpn_sup: Arc>) -> actix_web::Scope { - actix_web::web::scope(NET_API_PATH) - .app_data(Data::new(vpn_sup)) + let api_v1_subpath = api_subpath(NET_API_V1_VPN_PATH); + let api_v2_subpath = api_subpath(NET_API_V2_VPN_PATH); + + web::scope(API_ROOT_PATH) + .app_data(web::Data::new(vpn_sup)) + .service(vpn_web_scope(api_v1_subpath)) + .service(vpn_web_scope(api_v2_subpath)) +} + +fn api_subpath(path: &str) -> &str { + path.trim_start_matches(API_ROOT_PATH) +} + +fn vpn_web_scope(path: &str) -> actix_web::Scope { + web::scope(path) .service(get_networks) .service(create_network) .service(get_network) @@ -59,7 +75,7 @@ async fn create_network( let network = model.into_inner(); let mut supervisor = vpn_sup.lock().await; let network = supervisor - .create_network(&identity.identity, network) + .create_network(identity.identity, network) .await?; Ok::<_, ApiError>(web::Json(network)) } @@ -251,7 +267,7 @@ impl VpnWebSocket { vpn.send(Packet { data, meta }) .into_actor(self) .map(move |result, this, ctx| { - if let Err(_) = result { + if result.is_err() { log::error!("VPN WebSocket: VPN {} no longer exists", this.network_id); let _ = ctx.address().do_send(Shutdown {}); } @@ -313,7 +329,8 @@ impl Handler for VpnWebSocket { fn handle(&mut self, _: Shutdown, ctx: &mut Self::Context) -> Self::Result { log::warn!("VPN WebSocket: VPN {} is shutting down", self.network_id); - Ok(ctx.stop()) + ctx.stop(); + Ok(()) } } @@ -365,3 +382,15 @@ struct PathConnect { ip: String, port: u16, } + +#[test] +fn test_to_detect_breaking_ya_client_const_changes() { + assert!( + api_subpath(NET_API_V1_VPN_PATH).len() < NET_API_V1_VPN_PATH.len(), + "ya-client const NET_API_V1_VPN_PATH changed" + ); + assert!( + api_subpath(NET_API_V2_VPN_PATH).len() < NET_API_V2_VPN_PATH.len(), + "ya-client const NET_API_V2_VPN_PATH changed" + ) +} diff --git a/core/vpn/src/stack.rs b/core/vpn/src/stack.rs index 590fdb3517..698e3786e1 100644 --- a/core/vpn/src/stack.rs +++ b/core/vpn/src/stack.rs @@ -113,7 +113,7 @@ impl<'a> Stack<'a> { let socket = sockets.get::(handle); socket.endpoint().port } - _ => 0 as u16, + _ => 0_u16, }; sockets.remove(handle); @@ -146,7 +146,7 @@ impl<'a> Stack<'a> { let iface = self.iface.borrow(); iface.ip_addrs().iter().next().cloned() } - .ok_or_else(|| Error::NetEmpty) + .ok_or(Error::NetEmpty) } pub fn add_address(&self, address: IpCidr) { diff --git a/dod.yml b/dod.yml new file mode 100644 index 0000000000..01f6e4c3e9 --- /dev/null +++ b/dod.yml @@ -0,0 +1,3 @@ +dod: + - 'The code is tested enough' + - 'Testability insights are recorded on https://www.notion.so/golemnetwork/Testability-a42886f72cb649129cddd65bc9dfe2b9' diff --git a/exe-unit/Cargo.toml b/exe-unit/Cargo.toml index c442342bb3..8e356c3d63 100644 --- a/exe-unit/Cargo.toml +++ b/exe-unit/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ya-exe-unit" -version = "0.2.0" +version = "0.3.0" authors = ["Golem Factory "] edition = "2018" @@ -28,16 +28,16 @@ libproc = "0.7" winapi = { version = "0.3.8", features = ["jobapi2", "processthreadsapi"] } [dependencies] -ya-agreement-utils = { version = "^0.4" } -ya-manifest-utils = { version = "^0.1" } -ya-client-model = "0.4" +ya-agreement-utils = { version = "0.4" } +ya-manifest-utils = { version = "0.2" } +ya-client-model = "0.5" ya-compile-time-utils = "0.2" -ya-core-model = { version = "^0.6", features = ["activity", "appkey"] } -ya-runtime-api = { version = "0.4", path = "runtime-api", features = ["server"] } -ya-service-bus = "0.4" -ya-transfer = "0.1" +ya-core-model = { version = "^0.8", features = ["activity", "appkey"] } +ya-runtime-api = { version = "0.5", path = "runtime-api", features = ["server"] } +ya-service-bus = "0.6" +ya-transfer = "0.3" ya-utils-path = "0.1" -ya-utils-networking = { version = "0.1", default-features = false, features = ["dns", "vpn"]} +ya-utils-networking = { version = "0.2", default-features = false, features = ["dns", "vpn"]} actix = { version = "0.13", default-features = false } actix-rt = "2.7" @@ -50,7 +50,7 @@ derive_more = "0.99" dotenv = "0.15.0" flexi_logger = { version = "0.22", features = ["colors"] } futures = "0.3" -graphene-sgx = { version = "0.4", optional = true } +graphene-sgx = { version = "0.3.3", optional = true } hex = "0.4.2" ipnet = "2.3" lazy_static = "1.4.0" @@ -68,15 +68,16 @@ signal-hook = "0.3" socket2 = "0.4" structopt = "0.3" thiserror = "1.0" -tokio = { version = "1", features = ["process", "signal", "time", "net"] } +# keep the "rt-multi-thread" feature +tokio = { version = "1", features = ["process", "signal", "time", "net", "rt-multi-thread"] } tokio-util = { version = "0.7.2", features = ["codec", "net"] } tokio-stream = "0.1.6" url = "2.1" yansi = "0.5.0" [dev-dependencies] -ya-runtime-api = { version = "0.4", path = "runtime-api", features = ["codec", "server"] } -ya-sb-router = "0.4" +ya-runtime-api = { version = "0.5", path = "runtime-api", features = ["codec", "server"] } +ya-sb-router = "0.6" actix-files = "0.6" actix-web = "4" diff --git a/exe-unit/examples/exe-unit2gsb.rs b/exe-unit/examples/exe-unit2gsb.rs index 878ae12183..356eaa1dd9 100644 --- a/exe-unit/examples/exe-unit2gsb.rs +++ b/exe-unit/examples/exe-unit2gsb.rs @@ -82,8 +82,8 @@ mod mock_activity { fn started(&mut self, ctx: &mut Self::Context) { let addr = ctx.address(); - actix_rpc::bind::(&super::ACTIVITY_BUS_ID, addr.clone().recipient()); - actix_rpc::bind::(&super::ACTIVITY_BUS_ID, addr.clone().recipient()); + actix_rpc::bind::(super::ACTIVITY_BUS_ID, addr.clone().recipient()); + actix_rpc::bind::(super::ACTIVITY_BUS_ID, addr.recipient()); } } @@ -108,7 +108,10 @@ mod mock_activity { #[actix_rt::main] async fn main() -> anyhow::Result<()> { - env::set_var("RUST_LOG", env::var("RUST_LOG").unwrap_or("info".into())); + env::set_var( + "RUST_LOG", + env::var("RUST_LOG").unwrap_or_else(|_| "info".into()), + ); env_logger::init(); let args: Cli = Cli::from_args(); diff --git a/exe-unit/examples/http-get-put.rs b/exe-unit/examples/http-get-put.rs index 2a8cdb651c..ea1ed2e1ea 100644 --- a/exe-unit/examples/http-get-put.rs +++ b/exe-unit/examples/http-get-put.rs @@ -36,7 +36,10 @@ async fn upload( #[actix_rt::main] async fn main() -> anyhow::Result<()> { - env::set_var("RUST_LOG", env::var("RUST_LOG").unwrap_or("debug".into())); + env::set_var( + "RUST_LOG", + env::var("RUST_LOG").unwrap_or_else(|_| "debug".into()), + ); env_logger::init(); let args: Cli = Cli::from_args(); diff --git a/exe-unit/examples/runtime-server.rs b/exe-unit/examples/runtime-server.rs index 7406d2a585..7d09d0c73a 100644 --- a/exe-unit/examples/runtime-server.rs +++ b/exe-unit/examples/runtime-server.rs @@ -157,19 +157,21 @@ async fn mock_process(pid: u64, id: u64, run: request::RunProcess) { } async fn write_status(id: u64, status: response::ProcessStatus) { - let mut response = Response::default(); - response.id = id; - response.event = true; - response.command = Some(response::Command::Status(status)); - write(response).await; + write(Response { + id, + event: true, + command: Some(response::Command::Status(status)), + }) + .await; } async fn write_response(id: u64, command: response::Command) { - let mut response = Response::default(); - response.id = id; - response.event = false; - response.command = Some(command); - write(response).await; + write(Response { + id, + event: false, + command: Some(command), + }) + .await; } async fn write(res: Response) { diff --git a/exe-unit/examples/transfer.rs b/exe-unit/examples/transfer.rs index db53c410a0..40105bc158 100644 --- a/exe-unit/examples/transfer.rs +++ b/exe-unit/examples/transfer.rs @@ -39,7 +39,7 @@ fn create_file(path: &Path, name: &str, chunk_size: usize, chunk_count: usize) - .collect(); hasher.input(&input); - file_src.write(&input).unwrap(); + let _ = file_src.write(&input).unwrap(); } file_src.flush().unwrap(); hasher.result() @@ -144,7 +144,10 @@ fn verify_hash, P: AsRef>(hash: &HashOutput, path: P, file_n #[actix_rt::main] async fn main() -> anyhow::Result<()> { - env::set_var("RUST_LOG", env::var("RUST_LOG").unwrap_or("debug".into())); + env::set_var( + "RUST_LOG", + env::var("RUST_LOG").unwrap_or_else(|_| "debug".into()), + ); env_logger::init(); log::debug!("Creating directories"); @@ -172,8 +175,8 @@ async fn main() -> anyhow::Result<()> { }, ]; - let chunk_size = 4096 as usize; - let chunk_count = 256 as usize; + let chunk_size = 4096_usize; + let chunk_count = 256_usize; log::debug!( "Creating a random file of size {} * {}", @@ -279,8 +282,10 @@ async fn main() -> anyhow::Result<()> { println!(); log::warn!("[>>] Transfer container (archive TAR.GZ) -> HTTP"); - let mut args = TransferArgs::default(); - args.format = Some(String::from("tar.gz")); + let args = TransferArgs { + format: Some(String::from("tar.gz")), + ..Default::default() + }; // args.fileset = Some(FileSet::Pattern(SetEntry::Single("**/rnd-*".into()))); transfer_with_args( &addr, @@ -296,8 +301,10 @@ async fn main() -> anyhow::Result<()> { println!(); log::warn!("[>>] Transfer container (archive ZIP) -> HTTP"); - let mut args = TransferArgs::default(); - args.format = Some(String::from("zip")); + let args = TransferArgs { + format: Some(String::from("zip")), + ..Default::default() + }; // args.fileset = Some(FileSet::Pattern(SetEntry::Single("**/rnd-*".into()))); transfer_with_args( &addr, @@ -313,8 +320,10 @@ async fn main() -> anyhow::Result<()> { println!(); log::warn!("[>>] Transfer HTTP -> container (extract TAR.GZ)"); - let mut args = TransferArgs::default(); - args.format = Some(String::from("tar.gz")); + let args = TransferArgs { + format: Some(String::from("tar.gz")), + ..Default::default() + }; transfer_with_args( &addr, "http://127.0.0.1:8001/input.tar.gz", @@ -333,8 +342,10 @@ async fn main() -> anyhow::Result<()> { println!(); log::warn!("[>>] Transfer HTTP -> container (extract ZIP)"); - let mut args = TransferArgs::default(); - args.format = Some(String::from("zip")); + let args = TransferArgs { + format: Some(String::from("zip")), + ..Default::default() + }; transfer_with_args( &addr, "http://127.0.0.1:8001/input.zip", @@ -353,8 +364,10 @@ async fn main() -> anyhow::Result<()> { println!(); log::warn!("[>>] Transfer container (archive TAR.GZ) -> container (extract TAR.GZ)"); - let mut args = TransferArgs::default(); - args.format = Some(String::from("tar.gz")); + let args = TransferArgs { + format: Some(String::from("tar.gz")), + ..Default::default() + }; // args.fileset = Some(FileSet::Pattern(SetEntry::Single("**/rnd-*".into()))); transfer_with_args(&addr, "container:/input", "container:/extract", args).await?; log::warn!("Transfer complete"); diff --git a/exe-unit/examples/transfer_abort.rs b/exe-unit/examples/transfer_abort.rs index b9d49df41e..f877f82227 100644 --- a/exe-unit/examples/transfer_abort.rs +++ b/exe-unit/examples/transfer_abort.rs @@ -34,7 +34,7 @@ fn create_file(path: &PathBuf) { .collect(); for _ in 0..CHUNK_COUNT { - file.write(&input).unwrap(); + let _ = file.write(&input).unwrap(); } file.flush().unwrap(); } @@ -97,7 +97,7 @@ async fn interrupted_transfer( dest: &str, exe_ctx: &ExeUnitContext, ) -> anyhow::Result<()> { - let transfer_service = TransferService::new(&exe_ctx); + let transfer_service = TransferService::new(exe_ctx); let addr = transfer_service.start(); let addr_thread = addr.clone(); @@ -126,14 +126,17 @@ async fn interrupted_transfer( #[actix_rt::main] async fn main() -> anyhow::Result<()> { - env::set_var("RUST_LOG", env::var("RUST_LOG").unwrap_or("debug".into())); + env::set_var( + "RUST_LOG", + env::var("RUST_LOG").unwrap_or_else(|_| "debug".into()), + ); env_logger::init(); log::debug!("Creating directories"); let temp_dir = TempDir::new("transfer")?; - let work_dir = temp_dir.path().clone().join("work_dir"); - let cache_dir = temp_dir.path().clone().join("cache_dir"); + let work_dir = temp_dir.path().to_owned().join("work_dir"); + let cache_dir = temp_dir.path().to_owned().join("cache_dir"); let src_file = temp_dir.path().join("rnd"); let dest_file = temp_dir.path().join("rnd2"); diff --git a/exe-unit/examples/transfer_resume.rs b/exe-unit/examples/transfer_resume.rs index 6124df0019..6b6066f001 100644 --- a/exe-unit/examples/transfer_resume.rs +++ b/exe-unit/examples/transfer_resume.rs @@ -156,14 +156,14 @@ async fn download>(dst_path: P, args: Cli) -> anyhow::Result<()> async fn main() -> anyhow::Result<()> { env::set_var( "RUST_LOG", - env::var("RUST_LOG").unwrap_or( + env::var("RUST_LOG").unwrap_or_else(|_| { "debug,\ h2=info,\ trust_dns_proto=info,\ trust_dns_resolver=info,\ " - .into(), - ), + .into() + }), ); env_logger::init(); diff --git a/exe-unit/runtime-api/Cargo.toml b/exe-unit/runtime-api/Cargo.toml index c61ef40ada..bb52cea6c4 100644 --- a/exe-unit/runtime-api/Cargo.toml +++ b/exe-unit/runtime-api/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ya-runtime-api" description = "Communication API between the Runtime and ExeUnit Supervisor. Provides server implementation for Runtime and client implementation for Supervisor." -version = "0.4.1" +version = "0.5.0" authors = ["Golem Factory "] edition = "2018" license = "GPL-3.0" diff --git a/exe-unit/runtime-api/examples/runtime-server-mock.rs b/exe-unit/runtime-api/examples/runtime-server-mock.rs index de1b383c3a..496b741a17 100644 --- a/exe-unit/runtime-api/examples/runtime-server-mock.rs +++ b/exe-unit/runtime-api/examples/runtime-server-mock.rs @@ -5,7 +5,7 @@ use std::clone::Clone; use std::env; use std::sync::{Arc, Mutex}; use std::time::Duration; -use tokio; + use ya_runtime_api::server::*; // server @@ -25,8 +25,7 @@ impl RuntimeService for RuntimeMock { fn run_process(&self, _run: RunProcess) -> AsyncResponse { async move { - let mut resp: RunProcessResp = Default::default(); - resp.pid = 100; + let resp = RunProcessResp { pid: 100 }; log::debug!("before sleep"); tokio::time::sleep(Duration::from_secs(3)).await; log::debug!("after sleep"); @@ -112,9 +111,11 @@ async fn main() -> anyhow::Result<()> { let events = EventMock::new(); let c = spawn(cmd, events.clone()).await?; log::debug!("hello_result={:?}", c.hello("0.0.0x").await); - let mut run = RunProcess::default(); - run.bin = "sleep".to_owned(); - run.args = vec!["10".to_owned()]; + let run = RunProcess { + bin: "sleep".to_owned(), + args: vec!["10".to_owned()], + ..Default::default() + }; let sleep_1 = c.run_process(run.clone()); let sleep_2 = c.run_process(run.clone()); let sleep_3 = c.run_process(run); diff --git a/exe-unit/runtime-api/src/server.rs b/exe-unit/runtime-api/src/server.rs index e73fafc027..0229820344 100644 --- a/exe-unit/runtime-api/src/server.rs +++ b/exe-unit/runtime-api/src/server.rs @@ -1,4 +1,6 @@ pub mod proto { + #![allow(clippy::derive_partial_eq_without_eq)] + include!(concat!(env!("OUT_DIR"), "/ya_runtime_api.rs")); impl response::Error { diff --git a/exe-unit/runtime-api/src/server/client.rs b/exe-unit/runtime-api/src/server/client.rs index 3eea3a71ba..554c69b9ef 100644 --- a/exe-unit/runtime-api/src/server/client.rs +++ b/exe-unit/runtime-api/src/server/client.rs @@ -7,7 +7,7 @@ use std::collections::HashMap; use std::fmt::Debug; use std::sync::atomic::{AtomicU64, Ordering::Relaxed}; -const REQUEST_ID: AtomicU64 = AtomicU64::new(0); +static REQUEST_ID: AtomicU64 = AtomicU64::new(0); struct ClientInner { ids: u64, @@ -100,8 +100,9 @@ where Out::Error: Debug, { fn hello(&self, version: &str) -> AsyncResponse<'_, String> { + let id = REQUEST_ID.fetch_add(1, Relaxed); let request = proto::Request { - id: REQUEST_ID.fetch_add(1, Relaxed), + id, command: Some(proto::request::Command::Hello(proto::request::Hello { version: version.to_owned(), })), @@ -118,8 +119,9 @@ where } fn run_process(&self, run: RunProcess) -> AsyncResponse { + let id = REQUEST_ID.fetch_add(1, Relaxed); let request = proto::Request { - id: REQUEST_ID.fetch_add(1, Relaxed), + id, command: Some(proto::request::Command::Run(run)), }; let fut = self.call(request); @@ -134,8 +136,9 @@ where } fn kill_process(&self, kill: KillProcess) -> AsyncResponse<()> { + let id = REQUEST_ID.fetch_add(1, Relaxed); let request = proto::Request { - id: REQUEST_ID.fetch_add(1, Relaxed), + id, command: Some(proto::request::Command::Kill(kill)), }; let fut = self.call(request); @@ -150,8 +153,9 @@ where } fn create_network(&self, network: CreateNetwork) -> AsyncResponse { + let id = REQUEST_ID.fetch_add(1, Relaxed); let request = proto::Request { - id: REQUEST_ID.fetch_add(1, Relaxed), + id, command: Some(proto::request::Command::Network(network)), }; let fut = self.call(request); @@ -167,8 +171,10 @@ where fn shutdown(&self) -> AsyncResponse<'_, ()> { let shutdown = proto::request::Shutdown::default(); + let id = REQUEST_ID.fetch_add(1, Relaxed); + let request = proto::Request { - id: REQUEST_ID.fetch_add(1, Relaxed), + id, command: Some(proto::request::Command::Shutdown(shutdown)), }; let fut = self.call(request); @@ -226,7 +232,7 @@ pub async fn spawn( map_return_code(child.wait().await, pid) } }; - if let Err(_) = status_tx.send(code) { + if status_tx.send(code).is_err() { log::warn!("Unable to update process {} status: receiver is gone", pid); } }); @@ -236,10 +242,10 @@ pub async fn spawn( use proto::response::Command; match response.command { Some(Command::Status(status)) => { - let _ = handler.on_process_status(status).await; + handler.on_process_status(status).await; } Some(Command::RtStatus(status)) => { - let _ = handler.on_runtime_status(status).await; + handler.on_runtime_status(status).await; } cmd => log::warn!("invalid event: {:?}", cmd), } diff --git a/exe-unit/runtime-api/src/server/service.rs b/exe-unit/runtime-api/src/server/service.rs index ea89692f7c..edebe0efc4 100644 --- a/exe-unit/runtime-api/src/server/service.rs +++ b/exe-unit/runtime-api/src/server/service.rs @@ -35,19 +35,18 @@ async fn handle_command( } async fn handle(service: &impl RuntimeService, request: proto::Request) -> proto::Response { - let id = request.id; - let mut resp = proto::Response::default(); - resp.id = id; - - resp.command = Some(if let Some(command) = request.command { - match handle_command(service, command).await { - Ok(response) => response, - Err(err) => proto::response::Command::Error(err), - } - } else { - proto::response::Command::Error(ErrorResponse::msg("unknown command")) - }); - resp + proto::Response { + id: request.id, + command: Some(if let Some(command) = request.command { + match handle_command(service, command).await { + Ok(response) => response, + Err(err) => proto::response::Command::Error(err), + } + } else { + proto::response::Command::Error(ErrorResponse::msg("unknown command")) + }), + ..Default::default() + } } pub struct EventEmitter { diff --git a/exe-unit/src/bin.rs b/exe-unit/src/bin.rs index 878a8d63f2..879b72f753 100644 --- a/exe-unit/src/bin.rs +++ b/exe-unit/src/bin.rs @@ -265,7 +265,9 @@ async fn run() -> anyhow::Result<()> { let manifest_ctx = ManifestContext::try_new(&agreement.inner).context("Invalid app manifest")?; - agreement.task_package = manifest_ctx.payload().or(agreement.task_package.take()); + agreement.task_package = manifest_ctx + .payload() + .or_else(|| agreement.task_package.take()); log::info!("Manifest-enabled features: {:?}", manifest_ctx.features()); log::info!("User-provided payload: {:?}", agreement.task_package); diff --git a/exe-unit/src/error.rs b/exe-unit/src/error.rs index 148aa4f08e..81d8b04ab6 100644 --- a/exe-unit/src/error.rs +++ b/exe-unit/src/error.rs @@ -146,7 +146,7 @@ impl From for RpcError { #[cfg(feature = "sgx")] Error::Crypto(e) => RpcError::Service(e.to_string()), #[cfg(feature = "sgx")] - Error::Attestation(e) => RpcError::Service(e.to_string()), + Error::Attestation(e) => RpcError::Service(e), } } } diff --git a/exe-unit/src/handlers/local.rs b/exe-unit/src/handlers/local.rs index 97fda311c5..9905ab4602 100644 --- a/exe-unit/src/handlers/local.rs +++ b/exe-unit/src/handlers/local.rs @@ -56,7 +56,7 @@ impl Handler for ExeUnit { log::debug!("Entering state: {:?}", update.state); log::debug!("Report: {}", self.state.report()); - self.state.inner = update.state.clone(); + self.state.inner = update.state; if self.ctx.activity_id.is_none() || self.ctx.report_url.is_none() { return ActorResponse::reply(()); @@ -79,12 +79,12 @@ impl Handler for ExeUnit { ), ); - return ActorResponse::r#async( + ActorResponse::r#async( async move { fut.await; } .into_actor(self), - ); + ) } } @@ -92,16 +92,11 @@ impl Handler for ExeUnit { type Result = ::Result; fn handle(&mut self, msg: GetStdOut, _: &mut Context) -> Self::Result { - self.state - .batches - .get(&msg.batch_id) - .map(|b| { - b.results - .get(msg.idx) - .map(|r| r.stdout.output_string()) - .flatten() - }) - .flatten() + self.state.batches.get(&msg.batch_id).and_then(|b| { + b.results + .get(msg.idx) + .and_then(|r| r.stdout.output_string()) + }) } } @@ -280,7 +275,7 @@ impl Handler for ExeUnit { } let address = ctx.address(); - let services = std::mem::replace(&mut self.services, Vec::new()); + let services = std::mem::take(&mut self.services); let state = self.state.inner.to_pending(State::Terminated); let reason = format!("{}: {}", msg.0, self.state.report()); diff --git a/exe-unit/src/handlers/rpc.rs b/exe-unit/src/handlers/rpc.rs index 1ed146d74c..44ccd344ca 100644 --- a/exe-unit/src/handlers/rpc.rs +++ b/exe-unit/src/handlers/rpc.rs @@ -41,7 +41,7 @@ impl Handler> for ExeUnit { let (tx, rx) = oneshot::channel(); self.state.start_batch(msg.clone(), tx); - RuntimeRef::from_ctx(&ctx) + RuntimeRef::from_ctx(ctx) .exec( msg, self.runtime.clone(), @@ -63,7 +63,7 @@ impl Handler> for ExeUnit { self.ctx.verify_activity_id(&msg.activity_id)?; Ok(ActivityState { - state: self.state.inner.clone(), + state: self.state.inner, reason: None, error_message: None, }) @@ -156,17 +156,16 @@ impl Handler> for ExeUnit { let duration = Duration::from_secs_f32(msg.timeout.unwrap_or(0.)); let notifier = batch.notifier.clone(); - let idx = msg.command_index.clone(); + let idx = msg.command_index; let batch_id = msg.batch_id.clone(); let fut = async move { if timeout(duration, notifier.when(move |i| i >= await_idx)) .await .is_err() + && msg.command_index.is_some() { - if msg.command_index.is_some() { - return Err(RpcMessageError::Timeout); - } + return Err(RpcMessageError::Timeout); } match address.send(GetBatchResults { batch_id, idx }).await { Ok(results) => Ok(results.0), diff --git a/exe-unit/src/lib.rs b/exe-unit/src/lib.rs index dfc72a277d..09b52b67f3 100644 --- a/exe-unit/src/lib.rs +++ b/exe-unit/src/lib.rs @@ -256,20 +256,25 @@ impl RuntimeRef { _ => StatePair(*s, Some(*s)), }, }; - self.send(SetState::from(state_pre.clone())).await?; + self.send(SetState::from(state_pre)).await?; log::info!("Executing command: {:?}", runtime_cmd.command); - self.pre_runtime(&runtime_cmd, runtime, transfer_service) - .await?; + let result = async { + self.pre_runtime(&runtime_cmd, runtime, transfer_service) + .await?; - let exit_code = runtime.send(runtime_cmd.clone()).await??; - if exit_code != 0 { - return Err(Error::CommandExitCodeError(exit_code)); - } + let exit_code = runtime.send(runtime_cmd.clone()).await??; + if exit_code != 0 { + return Err(Error::CommandExitCodeError(exit_code)); + } - self.post_runtime(&runtime_cmd, runtime, transfer_service) - .await?; + self.post_runtime(&runtime_cmd, runtime, transfer_service) + .await?; + + Ok(()) + } + .await; let state_cur = self.send(GetState {}).await?.0; if state_cur != state_pre { @@ -281,7 +286,7 @@ impl RuntimeRef { } self.send(SetState::from(state_pre.1.unwrap())).await?; - Ok(()) + result } async fn pre_runtime( @@ -401,7 +406,7 @@ impl Actor for ExeUnit { Err(e) => { let err = Error::Other(format!("manifest initialization error: {}", e)); log::error!("Supervisor is shutting down due to {}", err); - let _ = ctx.address().do_send(Shutdown(ShutdownReason::Error(err))); + ctx.address().do_send(Shutdown(ShutdownReason::Error(err))); } }) .wait(ctx); diff --git a/exe-unit/src/logger.rs b/exe-unit/src/logger.rs index 996714774e..d9cd42565d 100644 --- a/exe-unit/src/logger.rs +++ b/exe-unit/src/logger.rs @@ -2,18 +2,18 @@ use chrono::{DateTime, Local}; use flexi_logger::{DeferredNow, Record}; use std::time::SystemTime; -const ENV_VAR_LOG_DIR: &'static str = "EXE_UNIT_LOG_DIR"; -const ENV_VAR_FILE_LOG_LEVEL: &'static str = "EXE_UNIT_FILE_LOG_LEVEL"; +const ENV_VAR_LOG_DIR: &str = "EXE_UNIT_LOG_DIR"; +const ENV_VAR_FILE_LOG_LEVEL: &str = "EXE_UNIT_FILE_LOG_LEVEL"; -const DEFAULT_LOG_LEVEL: &'static str = "info"; -const DEFAULT_FILE_LOG_LEVEL: &'static str = "debug"; -const DEFAULT_LOG_DIR: &'static str = "logs"; -const DEFAULT_LOG_FORMAT: &'static str = "%Y-%m-%dT%H:%M:%S%.3f%z"; +const DEFAULT_LOG_LEVEL: &str = "info"; +const DEFAULT_FILE_LOG_LEVEL: &str = "debug"; +const DEFAULT_LOG_DIR: &str = "logs"; +const DEFAULT_LOG_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%.3f%z"; pub fn start_file_logger() -> anyhow::Result { - let log_dir = std::env::var(ENV_VAR_LOG_DIR).unwrap_or(DEFAULT_LOG_DIR.to_string()); - let log_level = - std::env::var(ENV_VAR_FILE_LOG_LEVEL).unwrap_or(DEFAULT_FILE_LOG_LEVEL.to_string()); + let log_dir = std::env::var(ENV_VAR_LOG_DIR).unwrap_or_else(|_| DEFAULT_LOG_DIR.to_string()); + let log_level = std::env::var(ENV_VAR_FILE_LOG_LEVEL) + .unwrap_or_else(|_| DEFAULT_FILE_LOG_LEVEL.to_string()); Ok(build_logger(Some(log_level))? .log_to_file(flexi_logger::FileSpec::default().directory(log_dir)) @@ -28,7 +28,7 @@ pub fn start_logger() -> anyhow::Result { fn build_logger(log_level: Option) -> anyhow::Result { let level = match log_level { Some(level) => level.to_string(), - None => std::env::var("RUST_LOG").unwrap_or(DEFAULT_LOG_LEVEL.to_string()), + None => std::env::var("RUST_LOG").unwrap_or_else(|_| DEFAULT_LOG_LEVEL.to_string()), }; Ok(flexi_logger::Logger::try_with_str(level)? diff --git a/exe-unit/src/message.rs b/exe-unit/src/message.rs index 7e6ea698fd..12a0d898f2 100644 --- a/exe-unit/src/message.rs +++ b/exe-unit/src/message.rs @@ -12,7 +12,7 @@ use ya_client_model::activity::activity_state::{State, StatePair}; use ya_client_model::activity::exe_script_command::Network; use ya_client_model::activity::{CommandOutput, ExeScriptCommand, ExeScriptCommandResult}; -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Message)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Message)] #[rtype(result = "Result>")] pub struct GetMetrics; @@ -23,14 +23,14 @@ pub struct SetMetric { pub value: f64, } -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Message)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Message)] #[rtype(result = "GetStateResponse")] pub struct GetState; -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, MessageResponse)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, MessageResponse)] pub struct GetStateResponse(pub StatePair); -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Message)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Message)] #[rtype(result = "GetBatchResultsResponse")] pub struct GetBatchResults { pub batch_id: String, @@ -40,14 +40,14 @@ pub struct GetBatchResults { #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, MessageResponse)] pub struct GetBatchResultsResponse(pub Vec); -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Message)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Message)] #[rtype(result = "Option")] pub struct GetStdOut { pub batch_id: String, pub idx: usize, } -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, Message)] +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, Message)] #[rtype(result = "()")] pub struct SetState { pub state: StatePair, @@ -91,10 +91,10 @@ pub struct ExecuteCommand { impl ExecuteCommand { pub fn stateless(&self) -> bool { - match &self.command { - ExeScriptCommand::Sign { .. } | ExeScriptCommand::Terminate { .. } => true, - _ => false, - } + matches!( + &self.command, + ExeScriptCommand::Sign { .. } | ExeScriptCommand::Terminate { .. } + ) } pub fn split(self) -> (ExeScriptCommand, CommandContext) { @@ -174,7 +174,7 @@ pub struct UpdateDeployment { #[rtype(result = "Result<()>")] pub struct Initialize; -#[derive(Clone, Debug, PartialEq, Message)] +#[derive(Clone, Debug, PartialEq, Eq, Message)] #[rtype(result = "()")] pub struct Register(pub Addr) where diff --git a/exe-unit/src/metrics/mod.rs b/exe-unit/src/metrics/mod.rs index b347897477..bf9441c596 100644 --- a/exe-unit/src/metrics/mod.rs +++ b/exe-unit/src/metrics/mod.rs @@ -133,7 +133,7 @@ impl StorageMetric { } fn spawn(&self) { - let interval = self.interval.clone(); + let interval = self.interval; let path = self.path.clone(); let last = self.last.clone(); let running = self.running.clone(); @@ -162,10 +162,7 @@ impl StorageMetric { #[inline] fn read_dir_size(dir: fs::ReadDir) -> (u64, usize) { dir.fold((0, 0), |(sz, sk), file| { - let (size, skipped) = match Self::read_file_size(file) { - Ok(res) => res, - Err(_) => (0u64, 1), - }; + let (size, skipped) = Self::read_file_size(file).unwrap_or((0u64, 1)); (sz + size, sk + skipped) }) } diff --git a/exe-unit/src/metrics/os/unix.rs b/exe-unit/src/metrics/os/unix.rs index d805c45744..2a0b3236c5 100644 --- a/exe-unit/src/metrics/os/unix.rs +++ b/exe-unit/src/metrics/os/unix.rs @@ -11,7 +11,7 @@ lazy_static::lazy_static! { const MAX_UPDATE_RESOLUTION_MS: i64 = 100; pub fn cpu_time() -> Result { - let mut metrics = (&(*METRICS)).write().map_err(SystemError::from)?; + let mut metrics = (*METRICS).write().map_err(SystemError::from)?; metrics.sample()?; Ok(metrics.cpu_total) } @@ -22,7 +22,7 @@ pub fn mem_rss() -> Result { } pub fn mem_peak_rss() -> Result { - let mut metrics = (&(*METRICS)).write().map_err(SystemError::from)?; + let mut metrics = (*METRICS).write().map_err(SystemError::from)?; metrics.sample()?; Ok(metrics.mem_total) } @@ -71,7 +71,7 @@ impl Metrics { if usage.cpu_sec > self.cpu_total { let dv = usage.cpu_sec - self.cpu_total; - *self.cpu.entry(-1).or_insert_with(|| Duration::default()) += dv; + *self.cpu.entry(-1).or_insert_with(Duration::default) += dv; self.cpu_total += dv; } if usage.rss_gib > self.mem_total { diff --git a/exe-unit/src/network.rs b/exe-unit/src/network.rs index 5dd977da3c..157c4c9581 100644 --- a/exe-unit/src/network.rs +++ b/exe-unit/src/network.rs @@ -1,8 +1,9 @@ use std::convert::TryFrom; use std::path::Path; -use futures::channel::mpsc; +use bytes::{Buf, BytesMut}; use futures::Stream; +use tokio::sync::mpsc; use ya_runtime_api::deploy::ContainerEndpoint; use ya_runtime_api::server::Network; @@ -18,7 +19,7 @@ pub(crate) mod inet; pub(crate) mod vpn; pub(crate) struct Endpoint { - tx: mpsc::Sender>>, + tx: mpsc::UnboundedSender>>, rx: Option>> + Unpin>>, } @@ -32,24 +33,49 @@ impl Endpoint { #[cfg(unix)] async fn connect_to_socket>(path: P) -> Result { - use bytes::Bytes; - use futures::{future, SinkExt, StreamExt, TryStreamExt}; - use tokio::io; - use tokio_util::codec::{BytesCodec, FramedRead, FramedWrite}; + use futures::StreamExt; + use tokio::io::{self, AsyncReadExt, AsyncWriteExt}; + use tokio_stream::wrappers::UnboundedReceiverStream; + + const BUFFER_SIZE: usize = (DEFAULT_MAX_FRAME_SIZE + 2) * 4; + + type SocketChannel = ( + mpsc::UnboundedSender>>, + mpsc::UnboundedReceiver>>, + ); let socket = tokio::net::UnixStream::connect(path.as_ref()).await?; - let (read, write) = io::split(socket); - - let sink = FramedWrite::new(write, BytesCodec::new()).with(|v| future::ok(Bytes::from(v))); - let stream = FramedRead::with_capacity(read, BytesCodec::new(), DEFAULT_MAX_FRAME_SIZE) - .into_stream() - .map_ok(|b| b.to_vec()) - .map_err(Error::from); - - let (tx_si, rx_si) = mpsc::channel(1); - tokio::task::spawn_local(async move { - if let Err(e) = rx_si.forward(sink).await { - log::error!("Socket endpoint error: {}", e); + let (read, mut write) = io::split(socket); + let (tx_si, rx_si): SocketChannel = mpsc::unbounded_channel(); + + let stream = { + let buffer: [u8; BUFFER_SIZE] = [0u8; BUFFER_SIZE]; + futures::stream::unfold((read, buffer), |(mut r, mut b)| async move { + match r.read(&mut b).await { + Ok(0) => None, + Ok(n) => Some((Ok(b[..n].to_vec()), (r, b))), + Err(e) => Some((Err(e.into()), (r, b))), + } + }) + .boxed_local() + }; + + tokio::task::spawn(async move { + let mut rx_si = UnboundedReceiverStream::new(rx_si); + loop { + match StreamExt::next(&mut rx_si).await { + Some(Ok(data)) => { + if let Err(e) = write.write_all(data.as_slice()).await { + log::error!("error writing to VM socket endpoint: {e}"); + break; + } + } + Some(Err(e)) => { + log::error!("VM socket endpoint error: {e}"); + break; + } + None => break, + } } }); @@ -89,69 +115,46 @@ impl<'a> TryFrom<&'a DeploymentNetwork> for Network { type Prefix = u16; const PREFIX_SIZE: usize = std::mem::size_of::(); -pub(self) struct RxBuffer { - expected: usize, - inner: Vec, +pub struct RxBuffer { + inner: BytesMut, } impl Default for RxBuffer { fn default() -> Self { Self { - expected: 0, - inner: Vec::with_capacity(PREFIX_SIZE + DEFAULT_MAX_FRAME_SIZE), + inner: BytesMut::with_capacity(2 * (PREFIX_SIZE + DEFAULT_MAX_FRAME_SIZE)), } } } impl RxBuffer { pub fn process(&mut self, received: Vec) -> RxIterator { - RxIterator { - buffer: self, - received, - } + self.inner.extend(received); + RxIterator { buffer: self } } } -struct RxIterator<'a> { +pub struct RxIterator<'a> { buffer: &'a mut RxBuffer, - received: Vec, } impl<'a> Iterator for RxIterator<'a> { type Item = Vec; fn next(&mut self) -> Option { - if self.buffer.expected > 0 && self.received.len() > 0 { - let len = self.buffer.expected.min(self.received.len()); - self.buffer.inner.extend(self.received.drain(..len)); - } - if let Some(len) = read_prefix(&self.buffer.inner) { - if let Some(item) = take_next(&mut self.buffer.inner, len) { - self.buffer.expected = read_prefix(&self.buffer.inner).unwrap_or(0) as usize; - return Some(item); - } - } - - if let Some(len) = read_prefix(&self.received) { - if let Some(item) = take_next(&mut self.received, len) { - return Some(item); - } + return take_next(&mut self.buffer.inner, len); } - - self.buffer.inner.append(&mut self.received); - if let Some(len) = read_prefix(&self.buffer.inner) { - self.buffer.expected = len as usize; - } - None } } -fn take_next(src: &mut Vec, len: Prefix) -> Option> { - let p_len = PREFIX_SIZE + len as usize; +fn take_next(src: &mut BytesMut, len: Prefix) -> Option> { + let len = len as usize; + let p_len = PREFIX_SIZE + len; if src.len() >= p_len { - return Some(src.drain(..p_len).skip(PREFIX_SIZE).collect()); + src.advance(PREFIX_SIZE); + return Some(src.split_to(len).to_vec()); } None } @@ -174,7 +177,7 @@ fn write_prefix(dst: &mut Vec) { fn gsb_endpoint(node_id: &str, net_id: &str) -> DuoEndpoint { DuoEndpoint { tcp: typed::service(format!("/net/{}/vpn/{}", node_id, net_id)), - udp: typed::service(format!("/udp/net/{}/vpn/{}", node_id, net_id)), + udp: typed::service(format!("/udp/net/{}/vpn/{}/raw", node_id, net_id)), } } diff --git a/exe-unit/src/network/inet.rs b/exe-unit/src/network/inet.rs index cf51ab8920..3c0363623e 100644 --- a/exe-unit/src/network/inet.rs +++ b/exe-unit/src/network/inet.rs @@ -28,23 +28,23 @@ use net::{EgressReceiver, IngressEvent, IngressReceiver}; use net::{Error as NetError, Protocol}; use ya_runtime_api::server::{CreateNetwork, NetworkInterface, RuntimeService}; -use ya_utils_networking::vpn::common::{ntoh, DEFAULT_MAX_FRAME_SIZE}; +use ya_utils_networking::vpn::common::ntoh; use ya_utils_networking::vpn::stack as net; use ya_utils_networking::vpn::stack::smoltcp::wire::{EthernetAddress, HardwareAddress}; -use ya_utils_networking::vpn::stack::NetworkConfig; +use ya_utils_networking::vpn::stack::StackConfig; use ya_utils_networking::vpn::{ EtherFrame, EtherType, IpPacket, PeekPacket, SocketEndpoint, TcpPacket, UdpPacket, }; use crate::manifest::UrlValidator; use crate::message::Shutdown; -use crate::network; -use crate::network::{Endpoint, RxBuffer}; +use crate::network::{write_prefix, Endpoint, RxBuffer}; use crate::{Error, Result}; -const IP4_ADDRESS: std::net::Ipv4Addr = std::net::Ipv4Addr::new(9, 0, 0x0d, 0x01); -const IP6_ADDRESS: std::net::Ipv6Addr = IP4_ADDRESS.to_ipv6_mapped(); +const IP4_ADDRESS: Ipv4Addr = Ipv4Addr::new(9, 0, 0x0d, 0x01); +const IP6_ADDRESS: Ipv6Addr = IP4_ADDRESS.to_ipv6_mapped(); const TCP_KEEP_ALIVE: Duration = Duration::from_secs(30); +const DEFAULT_MAX_PACKET_SIZE: usize = 65536; const DEFAULT_PREFIX_LEN: u8 = 24; type TcpSender = Arc, Bytes>>>; @@ -70,7 +70,7 @@ pub(crate) async fn start_inet( let ip4_net = ipnet::Ipv4Net::new(IP4_ADDRESS, DEFAULT_PREFIX_LEN).unwrap(); // let ip6_net = ipnet::Ipv6Net::new(IP6_ADDRESS, 128 - DEFAULT_PREFIX_LEN).unwrap(); - let ip4_addr = ip4_net.hosts().skip(1).next().unwrap(); + let ip4_addr = ip4_net.hosts().nth(1).unwrap(); // let ip6_addr = ip6_net.hosts().skip(1).next().unwrap(); let networks = [ @@ -124,9 +124,9 @@ impl Inet { } fn create_network() -> net::Network { - let config = Rc::new(NetworkConfig { - max_transmission_unit: DEFAULT_MAX_FRAME_SIZE, - buffer_size_multiplier: 4, + let config = Rc::new(StackConfig { + max_transmission_unit: DEFAULT_MAX_PACKET_SIZE, + ..Default::default() }); let ethernet_addr = loop { @@ -271,15 +271,15 @@ async fn inet_ingress_handler(rx: IngressReceiver, proxy: Proxy) { async fn inet_egress_handler( rx: EgressReceiver, - mut fwd: impl Sink>, Error = E> + Unpin + 'static, + fwd: tokio::sync::mpsc::UnboundedSender, E>>, ) { let mut rx = UnboundedReceiverStream::new(rx); while let Some(event) = rx.next().await { let mut frame = event.payload.into_vec(); log::debug!("[inet] egress -> runtime packet {} B", frame.len()); - network::write_prefix(&mut frame); - if let Err(e) = fwd.send(Ok(frame)).await { + write_prefix(&mut frame); + if let Err(e) = fwd.send(Ok(frame)) { log::debug!("[inet] egress -> runtime error: {}", e); } } @@ -336,12 +336,12 @@ fn ip_packet_to_socket_desc(ip_packet: &IpPacket) -> Result { let (sender_port, listen_port) = match protocol { Protocol::Tcp => { - let _ = TcpPacket::peek(ip_packet.payload())?; + TcpPacket::peek(ip_packet.payload())?; let pkt = TcpPacket::packet(ip_packet.payload()); (pkt.src_port(), pkt.dst_port()) } Protocol::Udp => { - let _ = UdpPacket::peek(ip_packet.payload())?; + UdpPacket::peek(ip_packet.payload())?; let pkt = UdpPacket::packet(ip_packet.payload()); (pkt.src_port(), pkt.dst_port()) } @@ -402,12 +402,12 @@ impl Proxy { async fn exists(&self, key: &TransportKey) -> bool { let state = self.state.read().await; - state.remotes.contains_key(&key) + state.remotes.contains_key(key) } async fn get(&self, key: &TransportKey) -> Option { let state = self.state.read().await; - state.remotes.get(&key).cloned() + state.remotes.get(key).cloned() } async fn bind(&self, desc: SocketDesc) -> Result + 'static> { @@ -484,15 +484,8 @@ impl Proxy { conn ); - match network.send(vec, conn.clone()) { - Ok(fut) => { - if let Err(e) = fut.await { - log::debug!("[inet] proxy conn: forward error: {}", e); - } - } - Err(e) => { - log::debug!("[inet] proxy conn: send error: {}", e); - } + if let Err(e) = network.send(vec, conn).await { + log::debug!("[inet] proxy conn: send error: {}", e); }; } @@ -526,7 +519,7 @@ async fn inet_tcp_proxy<'a>(ip: IpAddr, port: u16) -> Result<(TransportSender, T .map_err(|e| Error::Other(e.to_string()))?; let _ = tcp_stream.set_nodelay(true); - let stream = Framed::with_capacity(tcp_stream, BytesCodec::new(), DEFAULT_MAX_FRAME_SIZE); + let stream = Framed::with_capacity(tcp_stream, BytesCodec::new(), DEFAULT_MAX_PACKET_SIZE); let (tx, rx) = stream.split(); Ok(( TransportSender::Tcp(Arc::new(Mutex::new(tx))), @@ -603,10 +596,10 @@ fn bind_local_address( ) -> anyhow::Result<()> { match (*dst_addr, local_addr_ipv4, local_addr_ipv6) { (SocketAddr::V4(_), Some(addr), _) => { - socket.bind(&SocketAddr::new(addr.clone().into(), 0).into())?; + socket.bind(&SocketAddr::new((*addr).into(), 0).into())?; } (SocketAddr::V6(_), _, Some(addr)) => { - socket.bind(&SocketAddr::new(addr.clone().into(), 0).into())?; + socket.bind(&SocketAddr::new((*addr).into(), 0).into())?; } _ => { if cfg!(windows) { diff --git a/exe-unit/src/network/vpn.rs b/exe-unit/src/network/vpn.rs index b15561468d..66aeb64835 100644 --- a/exe-unit/src/network/vpn.rs +++ b/exe-unit/src/network/vpn.rs @@ -1,14 +1,14 @@ use std::convert::TryFrom; -use std::ops::Not; use actix::prelude::*; -use futures::{future, FutureExt, SinkExt, TryFutureExt}; +use futures::{future, FutureExt}; -use ya_core_model::activity; -use ya_core_model::activity::{RpcMessageError, VpnControl, VpnPacket}; +use ya_client_model::NodeId; +use ya_core_model::activity::{self, RpcMessageError, VpnControl, VpnPacket}; +use ya_core_model::identity; use ya_runtime_api::server::{CreateNetwork, NetworkInterface, RuntimeService}; use ya_service_bus::typed::Endpoint as GsbEndpoint; -use ya_service_bus::{actix_rpc, typed, RpcEnvelope}; +use ya_service_bus::{actix_rpc, typed, RpcEndpoint, RpcEnvelope, RpcRawCall}; use ya_utils_networking::vpn::network::DuoEndpoint; use ya_utils_networking::vpn::{common::ntoh, Error as NetError, PeekPacket}; use ya_utils_networking::vpn::{ArpField, ArpPacket, EtherFrame, EtherType, IpPacket, Networks}; @@ -31,6 +31,13 @@ pub(crate) async fn start_vpn( log::info!("Starting VPN service..."); + let node_id = typed::service(identity::BUS_ID) + .send(identity::Get::ByDefault) + .await? + .map_err(|e| Error::Other(format!("failed to retrieve default identity: {e}")))? + .ok_or_else(|| Error::Other("no default identity set".to_string()))? + .node_id; + let networks = deployment .networks .values() @@ -44,28 +51,34 @@ pub(crate) async fn start_vpn( interface: NetworkInterface::Vpn as i32, }) .await - .map_err(|e| Error::Other(format!("[vpn] initialization error: {:?}", e)))?; + .map_err(|e| Error::Other(format!("initialization error: {:?}", e)))?; let endpoint = match response.endpoint { Some(endpoint) => Endpoint::connect(endpoint).await?, - None => return Err(Error::Other("[vpn] endpoint already connected".into())), + None => return Err(Error::Other("endpoint already connected".into())), }; - let vpn = Vpn::try_new(acl, endpoint, deployment.clone())?; + let vpn = Vpn::try_new(node_id, acl, endpoint, deployment.clone())?; Ok(Some(vpn.start())) } pub(crate) struct Vpn { + default_id: String, // TODO: Populate & use ACL #[allow(unused)] acl: Acl, networks: Networks>, endpoint: Endpoint, - rx_buf: Option, + rx_buf: RxBuffer, } impl Vpn { - fn try_new(acl: Acl, endpoint: Endpoint, deployment: Deployment) -> crate::Result { + fn try_new( + node_id: NodeId, + acl: Acl, + endpoint: Endpoint, + deployment: Deployment, + ) -> crate::Result { let mut networks = Networks::default(); deployment @@ -82,67 +95,116 @@ impl Vpn { })?; Ok(Self { + default_id: node_id.to_string(), acl, networks, endpoint, - rx_buf: Some(Default::default()), + rx_buf: Default::default(), }) } - fn handle_ip(&mut self, frame: EtherFrame, ctx: &mut Context) { + fn handle_packet( + &mut self, + packet: Packet, + _ctx: &mut Context, + ) -> ::Result { + let network_id = packet.network_id; + let node_id = packet.caller; + let mut data = packet.data; + + // fixme: should requestor be queried for unknown IP addresses instead? + // read and add unknown node id -> ip if it doesn't exist + if let Ok(ether_type) = EtherFrame::peek_type(&data) { + let payload = EtherFrame::peek_payload(&data).unwrap(); + let ip = match ether_type { + EtherType::Arp => { + let pkt = ArpPacket::packet(payload); + ntoh(pkt.get_field(ArpField::SPA)) + } + EtherType::Ip => { + let pkt = IpPacket::packet(payload); + ntoh(pkt.src_address()) + } + _ => None, + }; + + if let Some(ip) = ip { + let _ = self.networks.get_mut(&network_id).map(|network| { + if !network.nodes().contains_key(&node_id) { + log::debug!("[vpn] adding new node: {} {}", ip, node_id); + let _ = network.add_node(ip, &node_id, network::gsb_endpoint); + } + }); + } + } + + network::write_prefix(&mut data); + + if let Err(e) = self.endpoint.tx.send(Ok(data)) { + log::debug!("[vpn] ingress error: {}", e); + } + + Ok(()) + } + + fn handle_ip( + frame: EtherFrame, + networks: &Networks>, + default_id: &str, + ) { let ip_pkt = IpPacket::packet(frame.payload()); log::trace!("[vpn] egress packet to {:?}", ip_pkt.dst_address()); if ip_pkt.is_broadcast() { - let futs = self - .networks + let futs = networks .endpoints() .into_iter() - .map(|e| e.udp.call(VpnPacket(frame.as_ref().to_vec()))) + .map(|e| e.udp.push_raw_as(default_id, frame.as_ref().to_vec())) .collect::>(); - futs.is_empty().not().then(|| { - let fut = future::join_all(futs).then(|_| future::ready(())); - ctx.spawn(fut.into_actor(self)) + tokio::task::spawn_local(async move { + future::join_all(futs).then(|_| future::ready(())).await; }); } else { let ip = ip_pkt.dst_address(); - match self.networks.endpoint(ip) { - Some(endpoint) => self.forward_frame(endpoint, frame, ctx), + match networks.endpoint(ip) { + Some(endpoint) => Self::forward_frame(endpoint, default_id, frame), None => log::debug!("[vpn] no endpoint for {ip:?}"), } } } - fn handle_arp(&mut self, frame: EtherFrame, ctx: &mut Context) { + fn handle_arp( + frame: EtherFrame, + networks: &Networks>, + default_id: &str, + ) { let arp = ArpPacket::packet(frame.payload()); // forward only IP ARP packets - if arp.get_field(ArpField::PTYPE) != [08, 00] { + if arp.get_field(ArpField::PTYPE) != [8, 0] { return; } let ip = arp.get_field(ArpField::TPA); - match self.networks.endpoint(ip) { - Some(endpoint) => self.forward_frame(endpoint, frame, ctx), + match networks.endpoint(ip) { + Some(endpoint) => Self::forward_frame(endpoint, default_id, frame), None => log::debug!("[vpn] no endpoint for {ip:?}"), } } - fn forward_frame( - &mut self, - endpoint: DuoEndpoint, - frame: EtherFrame, - ctx: &mut Context, - ) { - let pkt: Vec<_> = frame.into(); - log::trace!("[vpn] egress {} b", pkt.len()); + fn forward_frame(endpoint: DuoEndpoint, default_id: &str, frame: EtherFrame) { + let data: Vec<_> = frame.into(); + log::trace!("[vpn] egress {} b", data.len()); - endpoint + let fut = endpoint .udp - .call(VpnPacket(pkt)) - .map_err(|err| log::debug!("[vpn] call error: {err}")) - .then(|_| future::ready(())) - .into_actor(self) - .spawn(ctx); + .push_raw_as(default_id, data) + .then(|result| async move { + if let Err(err) = result { + log::debug!("[vpn] call error: {err}"); + } + }); + + tokio::task::spawn_local(fut); } } @@ -156,6 +218,8 @@ impl Actor for Vpn { let vpn_id = activity::exeunit::network_id(&net_id); actix_rpc::bind::(&vpn_id, ctx.address().recipient()); + actix_rpc::bind_raw(&format!("{vpn_id}/raw"), ctx.address().recipient()); + typed::bind_with_caller::(&vpn_id, move |caller, pkt| { actor .send(Packet { @@ -201,21 +265,20 @@ impl Actor for Vpn { /// Egress traffic handler (Runtime -> VPN) impl StreamHandler>> for Vpn { - fn handle(&mut self, result: crate::Result>, ctx: &mut Context) { + fn handle(&mut self, result: crate::Result>, _ctx: &mut Context) { let received = match result { Ok(vec) => vec, Err(err) => return log::debug!("[vpn] error (egress): {err}"), }; - let mut rx_buf = match self.rx_buf.take() { - Some(buf) => buf, - None => return log::error!("[vpn] programming error: rx buffer already taken"), - }; + + let networks = &self.networks; + let rx_buf = &mut self.rx_buf; for packet in rx_buf.process(received) { match EtherFrame::try_from(packet) { Ok(frame) => match &frame { - EtherFrame::Arp(_) => self.handle_arp(frame, ctx), - EtherFrame::Ip(_) => self.handle_ip(frame, ctx), + EtherFrame::Arp(_) => Self::handle_arp(frame, networks, &self.default_id), + EtherFrame::Ip(_) => Self::handle_ip(frame, networks, &self.default_id), frame => log::debug!("[vpn] unimplemented EtherType: {}", frame), }, Err(err) => { @@ -227,61 +290,41 @@ impl StreamHandler>> for Vpn { } }; } - - self.rx_buf.replace(rx_buf); } } /// Ingress traffic handler (VPN -> Runtime) -impl Handler for Vpn { - type Result = ::Result; - - fn handle(&mut self, packet: Packet, ctx: &mut Context) -> Self::Result { - log::trace!("[vpn] ingress {} b", packet.data.len()); - - let network_id = packet.network_id; - let node_id = packet.caller; - let data = packet.data.into_boxed_slice(); - - // fixme: should requestor be queried for unknown IP addresses instead? - // read and add unknown node id -> ip if it doesn't exist - if let Ok(ether_type) = EtherFrame::peek_type(&data) { - let payload = EtherFrame::peek_payload(&data).unwrap(); - let ip = match ether_type { - EtherType::Arp => { - let pkt = ArpPacket::packet(payload); - ntoh(pkt.get_field(ArpField::SPA)) - } - EtherType::Ip => { - let pkt = IpPacket::packet(payload); - ntoh(pkt.src_address()) +impl Handler for Vpn { + type Result = Result, ya_service_bus::Error>; + + fn handle(&mut self, msg: RpcRawCall, ctx: &mut Self::Context) -> Self::Result { + let packet = { + let mut split = msg.addr.rsplit('/').skip(1); + match split.next() { + Some(network_id) => Packet { + network_id: network_id.to_string(), + caller: msg.caller.to_string(), + data: msg.body, + }, + None => { + return Err(ya_service_bus::Error::GsbBadRequest( + "Empty network id in a RpcRawCall message".to_string(), + )) } - _ => None, - }; - - if let Some(ip) = ip { - let _ = self.networks.get_mut(&network_id).map(|network| { - if !network.nodes().contains_key(&node_id) { - log::debug!("[vpn] adding new node: {} {}", ip, node_id); - let _ = network.add_node(ip, &node_id, network::gsb_endpoint); - } - }); } - } + }; - let mut data = data.into(); - network::write_prefix(&mut data); + self.handle_packet(packet, ctx) + .map(|_| Vec::new()) + .map_err(|e| ya_service_bus::Error::GsbBadRequest(e.to_string())) + } +} - let mut tx = self.endpoint.tx.clone(); - async move { - if let Err(e) = tx.send(Ok(data)).await { - log::debug!("[vpn] ingress error: {}", e); - } - } - .into_actor(self) - .spawn(ctx); +impl Handler for Vpn { + type Result = ::Result; - Ok(()) + fn handle(&mut self, packet: Packet, ctx: &mut Context) -> Self::Result { + self.handle_packet(packet, ctx) } } diff --git a/exe-unit/src/output.rs b/exe-unit/src/output.rs index 0f7a422144..0521909bae 100644 --- a/exe-unit/src/output.rs +++ b/exe-unit/src/output.rs @@ -14,9 +14,9 @@ where let stream = FramedRead::new(read, BytesCodec::new()) .filter_map(|result| async { result.ok() }) .ready_chunks(16) - .map(|v| v.into_iter().map(|b| b.to_vec()).flatten().collect()) + .map(|v| v.into_iter().flat_map(|b| b.to_vec()).collect()) .map(f) - .map(|evt| Ok(evt)); + .map(Ok); if let Err(e) = stream.forward(tx).await { log::error!("Error forwarding output: {:?}", e); @@ -60,18 +60,16 @@ impl CapturedOutput { } else { match self.format { CaptureFormat::Bin => Some(CommandOutput::Bin(output)), - CaptureFormat::Str => vec_to_string(output).map(|s| CommandOutput::Str(s)), + CaptureFormat::Str => vec_to_string(output).map(CommandOutput::Str), } } } pub fn output_string(&self) -> Option { - self.output() - .map(|o| match o { - CommandOutput::Bin(b) => vec_to_string(b), - CommandOutput::Str(s) => Some(s), - }) - .flatten() + self.output().and_then(|o| match o { + CommandOutput::Bin(b) => vec_to_string(b), + CommandOutput::Str(s) => Some(s), + }) } pub fn write + ?Sized>(&mut self, bytes: &B) -> Option { @@ -115,14 +113,14 @@ impl From> for CapturedOutput { CapturedOutput { stream: false, - format: format.unwrap_or_else(CaptureFormat::default), + format: format.unwrap_or_default(), head, tail, } } CaptureMode::Stream { limit, format } => CapturedOutput { stream: true, - format: format.unwrap_or_else(CaptureFormat::default), + format: format.unwrap_or_default(), head: match limit { Some(limit) => CaptureBuffer::capped(limit), None => CaptureBuffer::all(), @@ -231,10 +229,10 @@ pub(crate) fn vec_to_string(vec: Vec) -> Option { return None; } let string = match String::from_utf8(vec) { - Ok(utf8) => utf8.to_owned(), + Ok(utf8) => utf8, Err(error) => error .as_bytes() - .into_iter() + .iter() .map(|&c| c as char) .collect::(), }; diff --git a/exe-unit/src/process/unix.rs b/exe-unit/src/process/unix.rs index b937fa4541..aaed9fab64 100644 --- a/exe-unit/src/process/unix.rs +++ b/exe-unit/src/process/unix.rs @@ -37,7 +37,7 @@ impl From> for SystemError { impl From for SystemError { fn from(e: io::Error) -> Self { - SystemError::Error(format!("IO error: {}", e.to_string())) + SystemError::Error(format!("IO error: {e}")) } } @@ -113,14 +113,14 @@ impl StatStub { Self::parse_stat(stat.trim()) } + #[allow(clippy::field_reassign_with_default)] fn parse_stat(stat: &str) -> Result { #[inline(always)] fn next<'l, T: std::str::FromStr, I: Iterator>( it: &mut I, ) -> Result { it.next() - .map(|s| s.parse().ok()) - .flatten() + .and_then(|s| s.parse().ok()) .ok_or_else(|| SystemError::Error("proc stat: invalid entry".into())) } @@ -261,16 +261,16 @@ pub fn getrusage(resource: i32) -> Result { let ret = unsafe { libc::getrusage(resource as i32, usage.as_mut_ptr()) }; match ret { 0 => Ok(Usage::from(unsafe { usage.assume_init() })), - _ => Err(SystemError::from(nix::Error::last()).into()), + _ => Err(SystemError::from(nix::Error::last())), } } pub async fn kill(pid: i32, timeout: i64) -> Result<(), SystemError> { fn alive(pid: Pid) -> Result { - Ok(match waitpid(pid, Some(WaitPidFlag::WNOHANG))? { - WaitStatus::Exited(_, _) | WaitStatus::Signaled(_, _, _) => false, - _ => true, - }) + Ok(!matches!( + waitpid(pid, Some(WaitPidFlag::WNOHANG))?, + WaitStatus::Exited(_, _) | WaitStatus::Signaled(_, _, _) + )) } let pid = Pid::from_raw(pid); diff --git a/exe-unit/src/process/win.rs b/exe-unit/src/process/win.rs index 9ba71b3b97..c21fa6c05b 100644 --- a/exe-unit/src/process/win.rs +++ b/exe-unit/src/process/win.rs @@ -117,7 +117,7 @@ impl JobObject { ) } == 0 { - return Err(SystemError::last().into()); + return Err(SystemError::last()); } Ok(info) @@ -136,7 +136,7 @@ impl JobObject { ) } == 0 { - return Err(SystemError::last().into()); + return Err(SystemError::last()); } Ok(info) @@ -144,7 +144,7 @@ impl JobObject { pub fn terminate(&self) -> Result<(), SystemError> { if unsafe { um::jobapi2::TerminateJobObject(self.handle, 0) } == 0 { - return Err(SystemError::last().into()); + return Err(SystemError::last()); } Ok(()) } @@ -162,7 +162,7 @@ impl JobObject { ) } == 0 { - return Err(SystemError::last().into()); + return Err(SystemError::last()); } Ok(()) } @@ -170,10 +170,10 @@ impl JobObject { fn create_job(proc: HANDLE) -> Result { let handle = unsafe { um::jobapi2::CreateJobObjectW(ptr::null_mut(), ptr::null()) }; if handle.is_null() { - return Err(SystemError::NullPointer(format!("JobObject handle")).into()); + return Err(SystemError::NullPointer("JobObject handle".to_string())); } if unsafe { um::jobapi2::AssignProcessToJobObject(handle, proc) } == 0 { - return Err(SystemError::last().into()); + return Err(SystemError::last()); } Ok(handle) @@ -204,7 +204,7 @@ fn process_handle(pid: Option) -> Result { ) }; if handle.is_null() { - return Err(SystemError::NullPointer(format!("process {} handle", pid)).into()); + return Err(SystemError::NullPointer(format!("process {} handle", pid))); } Ok(handle) } diff --git a/exe-unit/src/runtime/event.rs b/exe-unit/src/runtime/event.rs index 3b008e9b04..addf4741ea 100644 --- a/exe-unit/src/runtime/event.rs +++ b/exe-unit/src/runtime/event.rs @@ -30,7 +30,7 @@ struct Inner { impl EventMonitor { pub fn any_process<'a>(&mut self, ctx: CommandContext) -> Handle<'a> { let mut inner = self.inner.lock().unwrap(); - inner.fallback.replace(Channel::fallback(ctx.clone())); + inner.fallback.replace(Channel::fallback(ctx)); Handle::Fallback {} } @@ -38,7 +38,7 @@ impl EventMonitor { pub fn next_process<'a>(&mut self, ctx: CommandContext) -> Handle<'a> { let mut inner = self.inner.lock().unwrap(); let channel = Channel::new(ctx, Default::default()); - let handle = Handle::process(&self, &channel); + let handle = Handle::process(self, &channel); inner.next_process.replace(channel); handle @@ -48,7 +48,7 @@ impl EventMonitor { pub fn process<'a>(&mut self, ctx: CommandContext, pid: u64) -> Handle<'a> { let mut inner = self.inner.lock().unwrap(); let channel = Channel::new(ctx, pid); - let handle = Handle::process(&self, &channel); + let handle = Handle::process(self, &channel); inner.processes.insert(pid, channel); handle @@ -61,6 +61,7 @@ impl ya_runtime_api::server::RuntimeHandler for EventMonitor { let (mut ctx, done_tx) = { let mut inner = self.inner.lock().unwrap(); + #[allow(clippy::map_entry)] if !inner.processes.contains_key(&status.pid) { if let Some(channel) = inner.next_process.take() { channel.waker.lock().unwrap().pid.replace(status.pid); @@ -168,16 +169,11 @@ impl<'a> Handle<'a> { impl<'a> Drop for Handle<'a> { fn drop(&mut self) { - match self { - Handle::Process { monitor, waker, .. } => { - if let Some(pid) = { waker.lock().unwrap().pid } { - let mut inner = monitor.inner.lock().unwrap(); - inner.processes.remove(&pid); - inner.next_process.take(); - } - } - _ => { - // ignore + if let Handle::Process { monitor, waker, .. } = self { + if let Some(pid) = { waker.lock().unwrap().pid } { + let mut inner = monitor.inner.lock().unwrap(); + inner.processes.remove(&pid); + inner.next_process.take(); } } } @@ -245,7 +241,7 @@ impl Channel { } fn done_tx(&mut self) -> Option> { - self.done.as_mut().map(|d| d.tx.take()).flatten() + self.done.as_mut().and_then(|d| d.tx.take()) } fn done_rx<'a>(&self) -> Option>> { diff --git a/exe-unit/src/runtime/process.rs b/exe-unit/src/runtime/process.rs index e3916e4fa8..5f7504cda4 100644 --- a/exe-unit/src/runtime/process.rs +++ b/exe-unit/src/runtime/process.rs @@ -91,7 +91,7 @@ impl RuntimeProcess { let result = child.wait_with_output()?; match result.status.success() { true => { - let stdout = vec_to_string(result.stdout).unwrap_or_else(String::new); + let stdout = vec_to_string(result.stdout).unwrap_or_default(); Ok(serde_json::from_str(&stdout).map_err(|e| { let msg = format!("Invalid offer template [{}]: {:?}", binary.display(), e); Error::Other(msg) @@ -330,9 +330,11 @@ impl RuntimeProcess { .ok_or_else(|| Error::runtime("Invalid binary name"))?; args.insert(0, name.to_string_lossy().to_string()); - let mut run_process = RunProcess::default(); - run_process.bin = entry_point; - run_process.args = args; + let run_process = RunProcess { + bin: entry_point, + args, + ..Default::default() + }; let handle = monitor.next_process(ctx); if let Err(error) = service.run_process(run_process).await { @@ -454,7 +456,7 @@ impl Handler for RuntimeProcess { let proc = self.service.take(); let vpn = self.vpn.take(); let inet = self.inet.take(); - let mut children = std::mem::replace(&mut self.children, HashSet::new()); + let mut children = std::mem::take(&mut self.children); log::info!("Shutting down the runtime process: {:?}", msg.0); @@ -535,10 +537,7 @@ struct ChildProcessGuard { impl ChildProcessGuard { fn new(inner: ChildProcess, addr: Addr) -> Self { addr.do_send(AddChildProcess(inner.clone())); - ChildProcessGuard { - inner, - addr: addr.clone(), - } + ChildProcessGuard { inner, addr } } } diff --git a/exe-unit/src/service/metrics.rs b/exe-unit/src/service/metrics.rs index 8aacc0508d..0e2e2bebfa 100644 --- a/exe-unit/src/service/metrics.rs +++ b/exe-unit/src/service/metrics.rs @@ -29,7 +29,7 @@ impl MetricsService { _ => None, }; - let mut metrics = Self::metrics(&ctx, backlog_limit, caps); + let mut metrics = Self::metrics(ctx, backlog_limit, caps); let mut custom_metrics = ctx .agreement .usage_vector @@ -146,7 +146,7 @@ impl Handler for MetricsService { let metric = self .metrics .get_mut(name) - .ok_or(MetricError::Unsupported(name.to_string()))?; + .ok_or_else(|| MetricError::Unsupported(name.to_string()))?; let report = metric.report(); metric.log_report(report.clone()); @@ -201,6 +201,7 @@ impl Metric for CustomMetric { } } +#[allow(clippy::type_complexity)] struct MetricProvider { metric: Box, backlog: Arc, MetricReport)>>>, diff --git a/exe-unit/src/service/signal.rs b/exe-unit/src/service/signal.rs index 0ca71b47e3..681854f726 100644 --- a/exe-unit/src/service/signal.rs +++ b/exe-unit/src/service/signal.rs @@ -59,11 +59,9 @@ where } fn stopped(&mut self, _: &mut Self::Context) { - std::mem::replace(&mut self.signals, Vec::new()) - .into_iter() - .for_each(|s| { - unregister(s); - }); + std::mem::take(&mut self.signals).into_iter().for_each(|s| { + unregister(s); + }); log::debug!("Signal monitoring service stopped"); } diff --git a/exe-unit/src/service/transfer.rs b/exe-unit/src/service/transfer.rs index e3e0416914..d2679c9b61 100644 --- a/exe-unit/src/service/transfer.rs +++ b/exe-unit/src/service/transfer.rs @@ -64,7 +64,7 @@ impl ContainerTransferProvider { fn resolve_path(&self, container_path: &str) -> std::result::Result { fn is_prefix_of(base: &str, path: &str) -> usize { - if path.starts_with(base) && (path == base || path[base.len()..].starts_with("/")) { + if path.starts_with(base) && (path == base || path[base.len()..].starts_with('/')) { base.len() + 1 } else { 0 @@ -85,7 +85,7 @@ impl ContainerTransferProvider { } let path = &container_path[c.path.len() + 1..]; - if path.starts_with("/") { + if path.starts_with('/') { return Err(TransferError::IoError(io::Error::new( io::ErrorKind::NotFound, anyhow::anyhow!("invalid path format: [{}]", container_path), @@ -167,8 +167,7 @@ impl TransferService { pub fn schemes() -> Vec { Self::default_providers() .values() - .map(|p| p.schemes()) - .flatten() + .flat_map(|p| p.schemes()) .collect::>() .into_iter() .map(ToString::to_string) @@ -199,7 +198,7 @@ impl TransferService { Ok(self .providers .get(scheme) - .ok_or(TransferError::UnsupportedSchemeError(scheme.to_owned()))? + .ok_or_else(|| TransferError::UnsupportedSchemeError(scheme.to_owned()))? .clone()) } } @@ -288,7 +287,7 @@ impl Handler for TransferService { Ok(Some(path)) }; - return ActorResponse::r#async(fut.into_actor(self)); + ActorResponse::r#async(fut.into_actor(self)) } #[cfg(feature = "sgx")] @@ -304,7 +303,7 @@ impl Handler for TransferService { std::fs::write(&path, bytes)?; Ok(Some(path)) }; - return ActorResponse::r#async(fut.into_actor(self)); + ActorResponse::r#async(fut.into_actor(self)) } } } @@ -363,7 +362,7 @@ impl Handler for TransferService { fn handle(&mut self, _: AbortTransfers, _: &mut Self::Context) -> Self::Result { { let mut guard = self.abort_handles.borrow_mut(); - std::mem::replace(&mut (*guard), Default::default()) + std::mem::take(&mut (*guard)) } .into_iter() .for_each(|h| h.abort()); diff --git a/exe-unit/src/state.rs b/exe-unit/src/state.rs index 3c014bd01a..2ccee7326e 100644 --- a/exe-unit/src/state.rs +++ b/exe-unit/src/state.rs @@ -27,12 +27,14 @@ use crate::runtime::RuntimeMode; fn invalid_state_err_msg(state_pair: &StatePair) -> String { match state_pair { StatePair(State::Initialized, None) => { - format!("Activity is initialized - deploy() command is expected now") + "Activity is initialized - deploy() command is expected now".to_string() } StatePair(State::Deployed, None) => { - format!("Activity is deployed - start() command is expected now") + "Activity is deployed - start() command is expected now".to_string() + } + StatePair(State::Ready, None) => { + "Cannot send command after a successful start()".to_string() } - StatePair(State::Ready, None) => format!("Cannot send command after a successful start()"), _ => format!( "This command is not allowed when activity is in the {:?} state", state_pair.0 @@ -60,6 +62,7 @@ pub struct Supervision { pub manifest: ManifestContext, } +#[derive(Default)] pub(crate) struct ExeUnitState { pub inner: StatePair, pub last_batch: Option, @@ -88,16 +91,6 @@ impl ExeUnitState { } } -impl Default for ExeUnitState { - fn default() -> Self { - ExeUnitState { - inner: Default::default(), - batches: Default::default(), - last_batch: None, - } - } -} - pub(crate) struct Batch { pub exec: Exec, pub results: Vec, @@ -191,8 +184,7 @@ impl Batch { .results .iter() .enumerate() - .filter(|(_, s)| s.result.is_none()) - .next() + .find(|(_, s)| s.result.is_none()) .map(|(i, s)| (i, s.message.clone())); match result { @@ -206,13 +198,13 @@ impl Batch { } pub fn results(&self, cmd_idx: Option) -> Vec { - let last_idx = cmd_idx.unwrap_or_else(|| self.exec.exe_script.len() - 1); + let last_idx = cmd_idx.unwrap_or(self.exec.exe_script.len() - 1); self.results .iter() .enumerate() .take_while(|(idx, s)| *idx <= last_idx && s.result.is_some()) .map(|(idx, s)| { - let result = s.result.clone().unwrap(); + let result = s.result.unwrap(); let output = cmd_idx.as_ref().map(|i| *i == idx).unwrap_or(true); ExeScriptCommandResult { index: idx as u32, @@ -289,7 +281,7 @@ impl Broadcast { fn initialize(&mut self) { let (tx, rx) = broadcast::channel(16); let receiver = tokio_stream::wrappers::BroadcastStream::new(rx); - tokio::task::spawn_local(receiver.for_each(|_| async { () })); + tokio::task::spawn_local(receiver.for_each(|_| async {})); self.sender = Some(tx); } } @@ -330,7 +322,7 @@ impl CommandState { #[allow(dead_code)] pub fn repr(&self) -> CommandStateRepr { CommandStateRepr { - result: self.result.clone(), + result: self.result, stdout: self.stdout.output_string(), stderr: self.stderr.output_string(), message: self.message.clone(), diff --git a/exe-unit/src/util/cache.rs b/exe-unit/src/util/cache.rs index c90728bae7..ed0dad09f4 100644 --- a/exe-unit/src/util/cache.rs +++ b/exe-unit/src/util/cache.rs @@ -14,18 +14,16 @@ pub(crate) struct Cache { impl Cache { pub fn new(dir: PathBuf) -> Self { - let tmp_dir = dir.clone().join("tmp"); - std::fs::create_dir_all(&tmp_dir).expect(&format!( - "Unable to create directory: {}", - tmp_dir.display() - )); + let tmp_dir = dir.join("tmp"); + std::fs::create_dir_all(&tmp_dir) + .unwrap_or_else(|_| panic!("Unable to create directory: {}", tmp_dir.display())); Cache { dir, tmp_dir } } pub fn name(transfer_url: &TransferUrl) -> Result { let hash = match &transfer_url.hash { Some(hash) => hash, - None => return Err(TransferError::InvalidUrlError("hash required".to_owned()).into()), + None => return Err(TransferError::InvalidUrlError("hash required".to_owned())), }; let name = transfer_url.file_name()?; @@ -55,12 +53,9 @@ impl TryFrom for TransferUrl { fn try_from(value: ProjectedPath) -> Result { TransferUrl::parse( - value - .to_path_buf() - .to_str() - .ok_or(Error::local(TransferError::InvalidUrlError( - "Invalid path".to_owned(), - )))?, + value.to_path_buf().to_str().ok_or_else(|| { + Error::local(TransferError::InvalidUrlError("Invalid path".to_owned())) + })?, "file", ) .map_err(Error::local) @@ -199,11 +194,7 @@ fn flatten_container_path(path: PathBuf) -> PathBuf { /// Remove the root dir and all prefixes from a path. Specific to the custom "container" scheme. fn remove_container_path_base(path: PathBuf) -> PathBuf { path.components() - .filter(|c| match c { - Component::RootDir => false, - Component::Prefix(_) => false, - _ => true, - }) + .filter(|c| !matches!(c, Component::RootDir | Component::Prefix(_))) .collect::() } diff --git a/exe-unit/tokio-process-ns/Cargo.toml b/exe-unit/tokio-process-ns/Cargo.toml index bb511821b8..eed3adf10d 100644 --- a/exe-unit/tokio-process-ns/Cargo.toml +++ b/exe-unit/tokio-process-ns/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tokio-process-ns" -version = "0.1.0" +version = "0.2.0" authors = ["Golem Factory "] edition = "2018" diff --git a/golem_cli/Cargo.toml b/golem_cli/Cargo.toml index 3919090196..faeb44212a 100644 --- a/golem_cli/Cargo.toml +++ b/golem_cli/Cargo.toml @@ -1,17 +1,17 @@ [package] name = "golemsp" description = "User friedly CLI for running Golem Provider" -version = "0.2.0" +version = "0.3.0" authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-client = { version = "0.6", features = ['cli'] } +ya-client = { version = "0.7", features = ['cli'] } ya-compile-time-utils = "0.2" -ya-core-model = { version = "^0.6", features=["payment", "version"] } -ya-provider = "0.2" +ya-core-model = { version = "^0.8", features=["payment", "version"] } +ya-provider = "0.3" ya-utils-path = "0.1.0" -ya-utils-process = { version = "0.1.0", features = ["lock"] } +ya-utils-process = { version = "0.2", features = ["lock"] } actix-rt="2.7" ansi_term="0.12.1" @@ -34,8 +34,8 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" strip-ansi-escapes = "0.1" structopt = "0.3" -strum = "0.20.0" -strum_macros = "0.20.0" +strum = "0.24" +strum_macros = "0.24" tokio = { version = "1", features = ["process", "signal", "time", "io-util", "io-std", "macros"] } url = "2.1" diff --git a/golem_cli/src/command/provider.rs b/golem_cli/src/command/provider.rs index 76f174876e..d73e1f67f2 100644 --- a/golem_cli/src/command/provider.rs +++ b/golem_cli/src/command/provider.rs @@ -8,7 +8,7 @@ pub use ya_provider::GlobalsState as ProviderConfig; use crate::command::{NetworkGroup, NETWORK_GROUP_MAP}; use crate::setup::RunConfig; -const CLASSIC_RUNTIMES: &'static [&'static str] = &["wasmtime", "vm"]; +const CLASSIC_RUNTIMES: &[&str] = &["wasmtime", "vm"]; pub struct YaProviderCommand { pub(super) cmd: Command, @@ -62,7 +62,7 @@ impl YaProviderCommand { if let Some(account) = &config.account { cmd.args(&["--account", &account.to_string()]); } - for n in NETWORK_GROUP_MAP[&network_group].iter() { + for n in NETWORK_GROUP_MAP[network_group].iter() { cmd.args(&["--payment-network", &n.to_string()]); } @@ -207,9 +207,9 @@ impl YaProviderCommand { exeunit_name: &str, usage_coeffs: &UsageDef, ) -> anyhow::Result<()> { - let mut cmd = &mut self.cmd; + let cmd = &mut self.cmd; cmd.args(&["preset", "update", "--no-interactive"]); - preset_command(&mut cmd, name, exeunit_name, usage_coeffs); + preset_command(cmd, name, exeunit_name, usage_coeffs); cmd.arg("--").arg(name); self.exec_no_output() .await diff --git a/golem_cli/src/command/yagna.rs b/golem_cli/src/command/yagna.rs index 68f94eba2e..31fac12d9b 100644 --- a/golem_cli/src/command/yagna.rs +++ b/golem_cli/src/command/yagna.rs @@ -21,7 +21,10 @@ pub struct PaymentPlatform { pub token: &'static str, } -pub struct PaymentDriver(pub HashMap<&'static str, PaymentPlatform>); +pub struct PaymentDriver { + pub platforms: HashMap<&'static str, PaymentPlatform>, + pub name: &'static str, +} lazy_static! { pub static ref ZKSYNC_DRIVER: PaymentDriver = { @@ -34,15 +37,19 @@ lazy_static! { token: "GLM", }, ); - zksync.insert( - NetworkName::Rinkeby.into(), - PaymentPlatform { - platform: "zksync-rinkeby-tglm", - driver: "zksync", - token: "tGLM", - }, - ); - PaymentDriver(zksync) + // zksync.insert( + // NetworkName::Rinkeby.into(), + // PaymentPlatform { + // platform: "zksync-rinkeby-tglm", + // driver: "zksync", + // token: "tGLM", + // }, + // ); + + PaymentDriver { + platforms: zksync, + name: "zksync", + } }; pub static ref ERC20_DRIVER: PaymentDriver = { let mut erc20 = HashMap::new(); @@ -86,17 +93,33 @@ lazy_static! { token: "GLM", }, ); - PaymentDriver(erc20) + + PaymentDriver { + platforms: erc20, + name: "erc20", + } }; + pub static ref DRIVERS: Vec<&'static PaymentDriver> = vec![&ZKSYNC_DRIVER, &ERC20_DRIVER]; } impl PaymentDriver { pub fn platform(&self, network: &NetworkName) -> anyhow::Result<&PaymentPlatform> { let net: &str = network.into(); - Ok(self.0.get(net).ok_or(anyhow!( - "Payment driver config for network '{}' not found.", - network - ))?) + self.platforms + .get(net) + .ok_or_else(|| anyhow!("Payment driver config for network '{}' not found.", network)) + } + + pub fn status_label(&self, network: &NetworkName) -> String { + if self.name == ZKSYNC_DRIVER.name { + return "zksync".to_string(); + } + + if network == &NetworkName::Mainnet { + "on-chain".to_string() + } else { + network.to_string().to_lowercase() + } } } @@ -342,7 +365,7 @@ impl YagnaCommand { Ok(_) => Ok(child), Err(e) => { log::error!("Killing Golem Service, since wait failed: {}", e); - let _ = child.kill().await?; + child.kill().await?; Err(e) } } diff --git a/golem_cli/src/main.rs b/golem_cli/src/main.rs index 9b2c1848f7..e1c8b7c707 100644 --- a/golem_cli/src/main.rs +++ b/golem_cli/src/main.rs @@ -92,7 +92,7 @@ async fn my_main() -> Result { let cli_args: StartupConfig = StartupConfig::from_args(); match cli_args.commands { - Commands::Setup(mut run_config) => setup::setup(&mut run_config, true).await, + Commands::Setup(run_config) => setup::setup(&run_config, true).await, Commands::Run(run_config) => service::run(run_config).await, Commands::Stop => service::stop().await, Commands::Settings(command) => match command { diff --git a/golem_cli/src/platform.rs b/golem_cli/src/platform.rs index 685e1e9868..e0c72805ad 100644 --- a/golem_cli/src/platform.rs +++ b/golem_cli/src/platform.rs @@ -2,7 +2,7 @@ use std::borrow::Cow; use std::ops::Not; #[allow(dead_code)] -#[derive(PartialEq)] +#[derive(PartialEq, Eq)] pub enum Status { Valid, Permission(Cow<'static, str>), diff --git a/golem_cli/src/service.rs b/golem_cli/src/service.rs index 112671e612..e20b892797 100644 --- a/golem_cli/src/service.rs +++ b/golem_cli/src/service.rs @@ -1,5 +1,5 @@ use crate::appkey; -use crate::command::{YaCommand, ERC20_DRIVER, NETWORK_GROUP_MAP, ZKSYNC_DRIVER}; +use crate::command::{YaCommand, DRIVERS, NETWORK_GROUP_MAP}; use crate::setup::RunConfig; use crate::utils::payment_account; use anyhow::{Context, Result}; @@ -111,8 +111,8 @@ pub async fn watch_for_vm() -> anyhow::Result<()> { } } -pub async fn run(mut config: RunConfig) -> Result { - crate::setup::setup(&mut config, false).await?; +pub async fn run(config: RunConfig) -> Result { + crate::setup::setup(&config, false).await?; let cmd = YaCommand::new()?; let service = cmd.yagna()?.service_run(&config).await?; @@ -122,19 +122,15 @@ pub async fn run(mut config: RunConfig) -> Result { let address = payment_account(&cmd, &config.account.account.or(provider_config.account)).await?; for nn in NETWORK_GROUP_MAP[&config.account.network].iter() { - cmd.yagna()? - .payment_init(&address, &nn, &ERC20_DRIVER) - .await?; - if ZKSYNC_DRIVER.platform(&nn).is_err() { - continue; + for driver in DRIVERS.iter() { + if driver.platform(nn).is_err() { + continue; + } + + if let Err(e) = cmd.yagna()?.payment_init(&address, nn, driver).await { + log::debug!("Failed to initialize {} driver. Error: {e}", driver.name); + } } - if let Err(e) = cmd - .yagna()? - .payment_init(&address, &nn, &ZKSYNC_DRIVER) - .await - { - log::debug!("Failed to initialize zkSync driver. e:{}", e); - }; } let provider = cmd.ya_provider()?.spawn(&app_key, &config).await?; @@ -162,9 +158,11 @@ pub async fn run(mut config: RunConfig) -> Result { if let Err(e) = provider.abort().await { log::warn!("provider exited with: {:?}", e); + return Ok(11); } if let Err(e) = service.abort().await { log::warn!("service exited with: {:?}", e); + return Ok(12); } Ok(0) } diff --git a/golem_cli/src/settings_show.rs b/golem_cli/src/settings_show.rs index 6dd551657a..2a821eeb9b 100644 --- a/golem_cli/src/settings_show.rs +++ b/golem_cli/src/settings_show.rs @@ -36,9 +36,9 @@ async fn get_resources() -> Result { let active_profile = move_string_out_of_json(active_profile).ok_or_else(|| anyhow!("Invalid format"))?; - Ok(profiles + profiles .remove(&active_profile) - .ok_or_else(|| anyhow!("Active profile not found???"))?) + .ok_or_else(|| anyhow!("Active profile not found???")) } pub async fn show_resources() -> Result<()> { @@ -87,7 +87,7 @@ pub async fn show_prices(cmd: &YaCommand) -> Result<()> { for (preset_name, prices) in presets_prices { println!("\n\nPricing for preset \"{}\":\n", preset_name); for (price_name, price_value) in prices { - let default: (&str, f64) = (&price_name.as_str(), 1.0); + let default: (&str, f64) = (price_name.as_str(), 1.0); let (price_desc, price_multiplier) = price_description .get(&price_name.as_str()) .unwrap_or(&default); diff --git a/golem_cli/src/setup.rs b/golem_cli/src/setup.rs index fbc4834763..5deabded63 100644 --- a/golem_cli/src/setup.rs +++ b/golem_cli/src/setup.rs @@ -81,7 +81,7 @@ pub async fn setup(run_config: &RunConfig, force: bool) -> Result { let account_msg = &config .account .map(|n| n.to_string()) - .unwrap_or("Internal Golem wallet".into()); + .unwrap_or_else(|| "Internal Golem wallet".into()); let message = format!( "Ethereum {} wallet address (default={})", run_config.account.network, account_msg diff --git a/golem_cli/src/status.rs b/golem_cli/src/status.rs index c12f58125a..5d3b0ef142 100644 --- a/golem_cli/src/status.rs +++ b/golem_cli/src/status.rs @@ -12,7 +12,8 @@ use ya_core_model::NodeId; use crate::appkey; use crate::command::{ - NetworkGroup, PaymentSummary, YaCommand, ERC20_DRIVER, NETWORK_GROUP_MAP, ZKSYNC_DRIVER, + NetworkGroup, PaymentSummary, YaCommand, DRIVERS, ERC20_DRIVER, NETWORK_GROUP_MAP, + ZKSYNC_DRIVER, }; use crate::platform::Status as KvmStatus; use crate::utils::{is_yagna_running, payment_account}; @@ -22,7 +23,7 @@ async fn payment_status( network: &NetworkName, account: &Option, ) -> anyhow::Result> { - let address = payment_account(&cmd, account).await?; + let address = payment_account(cmd, account).await?; let network_group = get_network_group(network); @@ -31,22 +32,17 @@ async fn payment_status( let mut f = vec![]; let mut l = vec![]; for nn in NETWORK_GROUP_MAP[&network_group].iter() { - if let Ok(_) = ZKSYNC_DRIVER.platform(&nn) { - l.push("zksync".to_string()); - f.push(cmd.yagna()?.payment_status(&address, nn, &ZKSYNC_DRIVER)); + for driver in DRIVERS.iter() { + if driver.platform(nn).is_ok() { + l.push(driver.status_label(nn)); + f.push(cmd.yagna()?.payment_status(&address, nn, driver)); + } } - if nn == &NetworkName::Mainnet { - l.push("on-chain".to_string()); - } else { - l.push(nn.to_string().to_lowercase()); - }; - f.push(cmd.yagna()?.payment_status(&address, nn, &ERC20_DRIVER)); } (f, l) }; let fr = future::join_all(futures).await; - let mut n = 0; - for r in fr { + for (n, r) in fr.into_iter().enumerate() { result.insert( labels[n].clone(), r.unwrap_or_else(|e| { @@ -54,7 +50,6 @@ async fn payment_status( StatusResult::default() }), ); - n += 1; } Ok(result) } @@ -68,7 +63,7 @@ fn get_network_group(network: &NetworkName) -> NetworkGroup { } pub async fn run() -> Result { - let size = crossterm::terminal::size().ok().unwrap_or_else(|| (80, 50)); + let size = crossterm::terminal::size().ok().unwrap_or((80, 50)); let cmd = YaCommand::new()?; let kvm_status = crate::platform::kvm_status(); @@ -244,9 +239,10 @@ async fn get_payment_network() -> Result<(usize, NetworkName)> { ya_client::web::WebClient::with_token(&app_key).interface()?; let offers = mkt_api.get_offers().await?; - let latest_offer = offers.iter().max_by_key(|o| o.timestamp).ok_or(anyhow!( - "Provider is not functioning properly. No offers Subscribed." - ))?; + let latest_offer = offers + .iter() + .max_by_key(|o| o.timestamp) + .ok_or_else(|| anyhow!("Provider is not functioning properly. No offers Subscribed."))?; let mut network = None; for net in NetworkName::VARIANTS { let net_to_check = net.parse()?; @@ -260,8 +256,8 @@ async fn get_payment_network() -> Result<(usize, NetworkName)> { }; } - let network = network.ok_or(anyhow!( - "Unable to determine payment network used by the Yagna Provider." - ))?; + let network = network.ok_or_else(|| { + anyhow!("Unable to determine payment network used by the Yagna Provider.") + })?; Ok((offers.len(), network)) } diff --git a/golem_cli/src/terminal.rs b/golem_cli/src/terminal.rs index 269d7fef0f..532b7ff5d7 100644 --- a/golem_cli/src/terminal.rs +++ b/golem_cli/src/terminal.rs @@ -99,10 +99,10 @@ pub fn fade_in(banner: &str) -> anyhow::Result<()> { } pub async fn clear_stdin() -> anyhow::Result<()> { - let _ = crossterm::terminal::enable_raw_mode()?; + crossterm::terminal::enable_raw_mode()?; while crossterm::event::poll(Duration::from_millis(100))? { let _ = crossterm::event::read()?; } - let _ = crossterm::terminal::disable_raw_mode()?; + crossterm::terminal::disable_raw_mode()?; Ok(()) } diff --git a/goth_tests/domain/payments/test_payment_validate_allocations.py b/goth_tests/domain/payments/test_payment_validate_allocations.py new file mode 100644 index 0000000000..7f9490e7e8 --- /dev/null +++ b/goth_tests/domain/payments/test_payment_validate_allocations.py @@ -0,0 +1,64 @@ +"""Tests payment validate allocations""" + +import logging +from pathlib import Path +from typing import List +from datetime import datetime, timezone + +import pytest +from ya_payment.exceptions import ApiException +from ya_payment.models import Allocation + +from goth.address import ( + PROXY_HOST, + YAGNA_REST_URL, +) +from goth.configuration import load_yaml, Override +from goth.runner import Runner +from goth.runner.probe import RequestorProbe + +logger = logging.getLogger("goth.test.payments.validate-allocations") + + +@pytest.mark.asyncio +async def test_payment_validate_allocations( + common_assets: Path, + default_config: Path, + config_overrides: List[Override], + log_dir: Path, +): + """Test just the requestor's CLI command and validate allocations, no need to set up provider.""" + + nodes = [ + {"name": "requestor", "type": "Requestor"}, + ] + config_overrides.append(("nodes", nodes)) + goth_config = load_yaml(default_config, config_overrides) + + runner = Runner( + base_log_dir=log_dir, + compose_config=goth_config.compose_config, + web_root_path=common_assets / "web-root", + ) + + async with runner(goth_config.containers): + requestor = runner.get_probes(probe_type=RequestorProbe)[0] + + # Test requestor's CLI command for allocation validation + + # Make initial payment fund 1000GLM + requestor.cli.payment_fund(payment_driver="erc20") + + # Allocation bigger than total amount should not be possible + with pytest.raises(ApiException): + await requestor.create_allocation(None, 2000) + + # Allocate 600 GLM + allocation = await requestor.create_allocation(None, 600) + + # Confirming that allocation exists + assert await requestor.get_allocation(allocation.allocation_id) + + # Allocation bigger than reserved amount should not be possible + with pytest.raises(ApiException): + await requestor.create_allocation(None, 600) diff --git a/goth_tests/e2e/vm/assets/outbound_certificate.cert b/goth_tests/e2e/vm/assets/outbound_certificate.cert new file mode 100644 index 0000000000..f77bb5a66f --- /dev/null +++ b/goth_tests/e2e/vm/assets/outbound_certificate.cert @@ -0,0 +1,29 @@ +-----BEGIN CERTIFICATE----- +MIIE6DCCA9CgAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwgYMxCzAJBgNVBAYTAlBM +MRMwEQYDVQQIDApNYWxvcG9sc2thMQ8wDQYDVQQKDAZGb28gQ28xFTATBgNVBAsM +DEZvbyBJbnRlciBIUTESMBAGA1UEAwwJRm9vIEludGVyMSMwIQYJKoZIhvcNAQkB +FhRvZmZpY2VAaW50ZXIuZm9vLmNvbTAgFw0yMjA4MTAxMjA1MjJaGA8yMTIyMDcx +NzEyMDUyMlowgY8xCzAJBgNVBAYTAkNaMRAwDgYDVQQIDAdCb2hlbWlhMQ8wDQYD +VQQHDAZQcmFndWUxEzARBgNVBAoMCkZvbyBSZXEgQ28xEzARBgNVBAsMCkZvbyBS +ZXEgSFExEDAOBgNVBAMMB0ZvbyBSZXExITAfBgkqhkiG9w0BCQEWEm9mZmljZUBy +ZXEuZm9vLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANbEu3AR +1V7cNu1z1N/CqQa7wfkRftMIwDj8Y22CIi6m87IA7smE4V4xMeAkJ0GKtO2WZInm +rNVDJPoeuqcKV+tk3wLStX8ut2Nl8u/+1CrTtrWB5Z/N6O6701vUU375DWVyrG+h +Z9zcmbLVAdi/azBuUWbqcU/NnxByJOFyWXBPMpBGKAcOJTgniQQ2NYURWh5frUo0 +w1xwgviZk596kHo1t57kkx3q9eRxygpKmnKmMgR9pYk8ZSZrOtW+c5ayEau2TmSQ +j+OA9Zu3Ec93IyS4bIbH6QChmB0I+ai8U97HLaR0UeEGeeiQZ9nKlyImOyIpuvuw +PevrGXfqUDheTcUCAwEAAaOCAVQwggFQMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEB +BAQDAgZAMDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBTZXJ2ZXIg +Q2VydGlmaWNhdGUwHQYDVR0OBBYEFKIuFI+TpSYHsit9mHqczL8a12+gMIG2BgNV +HSMEga4wgauAFPCS+f1ciYIsWn/tB8xFYkLcgSvIoYGOpIGLMIGIMQswCQYDVQQG +EwJQTDETMBEGA1UECAwKTWFsb3BvbHNrYTEPMA0GA1UEBwwGS3Jha293MQ8wDQYD +VQQKDAZGb28gQ28xEjAQBgNVBAsMCUZvbyBDbyBIUTEPMA0GA1UEAwwGRm9vIENv +MR0wGwYJKoZIhvcNAQkBFg5vZmZpY2VAZm9vLmNvbYICEAAwDgYDVR0PAQH/BAQD +AgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMA0GCSqGSIb3DQEBCwUAA4IBAQCKh/he +NHlFPyvWZZX48eCd54FFCsk4znR+oBY1aW1jw+7AFqKxqEfecdJR+uC+s0KVcaRQ +Z6Fq50icTEK7+Tf4s4Lc//+NJ/vPB1wVk5mQwvroTSqfn9tCucE/x0kAII/4BjYz +JQCrfyrUiCmPh0MoA6wCDb6ZLCC7NQfVAn7dY4uoZ3Nl4wMoH8jRUhXhtJ670OzF +ZBsisHpJKtRibd9Cocr4twoMAMnwi/y45f8H/2VoEvsJw0CcM0+qPQejJcr1xyMP +1fLBcMLyixcP5f1/9sYTaAczNE4yMQxOCKhgfpWpa5chUSazGdgjPf4DZ/ARA+0X +02c+thLU+W1GdmHS +-----END CERTIFICATE----- diff --git a/goth_tests/e2e/vm/assets/outbound_manifest.json b/goth_tests/e2e/vm/assets/outbound_manifest.json new file mode 100644 index 0000000000..225492de54 --- /dev/null +++ b/goth_tests/e2e/vm/assets/outbound_manifest.json @@ -0,0 +1,41 @@ +{ + "version": "0.1.0", + "createdAt": "2022-07-26T12:51:00.000000Z", + "expiresAt": "2100-01-01T00:01:00.000000Z", + "metadata": { + "name": "External API call example", + "description": "Example manifest of a service making an outbound call to the external API", + "version": "0.1.0" + }, + "payload": [ + { + "platform": { + "arch": "x86_64", + "os": "linux" + }, + "urls": [ + "http://yacn2.dev.golem.network:8000/docker-golem-script-curl-latest-d75268e752.gvmi" + ], + "hash": "sha3:e5f5ddfd649525dbe25d93d9ed51d1bdd0849933d9a5720adb4b5810" + } + ], + "compManifest": { + "version": "0.1.0", + "script": { + "commands": [ + "run .*curl.*", + "run .*request.sh.*", + "transfer .*output.txt" + ], + "match": "regex" + }, + "net": { + "inet": { + "out": { + "protocols": ["https"], + "urls": ["https://api.coingecko.com", "https://httpbin.org"] + } + } + } + } +} diff --git a/goth_tests/e2e/vm/assets/outbound_signature.sha256.base64 b/goth_tests/e2e/vm/assets/outbound_signature.sha256.base64 new file mode 100644 index 0000000000..a5e342a242 --- /dev/null +++ b/goth_tests/e2e/vm/assets/outbound_signature.sha256.base64 @@ -0,0 +1 @@ +mHzoxwa1CZgIWBeHauoUKZPvTuifVVS4CNGI6/Lqf2jErYNUs+RxVZZ7KSmaahjmXRdTwzbO15r1PFGbbVaCmywZ45EENRi57gzyD1654Z8YRsz/paEjFWQkGmIkAFTwUlZQ7xwTp26yCusKtmIn6jnDXQXNvTaCUQgQ2Hyaz0Fva7r+pT3YW1nufeG9enY650UIuamltrzOwXrqE6tOFDUgk5b0u60nUwkC2vmVqjCynKR+0Tp5GDpv560GHIvHbdBC4SZnPDpmgAo642sQ/27Gh54XlIHrK+fFN0q+y5ACbALuARL5BtTJoKnJGpAGs31Oq5xPcTftDWV70GKiqA== \ No newline at end of file diff --git a/goth_tests/e2e/vm/test_e2e_outbound.py b/goth_tests/e2e/vm/test_e2e_outbound.py new file mode 100644 index 0000000000..b78a5bed61 --- /dev/null +++ b/goth_tests/e2e/vm/test_e2e_outbound.py @@ -0,0 +1,111 @@ +"""End to end tests for requesting VM tasks using goth REST API client.""" + +import json +import logging +import os +import base64 +from pathlib import Path +from typing import List + +import pytest + +from goth.address import ( + PROXY_HOST, + YAGNA_REST_URL, +) +from goth.configuration import load_yaml, Override +from goth.node import node_environment +from goth.runner import Runner +from goth.runner.container.payment import PaymentIdPool +from goth.runner.container.yagna import YagnaContainerConfig +from goth.runner.probe import RequestorProbe + +from goth_tests.helpers.activity import vm_exe_script_outbound +from goth_tests.helpers.negotiation import DemandBuilder, negotiate_agreements +from goth_tests.helpers.probe import ProviderProbe + +logger = logging.getLogger("goth.test.e2e_outbound") + + +@pytest.mark.asyncio +async def test_e2e_outbound( + common_assets: Path, + default_config: Path, + config_overrides: List[Override], + log_dir: Path, +): + """Test successful flow requesting a outbound curl task with goth REST API client.""" + + # Test external api request just one Requestor and one Provider + nodes = [ + {"name": "requestor", "type": "Requestor"}, + {"name": "provider-1", "type": "VM-Wasm-Provider", "use-proxy": True}, + ] + config_overrides.append(("nodes", nodes)) + + goth_config = load_yaml(default_config, config_overrides) + + runner = Runner( + base_log_dir=log_dir, + compose_config=goth_config.compose_config, + web_root_path=Path(__file__).parent / "assets", + ) + + async with runner(goth_config.containers): + requestor = runner.get_probes(probe_type=RequestorProbe)[0] + provider = runner.get_probes(probe_type=ProviderProbe)[0] + + manifest = open(f"{runner.web_root_path}/outbound_manifest.json").read() + signature = open(f"{runner.web_root_path}/outbound_signature.sha256.base64").read() + certificate = open(f"{runner.web_root_path}/outbound_certificate.cert").read() + + # Market + demand = ( + DemandBuilder(requestor) + .props_from_template(task_package = None) + .property("golem.srv.comp.payload", base64.b64encode(manifest.encode()).decode()) + .property("golem.srv.comp.payload.sig", signature) + .property("golem.srv.comp.payload.sig.algorithm", "sha256") + .property("golem.srv.comp.payload.cert", base64.b64encode(certificate.encode()).decode()) + .constraints("(&(golem.runtime.name=vm))") + .build() + ) + + agreement_providers = await negotiate_agreements( + requestor, + demand, + [provider], + lambda proposal: proposal.properties.get("golem.runtime.name") == "vm", + ) + + agreement_id, provider = agreement_providers[0] + + # Activity + + output_file = "output.txt" + output_path = Path(runner.web_root_path) / "upload" / output_file + + exe_script = vm_exe_script_outbound(runner, output_file) + + num_commands = len(exe_script) + + logger.info("Running activity on %s", provider.name) + activity_id = await requestor.create_activity(agreement_id) + await provider.wait_for_exeunit_started() + batch_id = await requestor.call_exec(activity_id, json.dumps(exe_script)) + await requestor.collect_results( + activity_id, batch_id, num_commands, timeout=300 + ) + await requestor.destroy_activity(activity_id) + await provider.wait_for_exeunit_finished() + + assert output_path.is_file() + assert len(output_path.read_text()) > 0 + + # Payment + + await provider.wait_for_invoice_sent() + invoices = await requestor.gather_invoices(agreement_id) + assert all(inv.agreement_id == agreement_id for inv in invoices) + await requestor.pay_invoices(invoices) + await provider.wait_for_invoice_paid() diff --git a/goth_tests/goth-config.yml b/goth_tests/goth-config.yml index 98f016bace..8d6724d0fc 100644 --- a/goth_tests/goth-config.yml +++ b/goth_tests/goth-config.yml @@ -4,7 +4,6 @@ # home directory. docker-compose: - # Path to compose file to be used, relative to `docker-dir` compose-file: "docker-compose.yml" docker-dir: "assets/docker/" @@ -25,13 +24,10 @@ docker-compose: ethereum: ".*Wallets supplied." zksync: ".*Running on http://.*:3030/.*" - key-dir: "assets/keys" - web-root: "assets/web-root" - node-types: # Each node type is a collection of attributes common to a group of nodes. # Required attributes are "name" and "class". @@ -48,10 +44,11 @@ node-types: destination: "/root/.local/share/ya-provider/hardware.json" - read-write: "~/.local/share/ya-provider/vm-images" destination: "/root/.local/share/ya-provider/exe-unit/cache/tmp" + - read-write: "assets/provider/cert_dir" + destination: "/root/.local/share/ya-provider/cert_dir" privileged-mode: True nodes: - - name: "requestor" type: "Requestor" diff --git a/goth_tests/helpers/activity.py b/goth_tests/helpers/activity.py index 35300f8f18..8e83ebf0d8 100644 --- a/goth_tests/helpers/activity.py +++ b/goth_tests/helpers/activity.py @@ -109,9 +109,30 @@ def wasi_exe_script(runner: Runner, output_file: str = "upload_file"): } }, ] +def vm_exe_script_outbound(runner: Runner, output_file: str = "output.txt"): + """VM exe script builder.""" + """Create a VM exe script for running a outbound task.""" + + output_path = Path(runner.web_root_path) / output_file + if output_path.exists(): + os.remove(output_path) + + web_server_addr = f"http://{runner.host_address}:{runner.web_server_port}" + + return [ + {"deploy": {}}, + {"start": {}}, + {"run": {"entry_point": "/golem/entrypoints/request.sh", "args": []}}, + { + "transfer": { + "from": f"container:/golem/output/output.txt", + "to": f"{web_server_addr}/upload/{output_file}", + } + }, + ] -def wasi_sleeper_exe_script(duration: float = 10.): +def wasi_sleeper_exe_script(duration: float = 10.0): """WASI exe script builder.""" """Create a WASI exe script for running a WASI sleeper task.""" @@ -123,5 +144,5 @@ def wasi_sleeper_exe_script(duration: float = 10.): "entry_point": "rust-wasi-sleeper", "args": [f"{duration}"], } - } + }, ] diff --git a/goth_tests/poetry.lock b/goth_tests/poetry.lock index 8f9b1d7ab0..fcf81313d2 100644 --- a/goth_tests/poetry.lock +++ b/goth_tests/poetry.lock @@ -54,7 +54,7 @@ python-versions = ">=3.5.3" [[package]] name = "atomicwrites" -version = "1.4.0" +version = "1.4.1" description = "Atomic file writes." category = "main" optional = false @@ -62,29 +62,26 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "attrs" -version = "21.4.0" +version = "22.1.0" description = "Classes Without Boilerplate" category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.5" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "cloudpickle"] +tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] +tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "cloudpickle"] [[package]] name = "bcrypt" -version = "3.2.2" +version = "4.0.0" description = "Modern password hashing for your software and your servers" category = "main" optional = false python-versions = ">=3.6" -[package.dependencies] -cffi = ">=1.1" - [package.extras] tests = ["pytest (>=3.2.1,!=3.3.0)"] typecheck = ["mypy"] @@ -137,7 +134,7 @@ python-versions = "*" [[package]] name = "cffi" -version = "1.15.0" +version = "1.15.1" description = "Foreign Function Interface for Python calling C code." category = "main" optional = false @@ -156,11 +153,11 @@ python-versions = "*" [[package]] name = "charset-normalizer" -version = "2.0.12" +version = "2.1.1" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." category = "main" optional = false -python-versions = ">=3.5.0" +python-versions = ">=3.6.0" [package.extras] unicode_backport = ["unicodedata2"] @@ -175,7 +172,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "colorama" -version = "0.4.4" +version = "0.4.5" description = "Cross-platform colored terminal text." category = "main" optional = false @@ -280,7 +277,7 @@ python-versions = ">=3" [[package]] name = "fastcore" -version = "1.4.3" +version = "1.5.22" description = "Python supercharged for fastai development" category = "main" optional = false @@ -290,7 +287,7 @@ python-versions = ">=3.7" packaging = "*" [package.extras] -dev = ["numpy", "nbdev (>=0.2.39)", "matplotlib", "pillow", "torch", "pandas"] +dev = ["numpy", "nbdev (>=0.2.39)", "matplotlib", "pillow", "torch", "pandas", "jupyterlab"] [[package]] name = "flask" @@ -307,9 +304,9 @@ Jinja2 = ">=2.10.1,<3.0" Werkzeug = ">=0.15,<2.0" [package.extras] -dev = ["pytest", "coverage", "tox", "sphinx", "pallets-sphinx-themes", "sphinxcontrib-log-cabinet", "sphinx-issues"] -docs = ["sphinx", "pallets-sphinx-themes", "sphinxcontrib-log-cabinet", "sphinx-issues"] dotenv = ["python-dotenv"] +docs = ["sphinx-issues", "sphinxcontrib-log-cabinet", "pallets-sphinx-themes", "sphinx"] +dev = ["sphinx-issues", "sphinxcontrib-log-cabinet", "pallets-sphinx-themes", "sphinx", "tox", "coverage", "pytest"] [[package]] name = "func-timeout" @@ -321,14 +318,14 @@ python-versions = "*" [[package]] name = "ghapi" -version = "0.1.20" +version = "0.1.23" description = "A python client for the GitHub API" category = "main" optional = false python-versions = ">=3.6" [package.dependencies] -fastcore = "*" +fastcore = ">=1.5.4" packaging = "*" [package.extras] @@ -336,7 +333,7 @@ dev = ["jsonref"] [[package]] name = "goth" -version = "0.11.0" +version = "0.12.0" description = "Golem Test Harness - integration testing framework" category = "main" optional = false @@ -362,8 +359,8 @@ ya-aioclient = "^0.6" [package.source] type = "git" url = "https://github.com/golemfactory/goth.git" -reference = "cf88916d00b96acdac90933ca12c80c5048065d7" -resolved_reference = "cf88916d00b96acdac90933ca12c80c5048065d7" +reference = "5244075a4742cfdc848b8dfc4084f788be8b2adf" +resolved_reference = "5244075a4742cfdc848b8dfc4084f788be8b2adf" [[package]] name = "h11" @@ -524,8 +521,8 @@ dev = ["asynctest (>=0.12.0)", "Flask (>=1.0,<1.2)", "hypothesis (>=5.8,<6)", "p [[package]] name = "msgpack" -version = "1.0.3" -description = "MessagePack (de)serializer." +version = "1.0.4" +description = "MessagePack serializer" category = "main" optional = false python-versions = "*" @@ -632,8 +629,8 @@ optional = false python-versions = ">=3.6" [package.extras] -dev = ["pre-commit", "tox"] -testing = ["pytest", "pytest-benchmark"] +testing = ["pytest-benchmark", "pytest"] +dev = ["tox", "pre-commit"] [[package]] name = "poethepoet" @@ -832,7 +829,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" [[package]] name = "regex" -version = "2022.4.24" +version = "2022.8.17" description = "Alternative regular expression module, to replace re." category = "dev" optional = false @@ -840,21 +837,21 @@ python-versions = ">=3.6" [[package]] name = "requests" -version = "2.27.1" +version = "2.28.1" description = "Python HTTP for Humans." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +python-versions = ">=3.7, <4" [package.dependencies] certifi = ">=2017.4.17" -charset-normalizer = {version = ">=2.0.0,<2.1.0", markers = "python_version >= \"3\""} -idna = {version = ">=2.5,<4", markers = "python_version >= \"3\""} +charset-normalizer = ">=2,<3" +idna = ">=2.5,<4" urllib3 = ">=1.21.1,<1.27" [package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] -use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "ruamel.yaml" @@ -913,7 +910,7 @@ python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "tomlkit" -version = "0.11.0" +version = "0.11.4" description = "Style preserving TOML library" category = "dev" optional = false @@ -921,11 +918,11 @@ python-versions = ">=3.6,<4.0" [[package]] name = "tornado" -version = "6.1" +version = "6.2" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." category = "main" optional = false -python-versions = ">= 3.5" +python-versions = ">= 3.7" [[package]] name = "transitions" @@ -960,15 +957,15 @@ python-versions = "*" [[package]] name = "urllib3" -version = "1.26.9" +version = "1.26.12" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4" [package.extras] brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] -secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] +secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "urllib3-secure-extra", "ipaddress"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] @@ -999,8 +996,8 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.extras] -dev = ["pytest", "pytest-timeout", "coverage", "tox", "sphinx", "pallets-sphinx-themes", "sphinx-issues"] watchdog = ["watchdog"] +dev = ["sphinx-issues", "pallets-sphinx-themes", "sphinx", "tox", "coverage", "pytest-timeout", "pytest"] [[package]] name = "wsproto" @@ -1028,11 +1025,11 @@ python-dateutil = ">=2.8.1,<3.0.0" [[package]] name = "yarl" -version = "1.7.2" +version = "1.8.1" description = "Yet another URL library" category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] idna = ">=2.0" @@ -1049,7 +1046,7 @@ python-versions = "*" [metadata] lock-version = "1.1" python-versions = "^3.8.0" -content-hash = "d574b6d3892cfa4fa9ba71bf83ad95ad2cc679a817f042ceceab6564378afe61" +content-hash = "7a29be65adf81e316ab7a4cc1d2b4a64af7a76b1290d37f1c0d892d3617f448d" [metadata.files] aiohttp = [ @@ -1108,25 +1105,25 @@ async-timeout = [ {file = "async_timeout-3.0.1-py3-none-any.whl", hash = "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"}, ] atomicwrites = [ - {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, - {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, ] attrs = [ - {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, - {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, + {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, + {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, ] bcrypt = [ - {file = "bcrypt-3.2.2-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:7180d98a96f00b1050e93f5b0f556e658605dd9f524d0b0e68ae7944673f525e"}, - {file = "bcrypt-3.2.2-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:61bae49580dce88095d669226d5076d0b9d927754cedbdf76c6c9f5099ad6f26"}, - {file = "bcrypt-3.2.2-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88273d806ab3a50d06bc6a2fc7c87d737dd669b76ad955f449c43095389bc8fb"}, - {file = "bcrypt-3.2.2-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:6d2cb9d969bfca5bc08e45864137276e4c3d3d7de2b162171def3d188bf9d34a"}, - {file = "bcrypt-3.2.2-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2b02d6bfc6336d1094276f3f588aa1225a598e27f8e3388f4db9948cb707b521"}, - {file = "bcrypt-3.2.2-cp36-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a2c46100e315c3a5b90fdc53e429c006c5f962529bc27e1dfd656292c20ccc40"}, - {file = "bcrypt-3.2.2-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:7d9ba2e41e330d2af4af6b1b6ec9e6128e91343d0b4afb9282e54e5508f31baa"}, - {file = "bcrypt-3.2.2-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:cd43303d6b8a165c29ec6756afd169faba9396a9472cdff753fe9f19b96ce2fa"}, - {file = "bcrypt-3.2.2-cp36-abi3-win32.whl", hash = "sha256:4e029cef560967fb0cf4a802bcf4d562d3d6b4b1bf81de5ec1abbe0f1adb027e"}, - {file = "bcrypt-3.2.2-cp36-abi3-win_amd64.whl", hash = "sha256:7ff2069240c6bbe49109fe84ca80508773a904f5a8cb960e02a977f7f519b129"}, - {file = "bcrypt-3.2.2.tar.gz", hash = "sha256:433c410c2177057705da2a9f2cd01dd157493b2a7ac14c8593a16b3dab6b6bfb"}, + {file = "bcrypt-4.0.0-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:845b1daf4df2dd94d2fdbc9454953ca9dd0e12970a0bfc9f3dcc6faea3fa96e4"}, + {file = "bcrypt-4.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:8780e69f9deec9d60f947b169507d2c9816e4f11548f1f7ebee2af38b9b22ae4"}, + {file = "bcrypt-4.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c3334446fac200499e8bc04a530ce3cf0b3d7151e0e4ac5c0dddd3d95e97843"}, + {file = "bcrypt-4.0.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfb67f6a6c72dfb0a02f3df51550aa1862708e55128b22543e2b42c74f3620d7"}, + {file = "bcrypt-4.0.0-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:7c7dd6c1f05bf89e65261d97ac3a6520f34c2acb369afb57e3ea4449be6ff8fd"}, + {file = "bcrypt-4.0.0-cp36-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:594780b364fb45f2634c46ec8d3e61c1c0f1811c4f2da60e8eb15594ecbf93ed"}, + {file = "bcrypt-4.0.0-cp36-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2d0dd19aad87e4ab882ef1d12df505f4c52b28b69666ce83c528f42c07379227"}, + {file = "bcrypt-4.0.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:bf413f2a9b0a2950fc750998899013f2e718d20fa4a58b85ca50b6df5ed1bbf9"}, + {file = "bcrypt-4.0.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:ede0f506554571c8eda80db22b83c139303ec6b595b8f60c4c8157bdd0bdee36"}, + {file = "bcrypt-4.0.0-cp36-abi3-win32.whl", hash = "sha256:dc6ec3dc19b1c193b2f7cf279d3e32e7caf447532fbcb7af0906fe4398900c33"}, + {file = "bcrypt-4.0.0-cp36-abi3-win_amd64.whl", hash = "sha256:0b0f0c7141622a31e9734b7f649451147c04ebb5122327ac0bd23744df84be90"}, + {file = "bcrypt-4.0.0.tar.gz", hash = "sha256:c59c170fc9225faad04dde1ba61d85b413946e8ce2e5f5f5ff30dfd67283f319"}, ] black = [ {file = "black-20.8b1.tar.gz", hash = "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea"}, @@ -1203,72 +1200,86 @@ certifi = [ {file = "certifi-2020.12.5.tar.gz", hash = "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c"}, ] cffi = [ - {file = "cffi-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962"}, - {file = "cffi-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:23cfe892bd5dd8941608f93348c0737e369e51c100d03718f108bf1add7bd6d0"}, - {file = "cffi-1.15.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:41d45de54cd277a7878919867c0f08b0cf817605e4eb94093e7516505d3c8d14"}, - {file = "cffi-1.15.0-cp27-cp27m-win32.whl", hash = "sha256:4a306fa632e8f0928956a41fa8e1d6243c71e7eb59ffbd165fc0b41e316b2474"}, - {file = "cffi-1.15.0-cp27-cp27m-win_amd64.whl", hash = "sha256:e7022a66d9b55e93e1a845d8c9eba2a1bebd4966cd8bfc25d9cd07d515b33fa6"}, - {file = "cffi-1.15.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:14cd121ea63ecdae71efa69c15c5543a4b5fbcd0bbe2aad864baca0063cecf27"}, - {file = "cffi-1.15.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:d4d692a89c5cf08a8557fdeb329b82e7bf609aadfaed6c0d79f5a449a3c7c023"}, - {file = "cffi-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0104fb5ae2391d46a4cb082abdd5c69ea4eab79d8d44eaaf79f1b1fd806ee4c2"}, - {file = "cffi-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:91ec59c33514b7c7559a6acda53bbfe1b283949c34fe7440bcf917f96ac0723e"}, - {file = "cffi-1.15.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f5c7150ad32ba43a07c4479f40241756145a1f03b43480e058cfd862bf5041c7"}, - {file = "cffi-1.15.0-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:00c878c90cb53ccfaae6b8bc18ad05d2036553e6d9d1d9dbcf323bbe83854ca3"}, - {file = "cffi-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abb9a20a72ac4e0fdb50dae135ba5e77880518e742077ced47eb1499e29a443c"}, - {file = "cffi-1.15.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a5263e363c27b653a90078143adb3d076c1a748ec9ecc78ea2fb916f9b861962"}, - {file = "cffi-1.15.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f54a64f8b0c8ff0b64d18aa76675262e1700f3995182267998c31ae974fbc382"}, - {file = "cffi-1.15.0-cp310-cp310-win32.whl", hash = "sha256:c21c9e3896c23007803a875460fb786118f0cdd4434359577ea25eb556e34c55"}, - {file = "cffi-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:5e069f72d497312b24fcc02073d70cb989045d1c91cbd53979366077959933e0"}, - {file = "cffi-1.15.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:64d4ec9f448dfe041705426000cc13e34e6e5bb13736e9fd62e34a0b0c41566e"}, - {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2756c88cbb94231c7a147402476be2c4df2f6078099a6f4a480d239a8817ae39"}, - {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b96a311ac60a3f6be21d2572e46ce67f09abcf4d09344c49274eb9e0bf345fc"}, - {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:75e4024375654472cc27e91cbe9eaa08567f7fbdf822638be2814ce059f58032"}, - {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:59888172256cac5629e60e72e86598027aca6bf01fa2465bdb676d37636573e8"}, - {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:27c219baf94952ae9d50ec19651a687b826792055353d07648a5695413e0c605"}, - {file = "cffi-1.15.0-cp36-cp36m-win32.whl", hash = "sha256:4958391dbd6249d7ad855b9ca88fae690783a6be9e86df65865058ed81fc860e"}, - {file = "cffi-1.15.0-cp36-cp36m-win_amd64.whl", hash = "sha256:f6f824dc3bce0edab5f427efcfb1d63ee75b6fcb7282900ccaf925be84efb0fc"}, - {file = "cffi-1.15.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:06c48159c1abed75c2e721b1715c379fa3200c7784271b3c46df01383b593636"}, - {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c2051981a968d7de9dd2d7b87bcb9c939c74a34626a6e2f8181455dd49ed69e4"}, - {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:fd8a250edc26254fe5b33be00402e6d287f562b6a5b2152dec302fa15bb3e997"}, - {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:91d77d2a782be4274da750752bb1650a97bfd8f291022b379bb8e01c66b4e96b"}, - {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:45db3a33139e9c8f7c09234b5784a5e33d31fd6907800b316decad50af323ff2"}, - {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:263cc3d821c4ab2213cbe8cd8b355a7f72a8324577dc865ef98487c1aeee2bc7"}, - {file = "cffi-1.15.0-cp37-cp37m-win32.whl", hash = "sha256:17771976e82e9f94976180f76468546834d22a7cc404b17c22df2a2c81db0c66"}, - {file = "cffi-1.15.0-cp37-cp37m-win_amd64.whl", hash = "sha256:3415c89f9204ee60cd09b235810be700e993e343a408693e80ce7f6a40108029"}, - {file = "cffi-1.15.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4238e6dab5d6a8ba812de994bbb0a79bddbdf80994e4ce802b6f6f3142fcc880"}, - {file = "cffi-1.15.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0808014eb713677ec1292301ea4c81ad277b6cdf2fdd90fd540af98c0b101d20"}, - {file = "cffi-1.15.0-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:57e9ac9ccc3101fac9d6014fba037473e4358ef4e89f8e181f8951a2c0162024"}, - {file = "cffi-1.15.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b6c2ea03845c9f501ed1313e78de148cd3f6cad741a75d43a29b43da27f2e1e"}, - {file = "cffi-1.15.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:10dffb601ccfb65262a27233ac273d552ddc4d8ae1bf93b21c94b8511bffe728"}, - {file = "cffi-1.15.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:786902fb9ba7433aae840e0ed609f45c7bcd4e225ebb9c753aa39725bb3e6ad6"}, - {file = "cffi-1.15.0-cp38-cp38-win32.whl", hash = "sha256:da5db4e883f1ce37f55c667e5c0de439df76ac4cb55964655906306918e7363c"}, - {file = "cffi-1.15.0-cp38-cp38-win_amd64.whl", hash = "sha256:181dee03b1170ff1969489acf1c26533710231c58f95534e3edac87fff06c443"}, - {file = "cffi-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:45e8636704eacc432a206ac7345a5d3d2c62d95a507ec70d62f23cd91770482a"}, - {file = "cffi-1.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:31fb708d9d7c3f49a60f04cf5b119aeefe5644daba1cd2a0fe389b674fd1de37"}, - {file = "cffi-1.15.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6dc2737a3674b3e344847c8686cf29e500584ccad76204efea14f451d4cc669a"}, - {file = "cffi-1.15.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:74fdfdbfdc48d3f47148976f49fab3251e550a8720bebc99bf1483f5bfb5db3e"}, - {file = "cffi-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffaa5c925128e29efbde7301d8ecaf35c8c60ffbcd6a1ffd3a552177c8e5e796"}, - {file = "cffi-1.15.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f7d084648d77af029acb79a0ff49a0ad7e9d09057a9bf46596dac9514dc07df"}, - {file = "cffi-1.15.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ef1f279350da2c586a69d32fc8733092fd32cc8ac95139a00377841f59a3f8d8"}, - {file = "cffi-1.15.0-cp39-cp39-win32.whl", hash = "sha256:2a23af14f408d53d5e6cd4e3d9a24ff9e05906ad574822a10563efcef137979a"}, - {file = "cffi-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:3773c4d81e6e818df2efbc7dd77325ca0dcb688116050fb2b3011218eda36139"}, - {file = "cffi-1.15.0.tar.gz", hash = "sha256:920f0d66a896c2d99f0adbb391f990a84091179542c205fa53ce5787aff87954"}, + {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, + {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, + {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, + {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, + {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, + {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, + {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, + {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, + {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, + {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, + {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, + {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, + {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, + {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, + {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, + {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, + {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, + {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, + {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, ] chardet = [ {file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"}, {file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"}, ] charset-normalizer = [ - {file = "charset-normalizer-2.0.12.tar.gz", hash = "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597"}, - {file = "charset_normalizer-2.0.12-py3-none-any.whl", hash = "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df"}, + {file = "charset-normalizer-2.1.1.tar.gz", hash = "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845"}, + {file = "charset_normalizer-2.1.1-py3-none-any.whl", hash = "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"}, ] click = [ {file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"}, {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"}, ] colorama = [ - {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, - {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, + {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, + {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, ] cryptography = [ {file = "cryptography-3.2.1-cp27-cp27m-macosx_10_10_x86_64.whl", hash = "sha256:6dc59630ecce8c1f558277ceb212c751d6730bd12c80ea96b4ac65637c4f55e7"}, @@ -1317,8 +1328,8 @@ dpath = [ {file = "dpath-2.0.6.tar.gz", hash = "sha256:5a1ddae52233fbc8ef81b15fb85073a81126bb43698d3f3a1b6aaf561a46cdc0"}, ] fastcore = [ - {file = "fastcore-1.4.3-py3-none-any.whl", hash = "sha256:9013568a5bbe4a5f8a0d94eab9e883c22a00ac1a09cde211390731814d17bebb"}, - {file = "fastcore-1.4.3.tar.gz", hash = "sha256:b7ae674a713c55b0563da253a1c3aaa2acc8c5611dd0141b49fdb057d42cfa4d"}, + {file = "fastcore-1.5.22-py3-none-any.whl", hash = "sha256:96ec9b23cd95f969a3bd0a118aed92a60c7da80d89716ac3585cacc9b26a708a"}, + {file = "fastcore-1.5.22.tar.gz", hash = "sha256:73c3f268f882955038237bd7d134e5d872d116577ed586025641ecc08b2706c4"}, ] flask = [ {file = "Flask-1.1.4-py2.py3-none-any.whl", hash = "sha256:c34f04500f2cbbea882b1acb02002ad6fe6b7ffa64a6164577995657f50aed22"}, @@ -1328,8 +1339,8 @@ func-timeout = [ {file = "func_timeout-4.3.5.tar.gz", hash = "sha256:74cd3c428ec94f4edfba81f9b2f14904846d5ffccc27c92433b8b5939b5575dd"}, ] ghapi = [ - {file = "ghapi-0.1.20-py3-none-any.whl", hash = "sha256:f9b060b1d57629876b0903d7decd729188ba430617be5c8395efa1763cffbd43"}, - {file = "ghapi-0.1.20.tar.gz", hash = "sha256:7120031e7e1c08b3508312b635a39e3cd0659212e8a41792b82b9ef34b028472"}, + {file = "ghapi-0.1.23-py3-none-any.whl", hash = "sha256:492b5f82b5d4844a0297c8ae6afda638432cee6c74b0bd0a0458b35d19b92b5e"}, + {file = "ghapi-0.1.23.tar.gz", hash = "sha256:93fa097e394d743c6702e217d588a4123696f62d055f2acc495166676843d59f"}, ] goth = [] h11 = [ @@ -1453,40 +1464,58 @@ mitmproxy = [ {file = "mitmproxy-5.3.0-py3-none-any.whl", hash = "sha256:481940365fc08fc2318343e530ef01d35084e8b56d1c61b5e1a7b6ed9b664d24"}, ] msgpack = [ - {file = "msgpack-1.0.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:96acc674bb9c9be63fa8b6dabc3248fdc575c4adc005c440ad02f87ca7edd079"}, - {file = "msgpack-1.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2c3ca57c96c8e69c1a0d2926a6acf2d9a522b41dc4253a8945c4c6cd4981a4e3"}, - {file = "msgpack-1.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b0a792c091bac433dfe0a70ac17fc2087d4595ab835b47b89defc8bbabcf5c73"}, - {file = "msgpack-1.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c58cdec1cb5fcea8c2f1771d7b5fec79307d056874f746690bd2bdd609ab147"}, - {file = "msgpack-1.0.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2f97c0f35b3b096a330bb4a1a9247d0bd7e1f3a2eba7ab69795501504b1c2c39"}, - {file = "msgpack-1.0.3-cp310-cp310-win32.whl", hash = "sha256:36a64a10b16c2ab31dcd5f32d9787ed41fe68ab23dd66957ca2826c7f10d0b85"}, - {file = "msgpack-1.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:c1ba333b4024c17c7591f0f372e2daa3c31db495a9b2af3cf664aef3c14354f7"}, - {file = "msgpack-1.0.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c2140cf7a3ec475ef0938edb6eb363fa704159e0bf71dde15d953bacc1cf9d7d"}, - {file = "msgpack-1.0.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f4c22717c74d44bcd7af353024ce71c6b55346dad5e2cc1ddc17ce8c4507c6b"}, - {file = "msgpack-1.0.3-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d733a15ade190540c703de209ffbc42a3367600421b62ac0c09fde594da6ec"}, - {file = "msgpack-1.0.3-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7e03b06f2982aa98d4ddd082a210c3db200471da523f9ac197f2828e80e7770"}, - {file = "msgpack-1.0.3-cp36-cp36m-win32.whl", hash = "sha256:3d875631ecab42f65f9dce6f55ce6d736696ced240f2634633188de2f5f21af9"}, - {file = "msgpack-1.0.3-cp36-cp36m-win_amd64.whl", hash = "sha256:40fb89b4625d12d6027a19f4df18a4de5c64f6f3314325049f219683e07e678a"}, - {file = "msgpack-1.0.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6eef0cf8db3857b2b556213d97dd82de76e28a6524853a9beb3264983391dc1a"}, - {file = "msgpack-1.0.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d8c332f53ffff01953ad25131272506500b14750c1d0ce8614b17d098252fbc"}, - {file = "msgpack-1.0.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c0903bd93cbd34653dd63bbfcb99d7539c372795201f39d16fdfde4418de43a"}, - {file = "msgpack-1.0.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bf1e6bfed4860d72106f4e0a1ab519546982b45689937b40257cfd820650b920"}, - {file = "msgpack-1.0.3-cp37-cp37m-win32.whl", hash = "sha256:d02cea2252abc3756b2ac31f781f7a98e89ff9759b2e7450a1c7a0d13302ff50"}, - {file = "msgpack-1.0.3-cp37-cp37m-win_amd64.whl", hash = "sha256:2f30dd0dc4dfe6231ad253b6f9f7128ac3202ae49edd3f10d311adc358772dba"}, - {file = "msgpack-1.0.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:f201d34dc89342fabb2a10ed7c9a9aaaed9b7af0f16a5923f1ae562b31258dea"}, - {file = "msgpack-1.0.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bb87f23ae7d14b7b3c21009c4b1705ec107cb21ee71975992f6aca571fb4a42a"}, - {file = "msgpack-1.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8a3a5c4b16e9d0edb823fe54b59b5660cc8d4782d7bf2c214cb4b91a1940a8ef"}, - {file = "msgpack-1.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f74da1e5fcf20ade12c6bf1baa17a2dc3604958922de8dc83cbe3eff22e8b611"}, - {file = "msgpack-1.0.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:73a80bd6eb6bcb338c1ec0da273f87420829c266379c8c82fa14c23fb586cfa1"}, - {file = "msgpack-1.0.3-cp38-cp38-win32.whl", hash = "sha256:9fce00156e79af37bb6db4e7587b30d11e7ac6a02cb5bac387f023808cd7d7f4"}, - {file = "msgpack-1.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:9b6f2d714c506e79cbead331de9aae6837c8dd36190d02da74cb409b36162e8a"}, - {file = "msgpack-1.0.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:89908aea5f46ee1474cc37fbc146677f8529ac99201bc2faf4ef8edc023c2bf3"}, - {file = "msgpack-1.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:973ad69fd7e31159eae8f580f3f707b718b61141838321c6fa4d891c4a2cca52"}, - {file = "msgpack-1.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da24375ab4c50e5b7486c115a3198d207954fe10aaa5708f7b65105df09109b2"}, - {file = "msgpack-1.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a598d0685e4ae07a0672b59792d2cc767d09d7a7f39fd9bd37ff84e060b1a996"}, - {file = "msgpack-1.0.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4c309a68cb5d6bbd0c50d5c71a25ae81f268c2dc675c6f4ea8ab2feec2ac4e2"}, - {file = "msgpack-1.0.3-cp39-cp39-win32.whl", hash = "sha256:494471d65b25a8751d19c83f1a482fd411d7ca7a3b9e17d25980a74075ba0e88"}, - {file = "msgpack-1.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:f01b26c2290cbd74316990ba84a14ac3d599af9cebefc543d241a66e785cf17d"}, - {file = "msgpack-1.0.3.tar.gz", hash = "sha256:51fdc7fb93615286428ee7758cecc2f374d5ff363bdd884c7ea622a7a327a81e"}, + {file = "msgpack-1.0.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:4ab251d229d10498e9a2f3b1e68ef64cb393394ec477e3370c457f9430ce9250"}, + {file = "msgpack-1.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:112b0f93202d7c0fef0b7810d465fde23c746a2d482e1e2de2aafd2ce1492c88"}, + {file = "msgpack-1.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:002b5c72b6cd9b4bafd790f364b8480e859b4712e91f43014fe01e4f957b8467"}, + {file = "msgpack-1.0.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35bc0faa494b0f1d851fd29129b2575b2e26d41d177caacd4206d81502d4c6a6"}, + {file = "msgpack-1.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4733359808c56d5d7756628736061c432ded018e7a1dff2d35a02439043321aa"}, + {file = "msgpack-1.0.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb514ad14edf07a1dbe63761fd30f89ae79b42625731e1ccf5e1f1092950eaa6"}, + {file = "msgpack-1.0.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:c23080fdeec4716aede32b4e0ef7e213c7b1093eede9ee010949f2a418ced6ba"}, + {file = "msgpack-1.0.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:49565b0e3d7896d9ea71d9095df15b7f75a035c49be733051c34762ca95bbf7e"}, + {file = "msgpack-1.0.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:aca0f1644d6b5a73eb3e74d4d64d5d8c6c3d577e753a04c9e9c87d07692c58db"}, + {file = "msgpack-1.0.4-cp310-cp310-win32.whl", hash = "sha256:0dfe3947db5fb9ce52aaea6ca28112a170db9eae75adf9339a1aec434dc954ef"}, + {file = "msgpack-1.0.4-cp310-cp310-win_amd64.whl", hash = "sha256:4dea20515f660aa6b7e964433b1808d098dcfcabbebeaaad240d11f909298075"}, + {file = "msgpack-1.0.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e83f80a7fec1a62cf4e6c9a660e39c7f878f603737a0cdac8c13131d11d97f52"}, + {file = "msgpack-1.0.4-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c11a48cf5e59026ad7cb0dc29e29a01b5a66a3e333dc11c04f7e991fc5510a9"}, + {file = "msgpack-1.0.4-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1276e8f34e139aeff1c77a3cefb295598b504ac5314d32c8c3d54d24fadb94c9"}, + {file = "msgpack-1.0.4-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6c9566f2c39ccced0a38d37c26cc3570983b97833c365a6044edef3574a00c08"}, + {file = "msgpack-1.0.4-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:fcb8a47f43acc113e24e910399376f7277cf8508b27e5b88499f053de6b115a8"}, + {file = "msgpack-1.0.4-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:76ee788122de3a68a02ed6f3a16bbcd97bc7c2e39bd4d94be2f1821e7c4a64e6"}, + {file = "msgpack-1.0.4-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:0a68d3ac0104e2d3510de90a1091720157c319ceeb90d74f7b5295a6bee51bae"}, + {file = "msgpack-1.0.4-cp36-cp36m-win32.whl", hash = "sha256:85f279d88d8e833ec015650fd15ae5eddce0791e1e8a59165318f371158efec6"}, + {file = "msgpack-1.0.4-cp36-cp36m-win_amd64.whl", hash = "sha256:c1683841cd4fa45ac427c18854c3ec3cd9b681694caf5bff04edb9387602d661"}, + {file = "msgpack-1.0.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a75dfb03f8b06f4ab093dafe3ddcc2d633259e6c3f74bb1b01996f5d8aa5868c"}, + {file = "msgpack-1.0.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9667bdfdf523c40d2511f0e98a6c9d3603be6b371ae9a238b7ef2dc4e7a427b0"}, + {file = "msgpack-1.0.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11184bc7e56fd74c00ead4f9cc9a3091d62ecb96e97653add7a879a14b003227"}, + {file = "msgpack-1.0.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ac5bd7901487c4a1dd51a8c58f2632b15d838d07ceedaa5e4c080f7190925bff"}, + {file = "msgpack-1.0.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1e91d641d2bfe91ba4c52039adc5bccf27c335356055825c7f88742c8bb900dd"}, + {file = "msgpack-1.0.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2a2df1b55a78eb5f5b7d2a4bb221cd8363913830145fad05374a80bf0877cb1e"}, + {file = "msgpack-1.0.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:545e3cf0cf74f3e48b470f68ed19551ae6f9722814ea969305794645da091236"}, + {file = "msgpack-1.0.4-cp37-cp37m-win32.whl", hash = "sha256:2cc5ca2712ac0003bcb625c96368fd08a0f86bbc1a5578802512d87bc592fe44"}, + {file = "msgpack-1.0.4-cp37-cp37m-win_amd64.whl", hash = "sha256:eba96145051ccec0ec86611fe9cf693ce55f2a3ce89c06ed307de0e085730ec1"}, + {file = "msgpack-1.0.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:7760f85956c415578c17edb39eed99f9181a48375b0d4a94076d84148cf67b2d"}, + {file = "msgpack-1.0.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:449e57cc1ff18d3b444eb554e44613cffcccb32805d16726a5494038c3b93dab"}, + {file = "msgpack-1.0.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d603de2b8d2ea3f3bcb2efe286849aa7a81531abc52d8454da12f46235092bcb"}, + {file = "msgpack-1.0.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48f5d88c99f64c456413d74a975bd605a9b0526293218a3b77220a2c15458ba9"}, + {file = "msgpack-1.0.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6916c78f33602ecf0509cc40379271ba0f9ab572b066bd4bdafd7434dee4bc6e"}, + {file = "msgpack-1.0.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:81fc7ba725464651190b196f3cd848e8553d4d510114a954681fd0b9c479d7e1"}, + {file = "msgpack-1.0.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d5b5b962221fa2c5d3a7f8133f9abffc114fe218eb4365e40f17732ade576c8e"}, + {file = "msgpack-1.0.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:77ccd2af37f3db0ea59fb280fa2165bf1b096510ba9fe0cc2bf8fa92a22fdb43"}, + {file = "msgpack-1.0.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b17be2478b622939e39b816e0aa8242611cc8d3583d1cd8ec31b249f04623243"}, + {file = "msgpack-1.0.4-cp38-cp38-win32.whl", hash = "sha256:2bb8cdf50dd623392fa75525cce44a65a12a00c98e1e37bf0fb08ddce2ff60d2"}, + {file = "msgpack-1.0.4-cp38-cp38-win_amd64.whl", hash = "sha256:26b8feaca40a90cbe031b03d82b2898bf560027160d3eae1423f4a67654ec5d6"}, + {file = "msgpack-1.0.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:462497af5fd4e0edbb1559c352ad84f6c577ffbbb708566a0abaaa84acd9f3ae"}, + {file = "msgpack-1.0.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2999623886c5c02deefe156e8f869c3b0aaeba14bfc50aa2486a0415178fce55"}, + {file = "msgpack-1.0.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f0029245c51fd9473dc1aede1160b0a29f4a912e6b1dd353fa6d317085b219da"}, + {file = "msgpack-1.0.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed6f7b854a823ea44cf94919ba3f727e230da29feb4a99711433f25800cf747f"}, + {file = "msgpack-1.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0df96d6eaf45ceca04b3f3b4b111b86b33785683d682c655063ef8057d61fd92"}, + {file = "msgpack-1.0.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6a4192b1ab40f8dca3f2877b70e63799d95c62c068c84dc028b40a6cb03ccd0f"}, + {file = "msgpack-1.0.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0e3590f9fb9f7fbc36df366267870e77269c03172d086fa76bb4eba8b2b46624"}, + {file = "msgpack-1.0.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:1576bd97527a93c44fa856770197dec00d223b0b9f36ef03f65bac60197cedf8"}, + {file = "msgpack-1.0.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:63e29d6e8c9ca22b21846234913c3466b7e4ee6e422f205a2988083de3b08cae"}, + {file = "msgpack-1.0.4-cp39-cp39-win32.whl", hash = "sha256:fb62ea4b62bfcb0b380d5680f9a4b3f9a2d166d9394e9bbd9666c0ee09a3645c"}, + {file = "msgpack-1.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:4d5834a2a48965a349da1c5a79760d94a1a0172fbb5ab6b5b33cbf8447e109ce"}, + {file = "msgpack-1.0.4.tar.gz", hash = "sha256:f5d869c18f030202eb412f08b28d2afeea553d6613aee89e200d7aca7ef01f5f"}, ] multidict = [ {file = "multidict-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b9e95a740109c6047602f4db4da9949e6c5945cefbad34a1299775ddc9a62e2"}, @@ -1756,84 +1785,84 @@ pyyaml = [ {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"}, ] regex = [ - {file = "regex-2022.4.24-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f86aef546add4ff1202e1f31e9bb54f9268f17d996b2428877283146bf9bc013"}, - {file = "regex-2022.4.24-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e944268445b5694f5d41292c9228f0ca46d5a32a67f195d5f8547c1f1d91f4bc"}, - {file = "regex-2022.4.24-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0f8da3145f4b72f7ce6181c804eaa44cdcea313c8998cdade3d9e20a8717a9cb"}, - {file = "regex-2022.4.24-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0fd464e547dbabf4652ca5fe9d88d75ec30182981e737c07b3410235a44b9939"}, - {file = "regex-2022.4.24-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:071bcb625e890f28b7c4573124a6512ea65107152b1d3ca101ce33a52dad4593"}, - {file = "regex-2022.4.24-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c2de7f32fa87d04d40f54bce3843af430697aba51c3a114aa62837a0772f219"}, - {file = "regex-2022.4.24-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1a07e8366115069f26822c47732122ab61598830a69f5629a37ea8881487c107"}, - {file = "regex-2022.4.24-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:036d1c1fbe69eba3ee253c107e71749cdbb4776db93d674bc0d5e28f30300734"}, - {file = "regex-2022.4.24-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:af1e687ffab18a75409e5e5d6215b6ccd41a5a1a0ea6ce9665e01253f737a0d3"}, - {file = "regex-2022.4.24-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:165cc75cfa5aa0f12adb2ac6286330e7229a06dc0e6c004ec35da682b5b89579"}, - {file = "regex-2022.4.24-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:3e35c50b27f36176c792738cb9b858523053bc495044d2c2b44db24376b266f1"}, - {file = "regex-2022.4.24-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:43ee0df35925ae4b0cc6ee3f60b73369e559dd2ac40945044da9394dd9d3a51d"}, - {file = "regex-2022.4.24-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:58521abdab76583bd41ef47e5e2ddd93b32501aee4ee8cee71dee10a45ba46b1"}, - {file = "regex-2022.4.24-cp310-cp310-win32.whl", hash = "sha256:275afc7352982ee947fc88f67a034b52c78395977b5fc7c9be15f7dc95b76f06"}, - {file = "regex-2022.4.24-cp310-cp310-win_amd64.whl", hash = "sha256:253f858a0255cd91a0424a4b15c2eedb12f20274f85731b0d861c8137e843065"}, - {file = "regex-2022.4.24-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:85b7ee4d0c7a46296d884f6b489af8b960c4291d76aea4b22fd4fbe05e6ec08e"}, - {file = "regex-2022.4.24-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e0da7ef160d4f3eb3d4d3e39a02c3c42f7dbcfce62c81f784cc99fc7059765f"}, - {file = "regex-2022.4.24-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4f2e2cef324ca9355049ee1e712f68e2e92716eba24275e6767b9bfa15f1f478"}, - {file = "regex-2022.4.24-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6165e737acb3bea3271372e8aa5ebe7226c8a8e8da1b94af2d6547c5a09d689d"}, - {file = "regex-2022.4.24-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3f6bd8178cce5bb56336722d5569d19c50bba5915a69a2050c497fb921e7cb0f"}, - {file = "regex-2022.4.24-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:45b761406777a681db0c24686178532134c937d24448d9e085279b69e9eb7da4"}, - {file = "regex-2022.4.24-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3dfbadb7b74d95f72f9f9dbf9778f7de92722ab520a109ceaf7927461fa85b10"}, - {file = "regex-2022.4.24-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:9913bcf730eb6e9b441fb176832eea9acbebab6035542c7c89d90c803f5cd3be"}, - {file = "regex-2022.4.24-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:68aed3fb0c61296bd6d234f558f78c51671f79ccb069cbcd428c2eea6fee7a5b"}, - {file = "regex-2022.4.24-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:8e7d33f93cdd01868327d834d0f5bb029241cd293b47d51b96814dec27fc9b4b"}, - {file = "regex-2022.4.24-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:82b7fc67e49fdce671bdbec1127189fc979badf062ce6e79dc95ef5e07a8bf92"}, - {file = "regex-2022.4.24-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:c36906a7855ec33a9083608e6cd595e4729dab18aeb9aad0dd0b039240266239"}, - {file = "regex-2022.4.24-cp36-cp36m-win32.whl", hash = "sha256:b2df3ede85d778c949d9bd2a50237072cee3df0a423c91f5514f78f8035bde87"}, - {file = "regex-2022.4.24-cp36-cp36m-win_amd64.whl", hash = "sha256:dffd9114ade73137ab2b79a8faf864683dbd2dbbb6b23a305fbbd4cbaeeb2187"}, - {file = "regex-2022.4.24-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6a0ef57cccd8089b4249eebad95065390e56c04d4a92c51316eab4131bca96a9"}, - {file = "regex-2022.4.24-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12af15b6edb00e425f713160cfd361126e624ec0de86e74f7cad4b97b7f169b3"}, - {file = "regex-2022.4.24-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7f271d0831d8ebc56e17b37f9fa1824b0379221d1238ae77c18a6e8c47f1fdce"}, - {file = "regex-2022.4.24-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:37903d5ca11fa47577e8952d2e2c6de28553b11c70defee827afb941ab2c6729"}, - {file = "regex-2022.4.24-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b747cef8e5dcdaf394192d43a0c02f5825aeb0ecd3d43e63ae500332ab830b0"}, - {file = "regex-2022.4.24-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:582ea06079a03750b5f71e20a87cd99e646d796638b5894ff85987ebf5e04924"}, - {file = "regex-2022.4.24-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:aa6daa189db9104787ff1fd7a7623ce017077aa59eaac609d0d25ba95ed251a0"}, - {file = "regex-2022.4.24-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7dbc96419ef0fb6ac56626014e6d3a345aeb8b17a3df8830235a88626ffc8d84"}, - {file = "regex-2022.4.24-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:0fb6cb16518ac7eff29d1e0b0cce90275dfae0f17154165491058c31d58bdd1d"}, - {file = "regex-2022.4.24-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bea61de0c688198e3d9479344228c7accaa22a78b58ec408e41750ebafee6c08"}, - {file = "regex-2022.4.24-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:46cbc5b23f85e94161b093dba1b49035697cf44c7db3c930adabfc0e6d861b95"}, - {file = "regex-2022.4.24-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:50b77622016f03989cd06ecf6b602c7a6b4ed2e3ce04133876b041d109c934ee"}, - {file = "regex-2022.4.24-cp37-cp37m-win32.whl", hash = "sha256:2bde99f2cdfd6db1ec7e02d68cadd384ffe7413831373ea7cc68c5415a0cb577"}, - {file = "regex-2022.4.24-cp37-cp37m-win_amd64.whl", hash = "sha256:66fb765b2173d90389384708e3e1d3e4be1148bd8d4d50476b1469da5a2f0229"}, - {file = "regex-2022.4.24-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:709396c0c95b95045fac89b94f997410ff39b81a09863fe21002f390d48cc7d3"}, - {file = "regex-2022.4.24-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7a608022f4593fc67518c6c599ae5abdb03bb8acd75993c82cd7a4c8100eff81"}, - {file = "regex-2022.4.24-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb7107faf0168de087f62a2f2ed00f9e9da12e0b801582b516ddac236b871cda"}, - {file = "regex-2022.4.24-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aabc28f7599f781ddaeac168d0b566d0db82182cc3dcf62129f0a4fc2927b811"}, - {file = "regex-2022.4.24-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:92ad03f928675ca05b79d3b1d3dfc149e2226d57ed9d57808f82105d511d0212"}, - {file = "regex-2022.4.24-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b7ba3c304a4a5d8112dbd30df8b3e4ef59b4b07807957d3c410d9713abaee9a8"}, - {file = "regex-2022.4.24-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2acf5c66fbb62b5fe4c40978ddebafa50818f00bf79d60569d9762f6356336e"}, - {file = "regex-2022.4.24-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7c4d9770e579eb11b582b2e2fd19fa204a15cb1589ae73cd4dcbb63b64f3e828"}, - {file = "regex-2022.4.24-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:02543d6d5c32d361b7cc468079ba4cddaaf4a6544f655901ba1ff9d8e3f18755"}, - {file = "regex-2022.4.24-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:73ed1b06abadbf6b61f6033a07c06f36ec0ddca117e41ef2ac37056705e46458"}, - {file = "regex-2022.4.24-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:3241db067a7f69da57fba8bca543ac8a7ca415d91e77315690202749b9fdaba1"}, - {file = "regex-2022.4.24-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:d128e278e5e554c5c022c7bed410ca851e00bacebbb4460de546a73bc53f8de4"}, - {file = "regex-2022.4.24-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b1d53835922cd0f9b74b2742453a444865a70abae38d12eb41c59271da66f38d"}, - {file = "regex-2022.4.24-cp38-cp38-win32.whl", hash = "sha256:f2a5d9f612091812dee18375a45d046526452142e7b78c4e21ab192db15453d5"}, - {file = "regex-2022.4.24-cp38-cp38-win_amd64.whl", hash = "sha256:a850f5f369f1e3b6239da7fb43d1d029c1e178263df671819889c47caf7e4ff3"}, - {file = "regex-2022.4.24-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bedb3d01ad35ea1745bdb1d57f3ee0f996f988c98f5bbae9d068c3bb3065d210"}, - {file = "regex-2022.4.24-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8bf867ba71856414a482e4b683500f946c300c4896e472e51d3db8dfa8dc8f32"}, - {file = "regex-2022.4.24-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b415b82e5be7389ec5ee7ee35431e4a549ea327caacf73b697c6b3538cb5c87f"}, - {file = "regex-2022.4.24-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9dae5affbb66178dad6c6fd5b02221ca9917e016c75ee3945e9a9563eb1fbb6f"}, - {file = "regex-2022.4.24-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e65580ae3137bce712f505ec7c2d700aef0014a3878c4767b74aff5895fc454f"}, - {file = "regex-2022.4.24-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e9e983fc8e0d4d5ded7caa5aed39ca2cf6026d7e39801ef6f0af0b1b6cd9276"}, - {file = "regex-2022.4.24-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cfad3a770839aa456ff9a9aa0e253d98b628d005a3ccb37da1ff9be7c84fee16"}, - {file = "regex-2022.4.24-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ed625205f5f26984382b68e4cbcbc08e6603c9e84c14b38457170b0cc71c823b"}, - {file = "regex-2022.4.24-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c4fdf837666f7793a5c3cfa2f2f39f03eb6c7e92e831bc64486c2f547580c2b3"}, - {file = "regex-2022.4.24-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ed26c3d2d62c6588e0dad175b8d8cc0942a638f32d07b80f92043e5d73b7db67"}, - {file = "regex-2022.4.24-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:f89d26e50a4c7453cb8c415acd09e72fbade2610606a9c500a1e48c43210a42d"}, - {file = "regex-2022.4.24-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:97af238389cb029d63d5f2d931a7e8f5954ad96e812de5faaed373b68e74df86"}, - {file = "regex-2022.4.24-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:be392d9cd5309509175a9d7660dc17bf57084501108dbff0c5a8bfc3646048c3"}, - {file = "regex-2022.4.24-cp39-cp39-win32.whl", hash = "sha256:bcc6f7a3a95119c3568c572ca167ada75f8319890706283b9ba59b3489c9bcb3"}, - {file = "regex-2022.4.24-cp39-cp39-win_amd64.whl", hash = "sha256:5b9c7b6895a01204296e9523b3e12b43e013835a9de035a783907c2c1bc447f0"}, - {file = "regex-2022.4.24.tar.gz", hash = "sha256:92183e9180c392371079262879c6532ccf55f808e6900df5d9f03c9ca8807255"}, + {file = "regex-2022.8.17-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:840063aa8eeb1dda07d7d7dee15648838bffef1d415f5f79061854a182a429aa"}, + {file = "regex-2022.8.17-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1df31eaf147ecff3665ba861acb8f78221cd5501df072c9151dfa341dd24599f"}, + {file = "regex-2022.8.17-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:085ca3dc9360c0210e0a70e5d34d66454a06077644e7679fef6358b1f053e62e"}, + {file = "regex-2022.8.17-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:21b6f939916aa61beea56393ebc8a9999060632ac22b8193c2cb67d6fd7cb2c3"}, + {file = "regex-2022.8.17-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0a9d5a64e974bc5f160f30f76aaf993d49eeddb405676be6bf76a5a2c131e185"}, + {file = "regex-2022.8.17-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d13bd83284b46c304eb10de93f8a3f2c80361f91f4e8a4e1273caf83e16c4409"}, + {file = "regex-2022.8.17-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7a52d547259495a53e61e37ffc6d5cecf8d298aeb1bc0d9b25289d65ddb31183"}, + {file = "regex-2022.8.17-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:be6f5b453f7ed2219a9555bb6840663950b9ab1dc034216f68eac64db66633c2"}, + {file = "regex-2022.8.17-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:a1e283ad918df44bad3ccf042c2fe283c63d17617570eb91b8c370ef677b0b83"}, + {file = "regex-2022.8.17-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5d541bc430a74c787684d1ebcd205a5212a88c3de73848143e77489b2c25b911"}, + {file = "regex-2022.8.17-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:c78c72f7878071a78337510ec78ab856d60b4bdcd3a95fd68b939e7cb30434b3"}, + {file = "regex-2022.8.17-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:6b30c8d299ba48ee919064628fd8bc296bdc6e4827d315491bea39437130d3e1"}, + {file = "regex-2022.8.17-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:02b6dc102123f5178796dcdb5a90f6e88895607fd1a1d115d8de1af8161ca2b4"}, + {file = "regex-2022.8.17-cp310-cp310-win32.whl", hash = "sha256:5f14430535645712f546f1e07013507d1cc0c8abd851811dacce8c7fb584bf52"}, + {file = "regex-2022.8.17-cp310-cp310-win_amd64.whl", hash = "sha256:c4f6609f6e867a58cdf173e1cbe1f3736d25962108bd5cb01ad5a130875ff2c8"}, + {file = "regex-2022.8.17-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:4dad9d68574e93e1e23be53b4ecfb0f083bd5cc08cc7f1984a4ee3ebf12aa446"}, + {file = "regex-2022.8.17-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62d56a9d3c1e5a83076db4da060dad7ea35ac2f3cbd3c53ba5a51fe0caedb500"}, + {file = "regex-2022.8.17-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:61f6966371fa1cbf26c6209771a02bef80336cdaca0c0af4dfa33d51019c0b93"}, + {file = "regex-2022.8.17-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c76dd2c0615a28de21c97f9f6862e84faef58ff4d700196b4e395ef6a52291e4"}, + {file = "regex-2022.8.17-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:370b1d7aed26e29915c3fb3e72e327f194824a76cedb60c0b9f6c6af53e89d72"}, + {file = "regex-2022.8.17-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:634f090a388351eadf1dcc1d168a190718fb68efb4b8fdc1b119cf837ca01905"}, + {file = "regex-2022.8.17-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:79f34d5833cd0d53ecf48bc030e4da3216bd4846224d17eeb64509be5cb098fd"}, + {file = "regex-2022.8.17-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:b7ddecc80e87acf12c2cf12bf3721def47188c403f04e706f104b5e71fed2f31"}, + {file = "regex-2022.8.17-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:6f62c8a59f6b8e608880c61b138ae22668184bc266b025d33200dcf2cebe0872"}, + {file = "regex-2022.8.17-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:95fb62a3980cf43e76c2fe95edab06ec70dc495b8aa660975eb9f0b2ffdae1e1"}, + {file = "regex-2022.8.17-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:0de0ce11c0835e1117eacbfe8fa6fa98dc0e8e746b486735cb0fdebe46a02222"}, + {file = "regex-2022.8.17-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:abe1adb32e2535aaa171e8b2b2d3f083f863c9974a3e6e7dae6bf4827fc8b983"}, + {file = "regex-2022.8.17-cp36-cp36m-win32.whl", hash = "sha256:6059ae91667932d256d9dc03abd3512ebcade322b3a42d1b8354bd1db7f66dcc"}, + {file = "regex-2022.8.17-cp36-cp36m-win_amd64.whl", hash = "sha256:6af38997f178889d417851bae8fb5c00448f7405cfcab38734d771f1dd5d5973"}, + {file = "regex-2022.8.17-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:cfa62063c5eafb04e4435459ce15746b4ae6c14efeae8f16bd0e3d2895dad698"}, + {file = "regex-2022.8.17-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64ecfcc386420192fbe98fdde777d993f7f2dfec9552e4f4024d3447d3a3e637"}, + {file = "regex-2022.8.17-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5910bb355f9517309f77101238dbacb7151ede3434a2f1fad26ecc62f13d8324"}, + {file = "regex-2022.8.17-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ae85112da2d826b65aa7c7369c56ca41d9a89644312172979cbee5cf788e0b09"}, + {file = "regex-2022.8.17-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f0c8807bac16984901c0573725bad786f2f004f9bd5df8476c6431097b6c5b3"}, + {file = "regex-2022.8.17-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:53c9eca0d6070a8a3de42182ad26daf90ba12132eb74a2f45702332762aff84e"}, + {file = "regex-2022.8.17-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e37886929ee83a5fa5c73164abada00e7f3cc1cbf3f8f6e1e8cfecae9d6cfc47"}, + {file = "regex-2022.8.17-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b3379a83dc63fe06538c751961f9ed730b5d7f08f96a57bbad8d52db5820df1f"}, + {file = "regex-2022.8.17-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:3d3d769b3d485b28d6a591b46723dbacc696e6503f48a3ef52e6fc2c90edb482"}, + {file = "regex-2022.8.17-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:fafed60103132e74cdfbd651abe94801eb87a9765ce275b3dca9af8f3e06622a"}, + {file = "regex-2022.8.17-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:14750172c0a616140a8f496dfef28ed24080e87d06d5838e008f959ad307a8c5"}, + {file = "regex-2022.8.17-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:3aafbbf5076f2a48bcf31ceb42b410323daaa0ddb42544640592957bc906ace6"}, + {file = "regex-2022.8.17-cp37-cp37m-win32.whl", hash = "sha256:74d4aabd612d32282f3cb3ebb4436046fb840d25c754157a755bc9f66e7cd307"}, + {file = "regex-2022.8.17-cp37-cp37m-win_amd64.whl", hash = "sha256:4bd9443f7ff6e6288dd4496215c5d903f851e55cbc09d5963587af0c6d565a0a"}, + {file = "regex-2022.8.17-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b3c7c6c4aac19b964c1d12784aecae7f0315314640b0f41dd6f0d4e2bf439072"}, + {file = "regex-2022.8.17-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:bbaf6785d3f1cd3e617b9d0fb3c5528023ef7bc7cc1356234801dc1941df8ce9"}, + {file = "regex-2022.8.17-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d83fd6dd4263595d0e4f595d4abd54397cbed52c0147f7dd148a7b72910301e"}, + {file = "regex-2022.8.17-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1b6d2c579ffdcbb3d93f63b6a7f697364594e1c1b6856958b3e61e3ca22c140a"}, + {file = "regex-2022.8.17-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8e8ec94d1b1a0a297c2c69a0bf000baf9a79607ca0c084f577f811a9b447c319"}, + {file = "regex-2022.8.17-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4bdfd016ab12c4075ef93f025b3cf4c8962b9b7a5e52bb7039ab64cb7755930c"}, + {file = "regex-2022.8.17-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cb0c9a1476d279524538ba9a00ecec9eadcef31a6a60b2c8bd2f29f62044a559"}, + {file = "regex-2022.8.17-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:25bffa248b99b53a61b1f20fc7d19f711e38e9f0bc90d44c26670f8dc282ad7d"}, + {file = "regex-2022.8.17-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:0843cc977b9cc00eb2299b624db6481d25e7f5b093f7a7c2bb727028d4a26eda"}, + {file = "regex-2022.8.17-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4e12a3c2d4781ee5d03f229c940934fa1e4ea4f4995e68ab97a2815b139e0804"}, + {file = "regex-2022.8.17-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:dc32029b9cc784a529f9201289d4f841cc24a2ae3126a112cd467bc41bbc2f10"}, + {file = "regex-2022.8.17-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:4c6554073e3e554fbb3dff88376ada3da32ca789ea1b9e381f684d49ddb61199"}, + {file = "regex-2022.8.17-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2ada67e02fa3fcca9e3b90cf24c2c6bc77f0abc126209937956aea10eeba40c7"}, + {file = "regex-2022.8.17-cp38-cp38-win32.whl", hash = "sha256:1418d3506a9582b23a27373f125ea2b0da523c581e7cf678a6f036254d134faa"}, + {file = "regex-2022.8.17-cp38-cp38-win_amd64.whl", hash = "sha256:2c198921afc811bc0f105c6e5150fbdebf9520c9b7d43cfc0ab156ca97f506d7"}, + {file = "regex-2022.8.17-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7658d2dfc1dabfb008ffe12ae47b98559e2aedd8237bee12f5aafb74d90479e3"}, + {file = "regex-2022.8.17-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:242f546fc5e49bb7395624ac3b4fc168bf454e11ace9804c58c4c3a90d84e38f"}, + {file = "regex-2022.8.17-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f7b88bc7306136b123fd1a9beed16ca02900ee31d1c36e73fa33d9e525a5562d"}, + {file = "regex-2022.8.17-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2240fce3af236e4586a045c1be8bbf16c4f8831e68b7df918b72fc31a80143be"}, + {file = "regex-2022.8.17-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e0b55651db770b4b5a6c7d015f24d1a6ede307296bbdf0c47fc5f6a6adc7abee"}, + {file = "regex-2022.8.17-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9668da78bcc219542467f51c2cd01894222be6aceec4b5efb806705900b794d8"}, + {file = "regex-2022.8.17-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5e7c8f9f8824143c219dd93cdc733c20d2c12f154034c89bcb4911db8e45bd92"}, + {file = "regex-2022.8.17-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c2b6404631b22617b5127c6de2355393ccda693ca733a098b6802e7dabb3457a"}, + {file = "regex-2022.8.17-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:45cb798095b886e4df6ff4a1f7661eb70620ccdef127e3c3e00a1aaa22d30e53"}, + {file = "regex-2022.8.17-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:777ceea2860a48e9e362a4e2a9a691782ea97bd05c24627c92e876fdd2c22e61"}, + {file = "regex-2022.8.17-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:99a7c5786de9e92ff5ffee2e8bed745f5d25495206f3f14656c379031e518334"}, + {file = "regex-2022.8.17-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:d76e585368388d99ddd2f95989e6ac80a8fe23115e93931faad99fa34550612f"}, + {file = "regex-2022.8.17-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a25d251546acb5edb1635631c4ae0e330fa4ec7c6316c01d256728fbfb9bbff2"}, + {file = "regex-2022.8.17-cp39-cp39-win32.whl", hash = "sha256:fac611bde2609a46fcbd92da7171286faa2f5c191f84d22f61cd7dc27213f51d"}, + {file = "regex-2022.8.17-cp39-cp39-win_amd64.whl", hash = "sha256:ccb986e80674c929f198464bce55e995178dea26833421e2479ff04a6956afac"}, + {file = "regex-2022.8.17.tar.gz", hash = "sha256:5c77eab46f3a2b2cd8bbe06467df783543bf7396df431eb4a144cc4b89e9fb3c"}, ] requests = [ - {file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"}, - {file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"}, + {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"}, + {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"}, ] "ruamel.yaml" = [ {file = "ruamel.yaml-0.16.13-py2.py3-none-any.whl", hash = "sha256:64b06e7873eb8e1125525ecef7345447d786368cadca92a7cd9b59eae62e95a3"}, @@ -1883,51 +1912,21 @@ toml = [ {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] tomlkit = [ - {file = "tomlkit-0.11.0-py3-none-any.whl", hash = "sha256:0f4050db66fd445b885778900ce4dd9aea8c90c4721141fde0d6ade893820ef1"}, - {file = "tomlkit-0.11.0.tar.gz", hash = "sha256:71ceb10c0eefd8b8f11fe34e8a51ad07812cb1dc3de23247425fbc9ddc47b9dd"}, + {file = "tomlkit-0.11.4-py3-none-any.whl", hash = "sha256:25d4e2e446c453be6360c67ddfb88838cfc42026322770ba13d1fbd403a93a5c"}, + {file = "tomlkit-0.11.4.tar.gz", hash = "sha256:3235a9010fae54323e727c3ac06fb720752fe6635b3426e379daec60fbd44a83"}, ] tornado = [ - {file = "tornado-6.1-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:d371e811d6b156d82aa5f9a4e08b58debf97c302a35714f6f45e35139c332e32"}, - {file = "tornado-6.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:0d321a39c36e5f2c4ff12b4ed58d41390460f798422c4504e09eb5678e09998c"}, - {file = "tornado-6.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9de9e5188a782be6b1ce866e8a51bc76a0fbaa0e16613823fc38e4fc2556ad05"}, - {file = "tornado-6.1-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:61b32d06ae8a036a6607805e6720ef00a3c98207038444ba7fd3d169cd998910"}, - {file = "tornado-6.1-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:3e63498f680547ed24d2c71e6497f24bca791aca2fe116dbc2bd0ac7f191691b"}, - {file = "tornado-6.1-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:6c77c9937962577a6a76917845d06af6ab9197702a42e1346d8ae2e76b5e3675"}, - {file = "tornado-6.1-cp35-cp35m-win32.whl", hash = "sha256:6286efab1ed6e74b7028327365cf7346b1d777d63ab30e21a0f4d5b275fc17d5"}, - {file = "tornado-6.1-cp35-cp35m-win_amd64.whl", hash = "sha256:fa2ba70284fa42c2a5ecb35e322e68823288a4251f9ba9cc77be04ae15eada68"}, - {file = "tornado-6.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:0a00ff4561e2929a2c37ce706cb8233b7907e0cdc22eab98888aca5dd3775feb"}, - {file = "tornado-6.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:748290bf9112b581c525e6e6d3820621ff020ed95af6f17fedef416b27ed564c"}, - {file = "tornado-6.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:e385b637ac3acaae8022e7e47dfa7b83d3620e432e3ecb9a3f7f58f150e50921"}, - {file = "tornado-6.1-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:25ad220258349a12ae87ede08a7b04aca51237721f63b1808d39bdb4b2164558"}, - {file = "tornado-6.1-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:65d98939f1a2e74b58839f8c4dab3b6b3c1ce84972ae712be02845e65391ac7c"}, - {file = "tornado-6.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:e519d64089b0876c7b467274468709dadf11e41d65f63bba207e04217f47c085"}, - {file = "tornado-6.1-cp36-cp36m-win32.whl", hash = "sha256:b87936fd2c317b6ee08a5741ea06b9d11a6074ef4cc42e031bc6403f82a32575"}, - {file = "tornado-6.1-cp36-cp36m-win_amd64.whl", hash = "sha256:cc0ee35043162abbf717b7df924597ade8e5395e7b66d18270116f8745ceb795"}, - {file = "tornado-6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7250a3fa399f08ec9cb3f7b1b987955d17e044f1ade821b32e5f435130250d7f"}, - {file = "tornado-6.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:ed3ad863b1b40cd1d4bd21e7498329ccaece75db5a5bf58cd3c9f130843e7102"}, - {file = "tornado-6.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:dcef026f608f678c118779cd6591c8af6e9b4155c44e0d1bc0c87c036fb8c8c4"}, - {file = "tornado-6.1-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:70dec29e8ac485dbf57481baee40781c63e381bebea080991893cd297742b8fd"}, - {file = "tornado-6.1-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:d3f7594930c423fd9f5d1a76bee85a2c36fd8b4b16921cae7e965f22575e9c01"}, - {file = "tornado-6.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:3447475585bae2e77ecb832fc0300c3695516a47d46cefa0528181a34c5b9d3d"}, - {file = "tornado-6.1-cp37-cp37m-win32.whl", hash = "sha256:e7229e60ac41a1202444497ddde70a48d33909e484f96eb0da9baf8dc68541df"}, - {file = "tornado-6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:cb5ec8eead331e3bb4ce8066cf06d2dfef1bfb1b2a73082dfe8a161301b76e37"}, - {file = "tornado-6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:20241b3cb4f425e971cb0a8e4ffc9b0a861530ae3c52f2b0434e6c1b57e9fd95"}, - {file = "tornado-6.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:c77da1263aa361938476f04c4b6c8916001b90b2c2fdd92d8d535e1af48fba5a"}, - {file = "tornado-6.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:fba85b6cd9c39be262fcd23865652920832b61583de2a2ca907dbd8e8a8c81e5"}, - {file = "tornado-6.1-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:1e8225a1070cd8eec59a996c43229fe8f95689cb16e552d130b9793cb570a288"}, - {file = "tornado-6.1-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:d14d30e7f46a0476efb0deb5b61343b1526f73ebb5ed84f23dc794bdb88f9d9f"}, - {file = "tornado-6.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8f959b26f2634a091bb42241c3ed8d3cedb506e7c27b8dd5c7b9f745318ddbb6"}, - {file = "tornado-6.1-cp38-cp38-win32.whl", hash = "sha256:34ca2dac9e4d7afb0bed4677512e36a52f09caa6fded70b4e3e1c89dbd92c326"}, - {file = "tornado-6.1-cp38-cp38-win_amd64.whl", hash = "sha256:6196a5c39286cc37c024cd78834fb9345e464525d8991c21e908cc046d1cc02c"}, - {file = "tornado-6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f0ba29bafd8e7e22920567ce0d232c26d4d47c8b5cf4ed7b562b5db39fa199c5"}, - {file = "tornado-6.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:33892118b165401f291070100d6d09359ca74addda679b60390b09f8ef325ffe"}, - {file = "tornado-6.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:7da13da6f985aab7f6f28debab00c67ff9cbacd588e8477034c0652ac141feea"}, - {file = "tornado-6.1-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:e0791ac58d91ac58f694d8d2957884df8e4e2f6687cdf367ef7eb7497f79eaa2"}, - {file = "tornado-6.1-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:66324e4e1beede9ac79e60f88de548da58b1f8ab4b2f1354d8375774f997e6c0"}, - {file = "tornado-6.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:a48900ecea1cbb71b8c71c620dee15b62f85f7c14189bdeee54966fbd9a0c5bd"}, - {file = "tornado-6.1-cp39-cp39-win32.whl", hash = "sha256:d3d20ea5782ba63ed13bc2b8c291a053c8d807a8fa927d941bd718468f7b950c"}, - {file = "tornado-6.1-cp39-cp39-win_amd64.whl", hash = "sha256:548430be2740e327b3fe0201abe471f314741efcb0067ec4f2d7dcfb4825f3e4"}, - {file = "tornado-6.1.tar.gz", hash = "sha256:33c6e81d7bd55b468d2e793517c909b139960b6c790a60b7991b9b6b76fb9791"}, + {file = "tornado-6.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:20f638fd8cc85f3cbae3c732326e96addff0a15e22d80f049e00121651e82e72"}, + {file = "tornado-6.2-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:87dcafae3e884462f90c90ecc200defe5e580a7fbbb4365eda7c7c1eb809ebc9"}, + {file = "tornado-6.2-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba09ef14ca9893954244fd872798b4ccb2367c165946ce2dd7376aebdde8e3ac"}, + {file = "tornado-6.2-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b8150f721c101abdef99073bf66d3903e292d851bee51910839831caba341a75"}, + {file = "tornado-6.2-cp37-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d3a2f5999215a3a06a4fc218026cd84c61b8b2b40ac5296a6db1f1451ef04c1e"}, + {file = "tornado-6.2-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:5f8c52d219d4995388119af7ccaa0bcec289535747620116a58d830e7c25d8a8"}, + {file = "tornado-6.2-cp37-abi3-musllinux_1_1_i686.whl", hash = "sha256:6fdfabffd8dfcb6cf887428849d30cf19a3ea34c2c248461e1f7d718ad30b66b"}, + {file = "tornado-6.2-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:1d54d13ab8414ed44de07efecb97d4ef7c39f7438cf5e976ccd356bebb1b5fca"}, + {file = "tornado-6.2-cp37-abi3-win32.whl", hash = "sha256:5c87076709343557ef8032934ce5f637dbb552efa7b21d08e89ae7619ed0eb23"}, + {file = "tornado-6.2-cp37-abi3-win_amd64.whl", hash = "sha256:e5f923aa6a47e133d1cf87d60700889d7eae68988704e20c75fb2d65677a8e4b"}, + {file = "tornado-6.2.tar.gz", hash = "sha256:9b630419bde84ec666bfd7ea0a4cb2a8a651c2d5cccdbdd1972a0c859dfc3c13"}, ] transitions = [ {file = "transitions-0.8.11-py2.py3-none-any.whl", hash = "sha256:9525dd9b708b0a54bb4562a06a483d237e75c94547ba9831c81c6872d0ea1522"}, @@ -1971,8 +1970,8 @@ typing-extensions = [ {file = "typing_extensions-3.10.0.2.tar.gz", hash = "sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e"}, ] urllib3 = [ - {file = "urllib3-1.26.9-py2.py3-none-any.whl", hash = "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14"}, - {file = "urllib3-1.26.9.tar.gz", hash = "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e"}, + {file = "urllib3-1.26.12-py2.py3-none-any.whl", hash = "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997"}, + {file = "urllib3-1.26.12.tar.gz", hash = "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e"}, ] urwid = [ {file = "urwid-2.1.2.tar.gz", hash = "sha256:588bee9c1cb208d0906a9f73c613d2bd32c3ed3702012f51efe318a3f2127eae"}, @@ -1994,78 +1993,65 @@ ya-aioclient = [ {file = "ya_aioclient-0.6.4-py3-none-any.whl", hash = "sha256:1d36c2544c2d7629a00a2b1f18b4535e836586cae99bfbf5714ed5ca3f9caa0c"}, ] yarl = [ - {file = "yarl-1.7.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f2a8508f7350512434e41065684076f640ecce176d262a7d54f0da41d99c5a95"}, - {file = "yarl-1.7.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:da6df107b9ccfe52d3a48165e48d72db0eca3e3029b5b8cb4fe6ee3cb870ba8b"}, - {file = "yarl-1.7.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a1d0894f238763717bdcfea74558c94e3bc34aeacd3351d769460c1a586a8b05"}, - {file = "yarl-1.7.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfe4b95b7e00c6635a72e2d00b478e8a28bfb122dc76349a06e20792eb53a523"}, - {file = "yarl-1.7.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c145ab54702334c42237a6c6c4cc08703b6aa9b94e2f227ceb3d477d20c36c63"}, - {file = "yarl-1.7.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1ca56f002eaf7998b5fcf73b2421790da9d2586331805f38acd9997743114e98"}, - {file = "yarl-1.7.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1d3d5ad8ea96bd6d643d80c7b8d5977b4e2fb1bab6c9da7322616fd26203d125"}, - {file = "yarl-1.7.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:167ab7f64e409e9bdd99333fe8c67b5574a1f0495dcfd905bc7454e766729b9e"}, - {file = "yarl-1.7.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:95a1873b6c0dd1c437fb3bb4a4aaa699a48c218ac7ca1e74b0bee0ab16c7d60d"}, - {file = "yarl-1.7.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6152224d0a1eb254f97df3997d79dadd8bb2c1a02ef283dbb34b97d4f8492d23"}, - {file = "yarl-1.7.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:5bb7d54b8f61ba6eee541fba4b83d22b8a046b4ef4d8eb7f15a7e35db2e1e245"}, - {file = "yarl-1.7.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:9c1f083e7e71b2dd01f7cd7434a5f88c15213194df38bc29b388ccdf1492b739"}, - {file = "yarl-1.7.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f44477ae29025d8ea87ec308539f95963ffdc31a82f42ca9deecf2d505242e72"}, - {file = "yarl-1.7.2-cp310-cp310-win32.whl", hash = "sha256:cff3ba513db55cc6a35076f32c4cdc27032bd075c9faef31fec749e64b45d26c"}, - {file = "yarl-1.7.2-cp310-cp310-win_amd64.whl", hash = "sha256:c9c6d927e098c2d360695f2e9d38870b2e92e0919be07dbe339aefa32a090265"}, - {file = "yarl-1.7.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:9b4c77d92d56a4c5027572752aa35082e40c561eec776048330d2907aead891d"}, - {file = "yarl-1.7.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c01a89a44bb672c38f42b49cdb0ad667b116d731b3f4c896f72302ff77d71656"}, - {file = "yarl-1.7.2-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c19324a1c5399b602f3b6e7db9478e5b1adf5cf58901996fc973fe4fccd73eed"}, - {file = "yarl-1.7.2-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3abddf0b8e41445426d29f955b24aeecc83fa1072be1be4e0d194134a7d9baee"}, - {file = "yarl-1.7.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6a1a9fe17621af43e9b9fcea8bd088ba682c8192d744b386ee3c47b56eaabb2c"}, - {file = "yarl-1.7.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8b0915ee85150963a9504c10de4e4729ae700af11df0dc5550e6587ed7891e92"}, - {file = "yarl-1.7.2-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:29e0656d5497733dcddc21797da5a2ab990c0cb9719f1f969e58a4abac66234d"}, - {file = "yarl-1.7.2-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:bf19725fec28452474d9887a128e98dd67eee7b7d52e932e6949c532d820dc3b"}, - {file = "yarl-1.7.2-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:d6f3d62e16c10e88d2168ba2d065aa374e3c538998ed04996cd373ff2036d64c"}, - {file = "yarl-1.7.2-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:ac10bbac36cd89eac19f4e51c032ba6b412b3892b685076f4acd2de18ca990aa"}, - {file = "yarl-1.7.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:aa32aaa97d8b2ed4e54dc65d241a0da1c627454950f7d7b1f95b13985afd6c5d"}, - {file = "yarl-1.7.2-cp36-cp36m-win32.whl", hash = "sha256:87f6e082bce21464857ba58b569370e7b547d239ca22248be68ea5d6b51464a1"}, - {file = "yarl-1.7.2-cp36-cp36m-win_amd64.whl", hash = "sha256:ac35ccde589ab6a1870a484ed136d49a26bcd06b6a1c6397b1967ca13ceb3913"}, - {file = "yarl-1.7.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a467a431a0817a292121c13cbe637348b546e6ef47ca14a790aa2fa8cc93df63"}, - {file = "yarl-1.7.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ab0c3274d0a846840bf6c27d2c60ba771a12e4d7586bf550eefc2df0b56b3b4"}, - {file = "yarl-1.7.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d260d4dc495c05d6600264a197d9d6f7fc9347f21d2594926202fd08cf89a8ba"}, - {file = "yarl-1.7.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fc4dd8b01a8112809e6b636b00f487846956402834a7fd59d46d4f4267181c41"}, - {file = "yarl-1.7.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c1164a2eac148d85bbdd23e07dfcc930f2e633220f3eb3c3e2a25f6148c2819e"}, - {file = "yarl-1.7.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:67e94028817defe5e705079b10a8438b8cb56e7115fa01640e9c0bb3edf67332"}, - {file = "yarl-1.7.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:89ccbf58e6a0ab89d487c92a490cb5660d06c3a47ca08872859672f9c511fc52"}, - {file = "yarl-1.7.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:8cce6f9fa3df25f55521fbb5c7e4a736683148bcc0c75b21863789e5185f9185"}, - {file = "yarl-1.7.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:211fcd65c58bf250fb994b53bc45a442ddc9f441f6fec53e65de8cba48ded986"}, - {file = "yarl-1.7.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c10ea1e80a697cf7d80d1ed414b5cb8f1eec07d618f54637067ae3c0334133c4"}, - {file = "yarl-1.7.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:52690eb521d690ab041c3919666bea13ab9fbff80d615ec16fa81a297131276b"}, - {file = "yarl-1.7.2-cp37-cp37m-win32.whl", hash = "sha256:695ba021a9e04418507fa930d5f0704edbce47076bdcfeeaba1c83683e5649d1"}, - {file = "yarl-1.7.2-cp37-cp37m-win_amd64.whl", hash = "sha256:c17965ff3706beedafd458c452bf15bac693ecd146a60a06a214614dc097a271"}, - {file = "yarl-1.7.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:fce78593346c014d0d986b7ebc80d782b7f5e19843ca798ed62f8e3ba8728576"}, - {file = "yarl-1.7.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c2a1ac41a6aa980db03d098a5531f13985edcb451bcd9d00670b03129922cd0d"}, - {file = "yarl-1.7.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:39d5493c5ecd75c8093fa7700a2fb5c94fe28c839c8e40144b7ab7ccba6938c8"}, - {file = "yarl-1.7.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1eb6480ef366d75b54c68164094a6a560c247370a68c02dddb11f20c4c6d3c9d"}, - {file = "yarl-1.7.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ba63585a89c9885f18331a55d25fe81dc2d82b71311ff8bd378fc8004202ff6"}, - {file = "yarl-1.7.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e39378894ee6ae9f555ae2de332d513a5763276a9265f8e7cbaeb1b1ee74623a"}, - {file = "yarl-1.7.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c0910c6b6c31359d2f6184828888c983d54d09d581a4a23547a35f1d0b9484b1"}, - {file = "yarl-1.7.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6feca8b6bfb9eef6ee057628e71e1734caf520a907b6ec0d62839e8293e945c0"}, - {file = "yarl-1.7.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8300401dc88cad23f5b4e4c1226f44a5aa696436a4026e456fe0e5d2f7f486e6"}, - {file = "yarl-1.7.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:788713c2896f426a4e166b11f4ec538b5736294ebf7d5f654ae445fd44270832"}, - {file = "yarl-1.7.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:fd547ec596d90c8676e369dd8a581a21227fe9b4ad37d0dc7feb4ccf544c2d59"}, - {file = "yarl-1.7.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:737e401cd0c493f7e3dd4db72aca11cfe069531c9761b8ea474926936b3c57c8"}, - {file = "yarl-1.7.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:baf81561f2972fb895e7844882898bda1eef4b07b5b385bcd308d2098f1a767b"}, - {file = "yarl-1.7.2-cp38-cp38-win32.whl", hash = "sha256:ede3b46cdb719c794427dcce9d8beb4abe8b9aa1e97526cc20de9bd6583ad1ef"}, - {file = "yarl-1.7.2-cp38-cp38-win_amd64.whl", hash = "sha256:cc8b7a7254c0fc3187d43d6cb54b5032d2365efd1df0cd1749c0c4df5f0ad45f"}, - {file = "yarl-1.7.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:580c1f15500e137a8c37053e4cbf6058944d4c114701fa59944607505c2fe3a0"}, - {file = "yarl-1.7.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3ec1d9a0d7780416e657f1e405ba35ec1ba453a4f1511eb8b9fbab81cb8b3ce1"}, - {file = "yarl-1.7.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3bf8cfe8856708ede6a73907bf0501f2dc4e104085e070a41f5d88e7faf237f3"}, - {file = "yarl-1.7.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1be4bbb3d27a4e9aa5f3df2ab61e3701ce8fcbd3e9846dbce7c033a7e8136746"}, - {file = "yarl-1.7.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:534b047277a9a19d858cde163aba93f3e1677d5acd92f7d10ace419d478540de"}, - {file = "yarl-1.7.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c6ddcd80d79c96eb19c354d9dca95291589c5954099836b7c8d29278a7ec0bda"}, - {file = "yarl-1.7.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9bfcd43c65fbb339dc7086b5315750efa42a34eefad0256ba114cd8ad3896f4b"}, - {file = "yarl-1.7.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f64394bd7ceef1237cc604b5a89bf748c95982a84bcd3c4bbeb40f685c810794"}, - {file = "yarl-1.7.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:044daf3012e43d4b3538562da94a88fb12a6490652dbc29fb19adfa02cf72eac"}, - {file = "yarl-1.7.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:368bcf400247318382cc150aaa632582d0780b28ee6053cd80268c7e72796dec"}, - {file = "yarl-1.7.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:bab827163113177aee910adb1f48ff7af31ee0289f434f7e22d10baf624a6dfe"}, - {file = "yarl-1.7.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0cba38120db72123db7c58322fa69e3c0efa933040ffb586c3a87c063ec7cae8"}, - {file = "yarl-1.7.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:59218fef177296451b23214c91ea3aba7858b4ae3306dde120224cfe0f7a6ee8"}, - {file = "yarl-1.7.2-cp39-cp39-win32.whl", hash = "sha256:1edc172dcca3f11b38a9d5c7505c83c1913c0addc99cd28e993efeaafdfaa18d"}, - {file = "yarl-1.7.2-cp39-cp39-win_amd64.whl", hash = "sha256:797c2c412b04403d2da075fb93c123df35239cd7b4cc4e0cd9e5839b73f52c58"}, - {file = "yarl-1.7.2.tar.gz", hash = "sha256:45399b46d60c253327a460e99856752009fcee5f5d3c80b2f7c0cae1c38d56dd"}, + {file = "yarl-1.8.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:abc06b97407868ef38f3d172762f4069323de52f2b70d133d096a48d72215d28"}, + {file = "yarl-1.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:07b21e274de4c637f3e3b7104694e53260b5fc10d51fb3ec5fed1da8e0f754e3"}, + {file = "yarl-1.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9de955d98e02fab288c7718662afb33aab64212ecb368c5dc866d9a57bf48880"}, + {file = "yarl-1.8.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ec362167e2c9fd178f82f252b6d97669d7245695dc057ee182118042026da40"}, + {file = "yarl-1.8.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:20df6ff4089bc86e4a66e3b1380460f864df3dd9dccaf88d6b3385d24405893b"}, + {file = "yarl-1.8.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5999c4662631cb798496535afbd837a102859568adc67d75d2045e31ec3ac497"}, + {file = "yarl-1.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed19b74e81b10b592084a5ad1e70f845f0aacb57577018d31de064e71ffa267a"}, + {file = "yarl-1.8.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e4808f996ca39a6463f45182e2af2fae55e2560be586d447ce8016f389f626f"}, + {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2d800b9c2eaf0684c08be5f50e52bfa2aa920e7163c2ea43f4f431e829b4f0fd"}, + {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6628d750041550c5d9da50bb40b5cf28a2e63b9388bac10fedd4f19236ef4957"}, + {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:f5af52738e225fcc526ae64071b7e5342abe03f42e0e8918227b38c9aa711e28"}, + {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:76577f13333b4fe345c3704811ac7509b31499132ff0181f25ee26619de2c843"}, + {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0c03f456522d1ec815893d85fccb5def01ffaa74c1b16ff30f8aaa03eb21e453"}, + {file = "yarl-1.8.1-cp310-cp310-win32.whl", hash = "sha256:ea30a42dc94d42f2ba4d0f7c0ffb4f4f9baa1b23045910c0c32df9c9902cb272"}, + {file = "yarl-1.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:9130ddf1ae9978abe63808b6b60a897e41fccb834408cde79522feb37fb72fb0"}, + {file = "yarl-1.8.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0ab5a138211c1c366404d912824bdcf5545ccba5b3ff52c42c4af4cbdc2c5035"}, + {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0fb2cb4204ddb456a8e32381f9a90000429489a25f64e817e6ff94879d432fc"}, + {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:85cba594433915d5c9a0d14b24cfba0339f57a2fff203a5d4fd070e593307d0b"}, + {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1ca7e596c55bd675432b11320b4eacc62310c2145d6801a1f8e9ad160685a231"}, + {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0f77539733e0ec2475ddcd4e26777d08996f8cd55d2aef82ec4d3896687abda"}, + {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:29e256649f42771829974e742061c3501cc50cf16e63f91ed8d1bf98242e5507"}, + {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7fce6cbc6c170ede0221cc8c91b285f7f3c8b9fe28283b51885ff621bbe0f8ee"}, + {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:59ddd85a1214862ce7c7c66457f05543b6a275b70a65de366030d56159a979f0"}, + {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:12768232751689c1a89b0376a96a32bc7633c08da45ad985d0c49ede691f5c0d"}, + {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:b19255dde4b4f4c32e012038f2c169bb72e7f081552bea4641cab4d88bc409dd"}, + {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6c8148e0b52bf9535c40c48faebb00cb294ee577ca069d21bd5c48d302a83780"}, + {file = "yarl-1.8.1-cp37-cp37m-win32.whl", hash = "sha256:de839c3a1826a909fdbfe05f6fe2167c4ab033f1133757b5936efe2f84904c07"}, + {file = "yarl-1.8.1-cp37-cp37m-win_amd64.whl", hash = "sha256:dd032e8422a52e5a4860e062eb84ac94ea08861d334a4bcaf142a63ce8ad4802"}, + {file = "yarl-1.8.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:19cd801d6f983918a3f3a39f3a45b553c015c5aac92ccd1fac619bd74beece4a"}, + {file = "yarl-1.8.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6347f1a58e658b97b0a0d1ff7658a03cb79bdbda0331603bed24dd7054a6dea1"}, + {file = "yarl-1.8.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7c0da7e44d0c9108d8b98469338705e07f4bb7dab96dbd8fa4e91b337db42548"}, + {file = "yarl-1.8.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5587bba41399854703212b87071c6d8638fa6e61656385875f8c6dff92b2e461"}, + {file = "yarl-1.8.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:31a9a04ecccd6b03e2b0e12e82131f1488dea5555a13a4d32f064e22a6003cfe"}, + {file = "yarl-1.8.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:205904cffd69ae972a1707a1bd3ea7cded594b1d773a0ce66714edf17833cdae"}, + {file = "yarl-1.8.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ea513a25976d21733bff523e0ca836ef1679630ef4ad22d46987d04b372d57fc"}, + {file = "yarl-1.8.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d0b51530877d3ad7a8d47b2fff0c8df3b8f3b8deddf057379ba50b13df2a5eae"}, + {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d2b8f245dad9e331540c350285910b20dd913dc86d4ee410c11d48523c4fd546"}, + {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ab2a60d57ca88e1d4ca34a10e9fb4ab2ac5ad315543351de3a612bbb0560bead"}, + {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:449c957ffc6bc2309e1fbe67ab7d2c1efca89d3f4912baeb8ead207bb3cc1cd4"}, + {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a165442348c211b5dea67c0206fc61366212d7082ba8118c8c5c1c853ea4d82e"}, + {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b3ded839a5c5608eec8b6f9ae9a62cb22cd037ea97c627f38ae0841a48f09eae"}, + {file = "yarl-1.8.1-cp38-cp38-win32.whl", hash = "sha256:c1445a0c562ed561d06d8cbc5c8916c6008a31c60bc3655cdd2de1d3bf5174a0"}, + {file = "yarl-1.8.1-cp38-cp38-win_amd64.whl", hash = "sha256:56c11efb0a89700987d05597b08a1efcd78d74c52febe530126785e1b1a285f4"}, + {file = "yarl-1.8.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e80ed5a9939ceb6fda42811542f31c8602be336b1fb977bccb012e83da7e4936"}, + {file = "yarl-1.8.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6afb336e23a793cd3b6476c30f030a0d4c7539cd81649683b5e0c1b0ab0bf350"}, + {file = "yarl-1.8.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4c322cbaa4ed78a8aac89b2174a6df398faf50e5fc12c4c191c40c59d5e28357"}, + {file = "yarl-1.8.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fae37373155f5ef9b403ab48af5136ae9851151f7aacd9926251ab26b953118b"}, + {file = "yarl-1.8.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5395da939ffa959974577eff2cbfc24b004a2fb6c346918f39966a5786874e54"}, + {file = "yarl-1.8.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:076eede537ab978b605f41db79a56cad2e7efeea2aa6e0fa8f05a26c24a034fb"}, + {file = "yarl-1.8.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d1a50e461615747dd93c099f297c1994d472b0f4d2db8a64e55b1edf704ec1c"}, + {file = "yarl-1.8.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7de89c8456525650ffa2bb56a3eee6af891e98f498babd43ae307bd42dca98f6"}, + {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4a88510731cd8d4befaba5fbd734a7dd914de5ab8132a5b3dde0bbd6c9476c64"}, + {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2d93a049d29df172f48bcb09acf9226318e712ce67374f893b460b42cc1380ae"}, + {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:21ac44b763e0eec15746a3d440f5e09ad2ecc8b5f6dcd3ea8cb4773d6d4703e3"}, + {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:d0272228fabe78ce00a3365ffffd6f643f57a91043e119c289aaba202f4095b0"}, + {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:99449cd5366fe4608e7226c6cae80873296dfa0cde45d9b498fefa1de315a09e"}, + {file = "yarl-1.8.1-cp39-cp39-win32.whl", hash = "sha256:8b0af1cf36b93cee99a31a545fe91d08223e64390c5ecc5e94c39511832a4bb6"}, + {file = "yarl-1.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:de49d77e968de6626ba7ef4472323f9d2e5a56c1d85b7c0e2a190b2173d3b9be"}, + {file = "yarl-1.8.1.tar.gz", hash = "sha256:af887845b8c2e060eb5605ff72b6f2dd2aab7a761379373fd89d314f4752abbf"}, ] zstandard = [ {file = "zstandard-0.14.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ec1a20936484f3804fba4f29f7d8ed67c70e44536b0f0191a13eff4dc61c815c"}, diff --git a/goth_tests/pyproject.toml b/goth_tests/pyproject.toml index c9ee3835d7..9d54d432d0 100644 --- a/goth_tests/pyproject.toml +++ b/goth_tests/pyproject.toml @@ -22,7 +22,7 @@ pytest = "^6.2" pytest-asyncio = "^0.14" #goth = "^0.11" # to use development goth version uncomment below -goth = { git = "https://github.com/golemfactory/goth.git", rev = "cf88916d00b96acdac90933ca12c80c5048065d7" } +goth = { git = "https://github.com/golemfactory/goth.git", rev = "5244075a4742cfdc848b8dfc4084f788be8b2adf" } [tool.poetry.dev-dependencies] black = "^20.8b1" diff --git a/local-build-targz.sh b/local-build-targz.sh index 3fa1961049..44a40f7f1b 100755 --- a/local-build-targz.sh +++ b/local-build-targz.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Example usage `./local-build-targz.sh ubuntu v0.10-rc3` from yagna directory +# Example usage `./local-build-targz.sh ubuntu v0.11.0-rc14` from yagna directory # Result build will be named the same as CI build with tag `pre-rel-local-v0.10-rc3` # You need sudo to install musl and rust musl target. @@ -31,13 +31,19 @@ CURRENT_DIR=`pwd` SUBNET="hybrid" YAGNA_VERSION=${GITHUB_REF} RELEASE_DIR="${CURRENT_DIR}/releases" -BINARY_PATH="${RELEASE_DIR}/golem-provider-linux-${YAGNA_VERSION}.tar.gz" +PROVIDER_BINARY_PATH="${RELEASE_DIR}/golem-provider-linux-${YAGNA_VERSION}.tar.gz" +REQUESTOR_BINARY_PATH="${RELEASE_DIR}/golem-requestor-linux-${YAGNA_VERSION}.tar.gz" echo "" echo "Binaries generated in: ${RELEASE_DIR}" echo "" echo "To update devnet ${SUBNET} run following command from yagna-testnet-scripts/ansible:" -echo "ansible-playbook -i envs/production/${SUBNET} --extra-vars=\"{ya_provider_yagna_url: ${BINARY_PATH}, ya_provider_yagna_version: ${YAGNA_VERSION}}\" play_ya_provider.yml" - - +echo "ansible-playbook -i envs/production/${SUBNET} \ +--extra-vars=\"{\ +ya_provider_yagna_url: ${PROVIDER_BINARY_PATH}, \ +ya_provider_yagna_version: ${YAGNA_VERSION}, \ +checker_yagna_url: ${REQUESTOR_BINARY_PATH}, \ +checker_yagna_version: ${YAGNA_VERSION}\ +}\" \ +play_ya_provider.yml" diff --git a/utils/actix_utils/Cargo.toml b/utils/actix_utils/Cargo.toml index f55a700095..dfea29bf7d 100644 --- a/utils/actix_utils/Cargo.toml +++ b/utils/actix_utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ya-utils-actix" -version = "0.1.1" +version = "0.2.0" authors = ["Golem Factory "] edition = "2018" diff --git a/utils/actix_utils/src/actix_signal.rs b/utils/actix_utils/src/actix_signal.rs index 2237841a88..313536bbf0 100644 --- a/utils/actix_utils/src/actix_signal.rs +++ b/utils/actix_utils/src/actix_signal.rs @@ -30,18 +30,24 @@ where MessageType: Message + std::marker::Send + std::marker::Sync + std::clone::Clone, MessageType::Result: std::marker::Send + std::marker::Sync; -#[allow(dead_code)] -impl SignalSlot +impl Default for SignalSlot where MessageType: Message + std::marker::Send + std::marker::Sync + std::clone::Clone, MessageType::Result: std::marker::Send + std::marker::Sync, { - pub fn new() -> SignalSlot { - SignalSlot:: { + fn default() -> Self { + Self { subscribers: Default::default(), } } +} +#[allow(dead_code)] +impl SignalSlot +where + MessageType: Message + std::marker::Send + std::marker::Sync + std::clone::Clone, + MessageType::Result: std::marker::Send + std::marker::Sync, +{ /// Send signal to all subscribers pub fn send_signal(&self, message: MessageType) -> Result<()> { let subscribers = self.subscribers.lock().unwrap(); diff --git a/utils/actix_utils/src/deadline_checker.rs b/utils/actix_utils/src/deadline_checker.rs index ae967412c3..ed089aaba4 100644 --- a/utils/actix_utils/src/deadline_checker.rs +++ b/utils/actix_utils/src/deadline_checker.rs @@ -59,16 +59,18 @@ pub struct DeadlineChecker { actix_signal_handler!(DeadlineChecker, DeadlineElapsed, callback); -impl DeadlineChecker { - pub fn new() -> DeadlineChecker { - DeadlineChecker { - deadlines: HashMap::new(), +impl Default for DeadlineChecker { + fn default() -> Self { + Self { + deadlines: Default::default(), nearest_deadline: Utc::now() + Duration::weeks(50), - callback: SignalSlot::::new(), - handle: None, + handle: Default::default(), + callback: Default::default(), } } +} +impl DeadlineChecker { fn update_deadline(&mut self, ctx: &mut Context) -> anyhow::Result<()> { let top_deadline = self.top_deadline(); if self.nearest_deadline != top_deadline { @@ -76,7 +78,7 @@ impl DeadlineChecker { ctx.cancel_future(handle); } - let notify_timestamp = top_deadline.clone(); + let notify_timestamp = top_deadline; let wait_duration = (top_deadline - Utc::now()) .max(Duration::milliseconds(1)) .to_std() @@ -109,7 +111,7 @@ impl DeadlineChecker { let mut elapsed = self .deadlines .iter_mut() - .map(|(agreement_id, deadlines)| { + .flat_map(|(agreement_id, deadlines)| { let idx = match deadlines.binary_search_by(|element| element.deadline.cmp(×tamp)) { Ok(idx) => idx + 1, @@ -125,7 +127,6 @@ impl DeadlineChecker { .collect::>() .into_iter() }) - .flatten() .collect::>(); elapsed.sort_by(|dead1, dead2| dead1.deadline.cmp(&dead2.deadline)); @@ -146,8 +147,8 @@ impl DeadlineChecker { .iter() .filter_map(|element| { let dead_vec = element.1; - if dead_vec.len() > 0 { - Some(dead_vec[0].deadline.clone()) + if !dead_vec.is_empty() { + Some(dead_vec[0].deadline) } else { None } @@ -165,7 +166,7 @@ impl Handler for DeadlineChecker { type Result = (); fn handle(&mut self, msg: TrackDeadline, ctx: &mut Context) -> Self::Result { - if let None = self.deadlines.get(&msg.category) { + if self.deadlines.get(&msg.category).is_none() { self.deadlines.insert(msg.category.to_string(), vec![]); } @@ -198,7 +199,7 @@ impl Handler for DeadlineChecker { // We could store inverse mapping from entities to agreements, but there will never // be so many Agreements at the same time, to make it worth. for deadlines in self.deadlines.values_mut() { - if let Some(idx) = deadlines.iter().position(|element| &element.id == &msg.id) { + if let Some(idx) = deadlines.iter().position(|element| element.id == msg.id) { // Or we could remove all earlier entries?? deadlines.remove(idx); any = true; @@ -215,7 +216,7 @@ impl Handler for DeadlineChecker { type Result = (); fn handle(&mut self, msg: StopTrackingCategory, ctx: &mut Context) -> Self::Result { - if let Some(_) = self.deadlines.remove(&msg.category) { + if self.deadlines.remove(&msg.category).is_some() { self.update_deadline(ctx).unwrap() } } @@ -225,6 +226,7 @@ impl Actor for DeadlineChecker { type Context = Context; } +#[allow(clippy::type_complexity)] struct DeadlineFun { callback: Box Pin>> + 'static>, } @@ -294,7 +296,7 @@ mod test { } async fn init_checker(receiver: Addr) -> Addr { - let checker = DeadlineChecker::new().start(); + let checker = DeadlineChecker::default().start(); checker .send(Subscribe::(receiver.recipient())) .await @@ -367,7 +369,7 @@ mod test { checker .send(TrackDeadline { category: "agrrrrr-1".to_string(), - deadline: now + Duration::milliseconds(200) + Duration::milliseconds(1 * i), + deadline: now + Duration::milliseconds(200) + Duration::milliseconds(i), id: i.to_string(), }) .await diff --git a/utils/agreement-utils/Cargo.toml b/utils/agreement-utils/Cargo.toml index 52fcaca71b..4179ba7b63 100644 --- a/utils/agreement-utils/Cargo.toml +++ b/utils/agreement-utils/Cargo.toml @@ -13,7 +13,7 @@ keywords=["golem", "yagna"] default = [] [dependencies] -ya-client-model = "0.4" +ya-client-model = "0.5" regex = "1.5.4" serde = "1.0" diff --git a/utils/agreement-utils/examples/constraints.rs b/utils/agreement-utils/examples/constraints.rs index 36d27def0c..745d0edd4b 100644 --- a/utils/agreement-utils/examples/constraints.rs +++ b/utils/agreement-utils/examples/constraints.rs @@ -34,10 +34,10 @@ fn main() { ], ); let constraints_and = constraints_1.clone().and(constraints_2.clone()); - let constraints_or = constraints_1.clone().or(constraints_2.clone()); + let constraints_or = constraints_1.clone().or(constraints_2); - println!("And:\n{}", constraints_and.to_string()); - println!("Or:\n{}", constraints_or.to_string()); + println!("And:\n{}", constraints_and); + println!("Or:\n{}", constraints_or); println!("Iteration:"); for expr in constraints_1 { diff --git a/utils/agreement-utils/src/agreement.rs b/utils/agreement-utils/src/agreement.rs index c1753b9fda..ede5a4fb07 100644 --- a/utils/agreement-utils/src/agreement.rs +++ b/utils/agreement-utils/src/agreement.rs @@ -14,7 +14,7 @@ const DEFAULT_FORMAT: &str = "json"; // - 2 fields for parsed properties (demand, offer) // - other fields for agreement remain typed. // TODO: Move to ya-client to make it available for third party developers. -#[derive(Serialize, Deserialize, Clone, Debug)] +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] pub struct AgreementView { pub json: Value, pub agreement_id: String, @@ -31,12 +31,11 @@ impl AgreementView { pub fn pointer_typed<'a, T: Deserialize<'a>>(&self, pointer: &str) -> Result { let value = self - .json .pointer(pointer) - .ok_or(Error::NoKey(pointer.to_string()))? + .ok_or_else(|| Error::NoKey(pointer.to_string()))? .clone(); - Ok(::deserialize(value) - .map_err(|error| Error::UnexpectedType(pointer.to_string(), error))?) + ::deserialize(value) + .map_err(|error| Error::UnexpectedType(pointer.to_string(), error)) } pub fn properties<'a, T: Deserialize<'a>>( @@ -45,7 +44,7 @@ impl AgreementView { ) -> Result, Error> { let value = self .pointer(pointer) - .ok_or(Error::NoKey(pointer.to_string()))?; + .ok_or_else(|| Error::NoKey(pointer.to_string()))?; let map = flatten(value.clone()) .into_iter() @@ -64,29 +63,43 @@ impl AgreementView { } pub fn get_property<'a, T: Deserialize<'a>>(&self, property: &str) -> Result { - let pointer = format!("/{}", property.replace(".", "/")); - self.pointer_typed(pointer.as_str()) + let pointers = property_to_pointer_paths(property); + match self.pointer_typed(&pointers.path_w_tag) { + Err(Error::NoKey(_)) => self.pointer_typed(&pointers.path), + result => result, + } } pub fn remove_property(&mut self, pointer: &str) -> Result<(), Error> { let path: Vec<&str> = pointer.split('/').collect(); - Ok( - // Path should start with '/', so we must omit first element, which will be empty. - remove_property_impl(&mut self.json, &path[1..]).map_err(|e| match e { - Error::NoKey(_) => Error::NoKey(pointer.to_string()), - _ => e, - })?, - ) + // Path should start with '/', so we must omit first element, which will be empty. + remove_property_impl(&mut self.json, &path[1..]).map_err(|e| match e { + Error::NoKey(_) => Error::NoKey(pointer.to_string()), + _ => e, + }) } } +struct PointerPaths { + /// Pointer path + path: String, + /// Pointer path ending with `PROPERTY_TAG` + path_w_tag: String, +} + +fn property_to_pointer_paths(property: &str) -> PointerPaths { + let path = format!("/{}", property.replace('.', "/")); + let path_w_tag = format!("{path}/{PROPERTY_TAG}"); + PointerPaths { path, path_w_tag } +} + pub fn parse_constraints(input: &str, reg_expr: &str, group: usize) -> Option> { let re = regex::Regex::new(reg_expr).unwrap(); re.captures(input) .and_then(|cap| cap.get(group)) .map(|mat| { mat.as_str() - .split(",") + .split(',') .map(|s| s.trim().to_lowercase()) .collect::>() }) @@ -100,7 +113,7 @@ fn remove_property_impl(value: &mut serde_json::Value, path: &[&str]) -> Result< } else { let nested_value = value .pointer_mut(&["/", path[0]].concat()) - .ok_or(Error::NoKey(path[0].to_string()))?; + .ok_or_else(|| Error::NoKey(path[0].to_string()))?; remove_property_impl(nested_value, &path[1..])?; // Check if nested_value contains anything else. @@ -130,7 +143,7 @@ fn remove_value(value: &mut Value, name: &str) -> Result { ), Value::Object(object) => object .remove(name) - .ok_or(Error::InvalidValue(name.to_string()))?, + .ok_or_else(|| Error::InvalidValue(name.to_string()))?, _ => Err(Error::InvalidValue(name.to_string()))?, }) } @@ -167,7 +180,7 @@ impl TryFrom<&Agreement> for AgreementView { } } -impl<'a> std::fmt::Display for AgreementView { +impl std::fmt::Display for AgreementView { fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), FormatError> { let mut agreement = self.json.clone(); @@ -201,7 +214,7 @@ impl Default for OfferTemplate { } } -impl<'a> std::fmt::Display for OfferTemplate { +impl std::fmt::Display for OfferTemplate { fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), FormatError> { let mut template = self.clone(); template.properties = flatten_value(template.properties); @@ -277,7 +290,7 @@ impl TypedPointer for Option<&Value> { { self.map(f) .flatten() - .ok_or(Error::InvalidValue(format!("{:?}", self))) + .ok_or_else(|| Error::InvalidValue(format!("{:?}", self))) } } @@ -294,19 +307,17 @@ impl TypedArrayPointer for Option<&Value> { { let r: Option, Error>> = self.map(Value::as_array).flatten().map(|v| { v.iter() - .map(|i| f(i).ok_or(Error::InvalidValue(format!("{:?}", i)))) + .map(|i| f(i).ok_or_else(|| Error::InvalidValue(format!("{:?}", i)))) .collect::, Error>>() }); - r.ok_or(Error::InvalidValue( - "Unable to convert to an array".to_string(), - ))? + r.ok_or_else(|| Error::InvalidValue("Unable to convert to an array".to_string()))? } } pub fn try_from_path(path: &PathBuf) -> Result { let contents = std::fs::read_to_string(&path).map_err(Error::from)?; - let ext = match path.extension().map(|e| e.to_str()).flatten() { + let ext = match path.extension().and_then(|e| e.to_str()) { Some(ext) => ext, None => DEFAULT_FORMAT, }; @@ -386,7 +397,7 @@ fn merge_obj(a: &mut Value, b: Value) { Value::Null => (), _ => { let a = a.as_object_mut().unwrap(); - a.insert(PROPERTY_TAG.to_string(), b.clone()); + a.insert(PROPERTY_TAG.to_string(), b); } }, (a, b) => *a = b, @@ -517,12 +528,10 @@ constraints: | "#; fn check_values(o: &serde_json::Value) { - assert_eq!( - o.pointer("/properties/golem/srv/caps/multi-activity") - .as_typed(Value::as_bool) - .unwrap(), - true - ); + assert!(o + .pointer("/properties/golem/srv/caps/multi-activity") + .as_typed(Value::as_bool) + .unwrap()); assert_eq!( o.pointer("/properties/golem/inf/mem/gib") .as_typed(Value::as_f64) diff --git a/utils/agreement-utils/src/constraints.rs b/utils/agreement-utils/src/constraints.rs index c1d074abf6..758cfab562 100644 --- a/utils/agreement-utils/src/constraints.rs +++ b/utils/agreement-utils/src/constraints.rs @@ -74,9 +74,9 @@ impl fmt::Display for Constraints { 0 => Ok(()), 1 => write!(f, "{}", self.constraints[0]), _ => { - write!(f, "({}\n", self.operator.to_string())?; + writeln!(f, "({}", self.operator)?; for el in &self.constraints { - write!(f, " {}\n", el.to_string().replace("\n", "\n "))?; + writeln!(f, " {}", el.to_string().replace('\n', "\n "))?; } write!(f, ")") } @@ -93,7 +93,7 @@ impl std::iter::IntoIterator for Constraints { } } -#[derive(Copy, Clone, PartialEq)] +#[derive(Copy, Clone, PartialEq, Eq)] pub enum ClauseOperator { And, Or, @@ -135,7 +135,7 @@ impl fmt::Display for ConstraintOperator { } } -#[derive(Clone, PartialEq)] +#[derive(Clone, PartialEq, Eq)] pub struct ConstraintKey(serde_json::Value); impl ConstraintKey { @@ -182,19 +182,19 @@ impl fmt::Display for ConstraintExpr { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { match self { ConstraintExpr::KeyValue { key, ops_values } => { - if ops_values.len() == 0 { + if ops_values.is_empty() { write!(f, "({})", key.0.as_str().unwrap_or(&key.0.to_string())) } else { for (op, val) in ops_values { write!(f, "({}", key.0.as_str().unwrap_or(&key.0.to_string()))?; - write!(f, "{}", op.to_string())?; + write!(f, "{}", op)?; write!(f, "{}", val.0.as_str().unwrap_or(&val.0.to_string()))?; write!(f, ")")? } Ok(()) } } - ConstraintExpr::Constraints(c) => write!(f, "{}", c.to_string()), + ConstraintExpr::Constraints(c) => write!(f, "{}", c), } } } diff --git a/utils/agreement-utils/src/typed_props.rs b/utils/agreement-utils/src/typed_props.rs index 2270df329c..b232b27ba2 100644 --- a/utils/agreement-utils/src/typed_props.rs +++ b/utils/agreement-utils/src/typed_props.rs @@ -35,6 +35,7 @@ pub struct NodeInfo { pub name: Option, pub subnet: Option, pub geo_country_code: Option, + pub is_public: bool, } impl NodeInfo { @@ -43,6 +44,7 @@ impl NodeInfo { name: Some(name.into()), geo_country_code: None, subnet: None, + is_public: false, } } @@ -62,6 +64,10 @@ impl NodeInfo { if let Some(subnet) = self.subnet { let _ = node.insert("debug".into(), serde_json::json!({ "subnet": subnet })); } + let _ = node.insert( + "net".into(), + serde_json::json!({"is-public": self.is_public}), + ); map.insert("node".into(), node.into()); } } @@ -71,6 +77,7 @@ pub struct ServiceInfo { inf: InfNodeInfo, exeunit_info: Value, multi_activity: bool, + payload_manifest: bool, } impl ServiceInfo { @@ -79,6 +86,7 @@ impl ServiceInfo { inf, exeunit_info, multi_activity: true, + payload_manifest: true, } } @@ -89,11 +97,23 @@ impl ServiceInfo { } } + pub fn support_payload_manifest(self, payload_manifest: bool) -> Self { + Self { + payload_manifest, + ..self + } + } + fn write_json(self, map: &mut serde_json::Map) { self.inf.write_json(map); let _ = map.insert("runtime".into(), self.exeunit_info); - let srv_map = serde_json::json!({ "caps": {"multi-activity": self.multi_activity}}); + let srv_map = serde_json::json!({ + "caps": { + "multi-activity": self.multi_activity, + "payload-manifest": self.payload_manifest + }, + }); let _ = map.insert("srv".into(), srv_map); } } @@ -186,7 +206,7 @@ pub struct ComInfo { impl ComInfo { fn write_json(self, map: &mut serde_json::Map) { - let _ = map.insert("com".to_string(), self.params.clone()); + let _ = map.insert("com".to_string(), self.params); } } @@ -205,21 +225,34 @@ mod test { use super::*; #[test] - fn test_wasm_1() { + fn offer_json_test() { let offer = OfferDefinition { node_info: NodeInfo::with_name("dany"), srv_info: ServiceInfo { inf: InfNodeInfo::default().with_mem(5.0).with_storage(50.0), exeunit_info: serde_json::json!({"wasm.wasi.version@v".to_string(): "0.9.0".to_string()}), multi_activity: false, + payload_manifest: true, }, com_info: Default::default(), offer: OfferTemplate::default(), }; - eprintln!( - "j={}", - serde_json::to_string_pretty(&offer.into_json()).unwrap() - ); + let offer = serde_json::to_string_pretty(&offer.into_json()).unwrap(); + + let expected_offer = r#" +{ + "golem.com": null, + "golem.inf.mem.gib": 5.0, + "golem.inf.storage.gib": 50.0, + "golem.node.id.name": "dany", + "golem.node.net.is-public": false, + "golem.runtime.wasm.wasi.version@v": "0.9.0", + "golem.srv.caps.multi-activity": false, + "golem.srv.caps.payload-manifest": true +} +"#; + + assert_eq!(expected_offer.trim(), offer) } } diff --git a/utils/cli/Cargo.toml b/utils/cli/Cargo.toml new file mode 100644 index 0000000000..6278eeb600 --- /dev/null +++ b/utils/cli/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "ya-utils-cli" +version = "0.1.0" +authors = ["Golem Factory "] +edition = "2018" +homepage = "https://github.com/golemfactory/yagna" +repository = "https://github.com/golemfactory/yagna" +license = "LGPL-3.0" +description = "Yagna CLI utils" +keywords = ["golem", "yagna"] + +[dependencies] +anyhow = "1.0" +prettytable-rs = "0.9" +serde = "1.0" +serde_json = "1.0" +serde_yaml = "0.9" +lazy_static = "1.4" + +[dev-dependencies] diff --git a/utils/cli/src/cmd.rs b/utils/cli/src/cmd.rs new file mode 100644 index 0000000000..8dbca8e04b --- /dev/null +++ b/utils/cli/src/cmd.rs @@ -0,0 +1,72 @@ +use anyhow::Result; +use serde::Serialize; + +pub enum CommandOutput { + NoOutput, + Object(serde_json::Value), + Table { + columns: Vec, + values: Vec, + summary: Vec, + header: Option, + }, +} + +impl CommandOutput { + pub fn object(value: T) -> Result { + Ok(CommandOutput::Object(serde_json::to_value(value)?)) + } + + pub fn print(&self, json_output: bool) -> Result<()> { + if json_output { + self.print_json()?; + } else { + self.print_plain()?; + } + Ok(()) + } + + fn print_json(&self) -> anyhow::Result<()> { + match self { + CommandOutput::NoOutput => println!("null"), + CommandOutput::Table { + columns, + values, + summary: _, + header: _, + } => crate::table::print_json_table(columns, values)?, + CommandOutput::Object(value) => println!("{}", serde_json::to_string_pretty(&value)?), + } + Ok(()) + } + + fn print_plain(&self) -> anyhow::Result<()> { + match self { + CommandOutput::NoOutput => {} + CommandOutput::Table { + columns, + values, + summary, + header, + } => { + if let Some(txt) = header { + println!("{}", txt); + } + crate::table::print_table(columns, values, summary) + } + CommandOutput::Object(value) => match value { + serde_json::Value::String(s) => { + println!("{}", s); + } + value => println!("{}", serde_yaml::to_string(&value)?), + }, + } + Ok(()) + } +} + +impl From<()> for CommandOutput { + fn from(_: ()) -> Self { + CommandOutput::NoOutput + } +} diff --git a/utils/cli/src/lib.rs b/utils/cli/src/lib.rs new file mode 100644 index 0000000000..810ed97166 --- /dev/null +++ b/utils/cli/src/lib.rs @@ -0,0 +1,5 @@ +mod cmd; +mod table; + +pub use cmd::CommandOutput; +pub use table::ResponseTable; diff --git a/utils/cli/src/table.rs b/utils/cli/src/table.rs new file mode 100644 index 0000000000..fef743fee7 --- /dev/null +++ b/utils/cli/src/table.rs @@ -0,0 +1,183 @@ +use crate::cmd::CommandOutput; +use prettytable::{color, format, format::TableFormat, Attr, Cell, Row, Table}; +use std::collections::HashMap; + +pub fn print_table( + columns: &[String], + values: &Vec, + summary: &Vec, +) { + let mut table = Table::new(); + table.set_format(*FORMAT_BASIC); + + table.set_titles(Row::new( + columns + .iter() + .map(|c| { + Cell::new(c) + .with_style(Attr::Bold) + .with_style(Attr::ForegroundColor(color::GREEN)) + }) + .collect(), + )); + if values.is_empty() { + let _ = table.add_row(columns.iter().map(|_| Cell::new("")).collect()); + } + for row in values { + if let Some(row_items) = row.as_array() { + use serde_json::Value; + + let row_strings = row_items + .iter() + .map(|v| match v { + Value::String(s) => s.to_string(), + Value::Null => "".into(), + v => v.to_string(), + }) + .collect(); + table.add_row(row_strings); + } + } + if !summary.is_empty() { + table.add_row(Row::empty()); + table.add_empty_row(); + let l = summary.len(); + for (idx, row) in summary.iter().enumerate() { + if let Some(row_items) = row.as_array() { + use serde_json::Value; + + let row_strings = Row::new( + row_items + .iter() + .map(|v| { + let c = Cell::new(&match v { + Value::String(s) => s.to_string(), + Value::Null => "".into(), + v => v.to_string(), + }); + + if idx == l - 1 { + c.with_style(Attr::Bold) + } else { + c + } + }) + .collect(), + ); + table.add_row(row_strings); + } + } + } + table.printstd(); +} + +pub fn print_json_table( + columns: &Vec, + values: &Vec, +) -> Result<(), anyhow::Error> { + let columns_size_eq_values_size = values.iter().all(|row| match row { + serde_json::Value::Array(values) => values.len() == columns.len(), + _ => false, + }); + if columns_size_eq_values_size { + let kvs: Vec> = values + .iter() + .map(|row| match row { + serde_json::Value::Array(row_values) if columns.len() == row_values.len() => { + columns + .iter() + .enumerate() + .map(|(idx, key)| (key, &row_values[idx])) + .collect() + } + _ => unreachable!(), + }) + .collect(); + println!("{}", serde_json::to_string_pretty(&serde_json::json!(kvs))?) + } else { + println!( + "{}", + serde_json::to_string_pretty(&serde_json::json!({ + "headers": columns, + "values": values + }))? + ) + } + Ok(()) +} + +pub struct ResponseTable { + pub columns: Vec, + pub values: Vec, +} + +impl ResponseTable { + pub fn sort_by(mut self, arg_key: &Option>) -> Self { + let key = match arg_key { + None => return self, + Some(k) => k.as_ref(), + }; + let idx = + match self + .columns + .iter() + .enumerate() + .find_map(|(idx, v)| if v == key { Some(idx) } else { None }) + { + None => return self, + Some(idx) => idx, + }; + self.values + .sort_by_key(|v| Some(v.as_array()?.get(idx)?.to_string())); + self + } + + pub fn with_summary(self, summary: Vec) -> CommandOutput { + CommandOutput::Table { + columns: self.columns, + values: self.values, + summary, + header: None, + } + } + + pub fn with_header(self, header: String) -> CommandOutput { + CommandOutput::Table { + columns: self.columns, + values: self.values, + summary: Vec::new(), + header: Some(header), + } + } +} + +impl From for CommandOutput { + fn from(table: ResponseTable) -> Self { + CommandOutput::Table { + columns: table.columns, + values: table.values, + summary: Vec::new(), + header: None, + } + } +} + +lazy_static::lazy_static! { + pub static ref FORMAT_BASIC: TableFormat = format::FormatBuilder::new() + .column_separator('│') + .borders('│') + .separators( + &[format::LinePosition::Top], + format::LineSeparator::new('─', '┬', '┌', '┐') + ) + .separators( + &[format::LinePosition::Title], + format::LineSeparator::new('─', '┼', '├', '┤') + ) + .separators( + &[format::LinePosition::Bottom], + format::LineSeparator::new('─', '┴', '└', '┘') + ) + .padding(2, 2) + .build(); +} diff --git a/utils/compile-time-utils/src/lib.rs b/utils/compile-time-utils/src/lib.rs index c00f64e049..3a7fe68b10 100644 --- a/utils/compile-time-utils/src/lib.rs +++ b/utils/compile-time-utils/src/lib.rs @@ -19,7 +19,7 @@ pub fn build_number_str() -> Option<&'static str> { /// Returns Github Actions build number if available or None. pub fn build_number() -> Option { - build_number_str().map(|s| s.parse().ok()).flatten() + build_number_str().and_then(|s| s.parse().ok()) } /// Converts a tag to semantic version diff --git a/utils/file-logging/src/lib.rs b/utils/file-logging/src/lib.rs index 7caff31c2a..a4c2017c16 100644 --- a/utils/file-logging/src/lib.rs +++ b/utils/file-logging/src/lib.rs @@ -10,6 +10,7 @@ use std::path::Path; pub use flexi_logger::LoggerHandle; +#[allow(clippy::useless_conversion)] fn log_format_date(now: &mut DeferredNow) -> DelayedFormat { //use DateTime:: instead of DateTime:: to obtain local date let local_date = DateTime::::from(*now.now()); diff --git a/utils/futures/Cargo.toml b/utils/futures/Cargo.toml index a05b9f968a..e1fab19ccf 100644 --- a/utils/futures/Cargo.toml +++ b/utils/futures/Cargo.toml @@ -1,10 +1,9 @@ [package] name = "ya-utils-futures" -version = "0.1.0" +version = "0.2.0" authors = ["Golem Factory "] edition = "2018" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] tokio = { version = "1", features = ["time"] } diff --git a/utils/futures/src/timeout.rs b/utils/futures/src/timeout.rs index df2c8b4257..843bb97bf4 100644 --- a/utils/futures/src/timeout.rs +++ b/utils/futures/src/timeout.rs @@ -76,7 +76,7 @@ where fn timeout(self, duration: Option) -> Either, MapType> { match duration { Some(d) => Either::Left(timeout(d.into_duration(), self)), - None => Either::Right(self.map(|v| Result::Ok(v))), + None => Either::Right(self.map(Result::Ok)), } } } diff --git a/utils/manifest-utils/Cargo.toml b/utils/manifest-utils/Cargo.toml index 7429ef3312..db05772ae8 100644 --- a/utils/manifest-utils/Cargo.toml +++ b/utils/manifest-utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ya-manifest-utils" -version = "0.1.0" +version = "0.2.0" authors = ["Golem Factory "] edition = "2018" homepage = "https://github.com/golemfactory/yagna" @@ -11,28 +11,33 @@ keywords = ["golem", "yagna"] [dependencies] ya-agreement-utils = "0.4" +ya-utils-path = "0.1" +regex = "1.5" serde = "1.0" serde_json = "1.0" -serde_yaml = "0.8" +serde_yaml = "0.9" thiserror = "1.0" - anyhow = { version = "1.0" } base64 = { version = "0.13" } chrono = { version = "0.4", features = ["serde"] } -ethsign = { version = "0.8.0" } hex = { version = "0.4" } -petname = { version = "1.1.2" } semver = { version = "1.0", features = ["serde"] } serde_with = "1.14" sha2 = { version = "0.9" } snailquote = { version = "0.3" } structopt = { version = "0.3" } -strum = {version = "0.22" } -strum_macros = {version = "0.22" } +strum = { version = "0.24", features = ["derive"] } url = {version = "2.2", features = ["serde"] } +openssl = { version = "0.10", features = ["vendored"] } +md-5 = "0.10" +log = "0.4" [dev-dependencies] -anyhow = "1.0.31" -tempdir = "0.3.7" -shlex = "1.1.0" +tempfile = "3" +anyhow = "1.0" +serial_test = "0.9" +shlex = "1.1" +tar = "0.4" +test-case = "2.2" +ya-manifest-test-utils = "0.1" diff --git a/utils/manifest-utils/src/lib.rs b/utils/manifest-utils/src/lib.rs index a735cf6c15..6c9a31558d 100644 --- a/utils/manifest-utils/src/lib.rs +++ b/utils/manifest-utils/src/lib.rs @@ -1,5 +1,10 @@ pub mod manifest; +pub mod matching; pub mod policy; +pub mod util; pub use manifest::*; -pub use policy::{KeyMeta, Keystore, Policy, PolicyConfig}; +pub use policy::{Keystore, Policy, PolicyConfig}; +pub use util::{ + decode_data, DecodingError, KeystoreLoadResult, KeystoreManager, KeystoreRemoveResult, +}; diff --git a/utils/manifest-utils/src/manifest.rs b/utils/manifest-utils/src/manifest.rs index 4d69d93208..7e9095a43b 100644 --- a/utils/manifest-utils/src/manifest.rs +++ b/utils/manifest-utils/src/manifest.rs @@ -1,33 +1,36 @@ use std::collections::HashSet; use std::ops::Not; +use std::string::ToString; use chrono::{DateTime, Utc}; use semver::Version; use serde::{Deserialize, Deserializer, Serialize}; use serde_json::Value; -use sha2::{Digest, Sha256}; +use strum; +use strum::AsRefStr; +use strum::Display; +use strum::EnumString; use url::Url; use ya_agreement_utils::AgreementView; use ya_agreement_utils::Error as AgreementError; -pub const AGREEMENT_MANIFEST_PROPERTY: &str = - "demand.properties.golem.experimental.srv.comp.payload.@tag"; -pub const AGREEMENT_MANIFEST_SIG_PROPERTY: &str = - "demand.properties.golem.experimental.srv.comp.payload.sig"; +use crate::decode_data; pub const CAPABILITIES_PROPERTY: &str = "golem.runtime.capabilities"; -pub const DEMAND_MANIFEST_PROPERTY: &str = "golem.experimental.srv.comp.payload.@tag"; -pub const DEMAND_MANIFEST_SIG_PROPERTY: &str = "golem.experimental.srv.comp.payload.sig"; +pub const DEMAND_MANIFEST_PROPERTY: &str = "golem.srv.comp.payload"; +pub const DEMAND_MANIFEST_SIG_PROPERTY: &str = "golem.srv.comp.payload.sig"; +pub const DEMAND_MANIFEST_SIG_ALGORITHM_PROPERTY: &str = "golem.srv.comp.payload.sig.algorithm"; +pub const DEMAND_MANIFEST_CERT_PROPERTY: &str = "golem.srv.comp.payload.cert"; + +pub const AGREEMENT_MANIFEST_PROPERTY: &str = "demand.properties.golem.srv.comp.payload"; #[derive(Debug, thiserror::Error)] pub enum Error { #[error("agreement error: {0}")] AgreementError(#[from] AgreementError), - #[error("invalid input base64: {0}")] - BlobBase64(#[from] base64::DecodeError), - #[error("invalid escaped json string: {0}")] - BlobJsonString(#[from] snailquote::UnescapeError), + #[error(transparent)] + DecodingError(#[from] crate::DecodingError), #[error("invalid input json encoding: {0}")] BlobJsonEncoding(#[from] snailquote::ParseUnicodeError), #[error("invalid hash format: '{0}'")] @@ -36,8 +39,6 @@ pub enum Error { HashHexValue(#[from] hex::FromHexError), #[error("invalid manifest format: {0}")] ManifestFormat(#[from] serde_json::Error), - #[error("ECDSA error: {0}")] - SignatureEcdsa(#[from] ethsign::Error), #[error("invalid signature format: {0}")] SignatureFormat(String), #[error("unsupported signature algorithm")] @@ -53,38 +54,21 @@ pub fn read_manifest(view: &AgreementView) -> Result, Error> Ok(Some(decode_manifest(manifest)?)) } -pub fn decode_manifest>(input: S) -> Result { - match decode_base64(&input) { - Ok(manifest) => Ok(manifest), - Err(_) => decode_escaped_json(input), - } -} - -fn decode_base64>(input: S) -> Result { - let decoded = base64::decode(input.as_ref())?; - Ok(serde_json::de::from_slice(&decoded)?) -} - -fn decode_escaped_json>(input: S) -> Result { - let decoded = snailquote::unescape(input.as_ref())?; - Ok(serde_json::de::from_str(&decoded)?) +pub fn decode_manifest>(data: S) -> Result { + let data = decode_data(data)?; + Ok(serde_json::de::from_slice(&data)?) } #[non_exhaustive] -#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize, Display)] +#[serde(rename_all = "kebab-case")] +#[strum(serialize_all = "kebab-case")] pub enum Feature { Inet, Vpn, -} - -impl ToString for Feature { - fn to_string(&self) -> String { - match self { - Self::Inet => "inet", - Self::Vpn => "vpn", - } - .to_string() - } + ManifestSupport, + #[serde(other)] + Other, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -243,10 +227,12 @@ impl<'de> Deserialize<'de> for Command { } } -#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, Serialize, Deserialize, EnumString, AsRefStr)] #[serde(rename_all = "camelCase")] pub enum ArgMatch { + #[strum(ascii_case_insensitive)] Strict, + #[strum(ascii_case_insensitive)] Regex, } @@ -281,52 +267,6 @@ pub struct InetOut { pub urls: Option>, } -#[non_exhaustive] -#[derive(Clone, Eq, PartialEq, Hash)] -pub enum Signature { - Secp256k1(Vec), - Secp256k1Hex(String), -} - -impl Signature { - pub fn verify(&self, input: &[u8]) -> Result, Error> { - match self { - Signature::Secp256k1(vec) => verify_secp256k1(input, vec), - Signature::Secp256k1Hex(string) => { - let sig = hex::decode(normalize_hex_string(string))?; - verify_secp256k1(input, &sig) - } - } - } - - #[inline] - pub fn verify_str>(&self, input: S) -> Result, Error> { - self.verify(input.as_ref().as_bytes()) - } -} - -fn verify_secp256k1(input: &[u8], sig: &[u8]) -> Result, Error> { - if sig.len() < 65 { - return Err(Error::SignatureFormat( - "invalid signature length".to_string(), - )); - } - - let v = sig[0]; - let mut r = [0; 32]; - let mut s = [0; 32]; - - r.copy_from_slice(&sig[1..33]); - s.copy_from_slice(&sig[33..65]); - - let hash = Sha256::digest(input); - let key = ethsign::Signature { v, r, s } - .recover(hash.as_slice()) - .map_err(ethsign::Error::Secp256k1)?; - - Ok(key.bytes().to_vec()) -} - pub fn default_protocols() -> Vec { ["http", "https", "ws", "wss"] .iter() @@ -334,15 +274,6 @@ pub fn default_protocols() -> Vec { .collect() } -fn normalize_hex_string>(input: S) -> String { - let input = input.as_ref(); - if input.starts_with("0x") || input.starts_with("0X") { - input[2..].to_string() - } else { - input.to_string() - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/utils/manifest-utils/src/matching/domain.rs b/utils/manifest-utils/src/matching/domain.rs new file mode 100644 index 0000000000..46ed249a52 --- /dev/null +++ b/utils/manifest-utils/src/matching/domain.rs @@ -0,0 +1,140 @@ +use std::{ + collections::HashSet, + convert::TryFrom, + fs::OpenOptions, + io::BufReader, + path::Path, + sync::{Arc, Mutex, RwLock}, +}; + +use regex::RegexSetBuilder; +use serde::{Deserialize, Serialize}; +use ya_utils_path::SwapSave; + +use super::{CompositeMatcher, Matcher, RegexMatcher, StrictMatcher}; +use crate::{util::str_to_short_hash, ArgMatch}; + +pub type DomainsMatcher = CompositeMatcher; +pub type SharedDomainPatterns = Arc>; +pub type SharedDomainMatchers = Arc>; + +#[derive(Clone, Default, Debug)] +pub struct DomainWhitelistState { + pub patterns: SharedDomainPatterns, + pub matchers: SharedDomainMatchers, +} + +impl DomainWhitelistState { + /// Creates a new `DomainWhitelistState` with patterns matching generated from them matchers + pub fn try_new(patterns: DomainPatterns) -> Result { + let matcher = DomainsMatcher::try_from(&patterns)?; + let matcher = Arc::new(RwLock::new(matcher)); + let patterns = Arc::new(Mutex::new(patterns)); + Ok(Self { + patterns, + matchers: matcher, + }) + } +} + +impl DomainsMatcher { + pub fn load_or_create(path: &Path) -> anyhow::Result { + Ok(DomainsMatcher::try_from(&DomainPatterns::load_or_create( + path, + )?)?) + } +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct DomainPatterns { + pub patterns: Vec, +} + +impl DomainPatterns { + pub fn load(path: &Path) -> anyhow::Result { + if path.exists() { + log::debug!("Loading domain patterns from: {}", path.display()); + let patterns = OpenOptions::new().read(true).open(path)?; + let patterns = BufReader::new(patterns); + Ok(serde_json::from_reader(patterns)?) + } else { + Ok(Self::default()) + } + } + + pub fn load_or_create(path: &Path) -> anyhow::Result { + if path.exists() { + Self::load(path) + } else { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::File::create(&path)?; + let patterns = Self::default(); + patterns.save(path)?; + Ok(patterns) + } + } + + pub fn update_and_save(&mut self, path: &Path, patterns: DomainPatterns) -> anyhow::Result<()> { + self.patterns = patterns.patterns; + self.save(path) + } + + pub fn save(&self, path: &Path) -> anyhow::Result<()> { + Ok(path.swap_save(serde_json::to_string_pretty(self)?)?) + } +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DomainPattern { + pub domain: String, + #[serde(rename = "match", default = "DomainPattern::default_domain_match")] + pub domain_match: ArgMatch, +} + +impl DomainPattern { + fn default_domain_match() -> ArgMatch { + ArgMatch::Regex + } +} + +impl TryFrom<&DomainPatterns> for DomainsMatcher { + type Error = anyhow::Error; + + fn try_from(domain_patterns: &DomainPatterns) -> Result { + let mut strict_patterns = HashSet::new(); + let mut regex_patterns = HashSet::new(); + for domain_pattern in &domain_patterns.patterns { + match domain_pattern.domain_match { + ArgMatch::Strict => strict_patterns.insert(domain_pattern.domain.to_lowercase()), + ArgMatch::Regex => regex_patterns.insert(domain_pattern.domain.to_lowercase()), + }; + } + let mut matchers: Vec> = Vec::new(); + if !strict_patterns.is_empty() { + let matcher = StrictMatcher { + values: strict_patterns, + }; + matchers.push(Box::new(matcher)); + } + if !regex_patterns.is_empty() { + let regex_patterns = regex_patterns.into_iter().collect::>(); + let regex_patterns = RegexSetBuilder::new(®ex_patterns) + .case_insensitive(true) + .ignore_whitespace(true) + .build()?; + let matcher = RegexMatcher { + patterns: regex_patterns, + }; + matchers.push(Box::new(matcher)); + } + Ok(CompositeMatcher { matchers }) + } +} + +pub fn pattern_to_id(pattern: &DomainPattern) -> String { + let pattern = &pattern.domain; + str_to_short_hash(pattern) +} diff --git a/utils/manifest-utils/src/matching/mod.rs b/utils/manifest-utils/src/matching/mod.rs new file mode 100644 index 0000000000..bb161d5269 --- /dev/null +++ b/utils/manifest-utils/src/matching/mod.rs @@ -0,0 +1,42 @@ +pub mod domain; + +use std::{collections::HashSet, fmt::Debug}; + +use regex::RegexSet; + +pub trait Matcher: Debug + Send + Sync { + fn matches(&self, txt: &str) -> bool; +} + +#[derive(Debug)] +struct RegexMatcher { + patterns: RegexSet, +} + +impl Matcher for RegexMatcher { + fn matches(&self, txt: &str) -> bool { + self.patterns.matches(txt).matched_any() + } +} + +#[derive(Debug)] +struct StrictMatcher { + values: HashSet, +} + +impl Matcher for StrictMatcher { + fn matches(&self, txt: &str) -> bool { + self.values.contains(&txt.to_lowercase()) + } +} + +#[derive(Debug, Default)] +pub struct CompositeMatcher { + matchers: Vec>, +} + +impl Matcher for CompositeMatcher { + fn matches(&self, txt: &str) -> bool { + self.matchers.iter().any(|matcher| matcher.matches(txt)) + } +} diff --git a/utils/manifest-utils/src/policy.rs b/utils/manifest-utils/src/policy.rs index b5bdb279e3..9eceb7211a 100644 --- a/utils/manifest-utils/src/policy.rs +++ b/utils/manifest-utils/src/policy.rs @@ -1,18 +1,22 @@ use serde::{Deserialize, Serialize}; -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::fs::OpenOptions; -use std::io::Write; +use std::collections::{HashMap, HashSet}; +use std::fmt::Debug; +use std::hash::Hash; use std::ops::Not; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::str::FromStr; use std::sync::{Arc, RwLock}; -use ethsign::PublicKey; +use openssl::hash::MessageDigest; +use openssl::pkey::{PKey, Public}; +use openssl::sign::Verifier; +use openssl::x509::store::{X509Store, X509StoreBuilder}; +use openssl::x509::{X509ObjectRef, X509StoreContext, X509}; use structopt::StructOpt; -use strum::{IntoEnumIterator, VariantNames}; -use strum_macros::{Display, EnumIter, EnumString, EnumVariantNames}; +use strum::{Display, EnumIter, EnumString, EnumVariantNames, IntoEnumIterator, VariantNames}; -const SCHEME_SECP256K1: &str = "secp256k1"; +use crate::matching::domain::DomainWhitelistState; +use crate::util::{CertBasicDataVisitor, X509Visitor}; /// Policy configuration #[derive(StructOpt, Clone, Debug, Default, Serialize, Deserialize)] @@ -39,6 +43,9 @@ pub struct PolicyConfig { #[structopt(skip)] #[serde(skip_serializing, skip_deserializing)] pub trusted_keys: Option, + #[structopt(skip)] + #[serde(skip_serializing, skip_deserializing)] + pub domain_patterns: DomainWhitelistState, } impl PolicyConfig { @@ -104,121 +111,117 @@ impl FromStr for Match { } } -#[derive(Clone, Default)] +#[derive(Clone)] pub struct Keystore { - inner: Arc, KeyMeta>>>, -} - -#[derive(Clone, Debug)] -pub struct KeyMeta { - pub scheme: String, - pub name: String, + inner: Arc>, } -impl Default for KeyMeta { +impl Default for Keystore { fn default() -> Self { - KeyMeta::new(None, None) - } -} - -impl KeyMeta { - pub fn new(scheme: Option, name: Option) -> Self { - KeyMeta { - scheme: scheme.unwrap_or_else(|| SCHEME_SECP256K1.to_string()), - name: name.unwrap_or_else(|| petname::petname(3, "-")), + let store = X509StoreBuilder::new().expect("SSL works").build(); + Self { + inner: Arc::new(RwLock::new(store)), } } } impl Keystore { - pub fn load(path: impl AsRef) -> anyhow::Result { - let path = path.as_ref(); - let contents = std::fs::read_to_string(path) - .map_err(|e| anyhow::anyhow!("cannot read the keystore file: {}", e))?; - - let map = contents - .lines() - .map(|s| s.trim()) - .filter(|s| !s.is_empty() && !s.starts_with('#')) - .map(parse_key_entry) - .collect::>()?; - - Ok(Keystore { - inner: Arc::new(RwLock::new(map)), - }) + /// Reads DER or PEM certificates (or PEM certificate stacks) from `cert_dir` and creates new `X509Store`. + pub fn load(cert_dir: impl AsRef + Debug) -> anyhow::Result { + std::fs::create_dir_all(&cert_dir)?; + let mut store = X509StoreBuilder::new()?; + let cert_dir = std::fs::read_dir(cert_dir)?; + for dir_entry in cert_dir { + let cert = dir_entry?; + let cert = cert.path(); + if cert.is_file() { + Self::load_file(&mut store, &cert)?; + } else { + log::debug!("Skipping '{:?}' while loading a keystore", cert); + } + } + let store = store.build(); + let inner = Arc::new(RwLock::new(store)); + Ok(Keystore { inner }) } - fn entries(&self) -> Vec { - let inner = self.inner.read().unwrap(); - inner.iter().map(key_entry_to_string).collect() + pub fn replace(&self, other: Keystore) { + let store = { + let mut inner = other.inner.write().unwrap(); + std::mem::replace(&mut (*inner), X509StoreBuilder::new().unwrap().build()) + }; + let mut inner = self.inner.write().unwrap(); + *inner = store; } - pub fn save(&self, path: impl AsRef) -> anyhow::Result<()> { - let lines: Vec = self.entries(); - - let mut file = OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .open(path)?; - lines - .into_iter() - .try_for_each(|line| file.write_all(line.as_bytes()))?; + /// Decodes byte64 `sig`, verifies `cert`and reads its pub key, + /// prepares digest using `sig_alg`, verifies `data` using `sig` and pub key. + pub fn verify_signature( + &self, + cert: impl AsRef, + sig: impl AsRef, + sig_alg: impl AsRef, + data: impl AsRef, + ) -> anyhow::Result<()> { + let sig = crate::decode_data(sig)?; - file.flush()?; - file.sync_all()?; + let pkey = self.verify_cert(cert)?; + let msg_digest = MessageDigest::from_name(sig_alg.as_ref()) + .ok_or_else(|| anyhow::anyhow!("Unknown signature algorithm: {}", sig_alg.as_ref()))?; + let mut verifier = Verifier::new(msg_digest, pkey.as_ref())?; + if !(verifier.verify_oneshot(&sig, data.as_ref().as_bytes())?) { + return Err(anyhow::anyhow!("Invalid signature")); + } Ok(()) } - pub fn replace(&self, other: Keystore) { - let map = { - let mut inner = other.inner.write().unwrap(); - std::mem::take(&mut (*inner)) - }; - let mut inner = self.inner.write().unwrap(); - *inner = map; - } -} - -impl Keystore { - pub fn contains(&self, key: &[u8]) -> bool { + pub(crate) fn certs_ids(&self) -> anyhow::Result> { let inner = self.inner.read().unwrap(); - inner.contains_key(key) + let mut ids = HashSet::new(); + for cert in inner.objects() { + if let Some(cert) = cert.x509() { + let id = crate::util::cert_to_id(cert)?; + ids.insert(id); + } + } + Ok(ids) } - pub fn insert( + pub(crate) fn visit_certs( &self, - key: impl Into>, - scheme: Option, - name: Option, + visitor: &mut X509Visitor, ) -> anyhow::Result<()> { - let mut inner = self.inner.write().unwrap(); - let meta = KeyMeta::new(scheme, name); - let boxed_key = key.into(); - let verification_key = boxed_key.clone(); - // Verify key - PublicKey::from_slice(&*verification_key) - .map_err(|e| anyhow::anyhow!(format!("invalid key provided: {:?}", e)))?; - - inner.insert(boxed_key, meta); + let inner = self.inner.read().unwrap(); + for cert in inner.objects().iter().flat_map(X509ObjectRef::x509) { + visitor.accept(cert)?; + } Ok(()) } - pub fn remove_by_name(&self, name: impl AsRef) -> Option> { - let name = name.as_ref(); - let mut inner = self.inner.write().unwrap(); - let key = inner - .iter() - .find(|(_, meta)| meta.name.as_str() == name) - .map(|(key, _)| key.clone())?; - inner.remove(&key); - Some(key) + fn load_file(store: &mut X509StoreBuilder, cert: &PathBuf) -> anyhow::Result<()> { + for cert in crate::util::parse_cert_file(cert)? { + store.add_cert(cert)? + } + Ok(()) } - pub fn keys(&self) -> BTreeMap, KeyMeta> { - let inner = self.inner.read().unwrap(); - (*inner).clone() + fn verify_cert>(&self, cert: S) -> anyhow::Result> { + let cert = crate::decode_data(cert)?; + let cert = match X509::from_der(&cert) { + Ok(cert) => cert, + Err(_) => X509::from_pem(&cert)?, + }; + let store = self + .inner + .read() + .map_err(|err| anyhow::anyhow!("Err: {}", err.to_string()))?; + let cert_chain = openssl::stack::Stack::new()?; + let mut ctx = X509StoreContext::new()?; + if !(ctx.init(&store, &cert, &cert_chain, |ctx| ctx.verify_cert())?) { + return Err(anyhow::anyhow!("Invalid certificate")); + } + Ok(cert.public_key()?) } } @@ -241,37 +244,6 @@ fn parse_property_match(input: &str) -> anyhow::Result<(String, Match)> { Ok((property, values)) } -fn parse_key(scheme: &str, key: &str) -> anyhow::Result> { - match scheme.to_lowercase().as_str() { - SCHEME_SECP256K1 => { - let key_bytes = hex::decode(key)?; - let key = PublicKey::from_slice(key_bytes.as_slice()) - .map_err(|_| anyhow::anyhow!("invalid key"))?; - Ok(key.bytes().to_vec().into()) - } - _ => anyhow::bail!("invalid scheme: {}", scheme), - } -} - -fn parse_key_entry(line: &str) -> anyhow::Result<(Box<[u8]>, KeyMeta)> { - let mut split = line.trim().split_whitespace(); - let scheme = match split.next() { - Some(scheme) => scheme.to_string(), - None => anyhow::bail!("scheme missing"), - }; - let key = match split.next() { - Some(key_hex) => parse_key(scheme.as_str(), key_hex)?, - None => anyhow::bail!("key missing"), - }; - let name = split.next().map(|s| s.to_string()); - - Ok((key, KeyMeta::new(Some(scheme), name))) -} - -fn key_entry_to_string((key, meta): (&Box<[u8]>, &KeyMeta)) -> String { - format!("{}\t{}\t{}\n", meta.scheme, hex::encode(key), meta.name) -} - #[cfg(test)] mod tests { use super::*; diff --git a/utils/manifest-utils/src/util.rs b/utils/manifest-utils/src/util.rs new file mode 100644 index 0000000000..280619322b --- /dev/null +++ b/utils/manifest-utils/src/util.rs @@ -0,0 +1,350 @@ +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::convert::TryFrom; +use std::ffi::OsStr; +use std::fs::{self, DirEntry}; +use std::path::Path; +use std::{fs::File, path::PathBuf}; + +use md5::{Digest, Md5}; +use openssl::nid::Nid; +use openssl::x509::{X509Ref, X509}; +use std::io::prelude::*; + +use crate::Keystore; + +/// Tries do decode base64. On failure tries to unescape snailquotes. +pub fn decode_data>(input: S) -> Result, DecodingError> { + let no_whitespace: String = input.as_ref().split_whitespace().collect(); + match base64::decode(no_whitespace) { + Ok(data) => Ok(data), + Err(_) => Ok(snailquote::unescape(input.as_ref()).map(String::into_bytes)?), + } +} + +#[derive(Debug, thiserror::Error)] +pub enum DecodingError { + #[error("invalid input base64: {0}")] + BlobBase64(#[from] base64::DecodeError), + #[error("invalid escaped json string: {0}")] + BlobJsonString(#[from] snailquote::UnescapeError), +} + +pub fn parse_cert_file(cert: &PathBuf) -> anyhow::Result> { + let extension = get_file_extension(cert); + let mut cert = File::open(cert)?; + let mut cert_buffer = Vec::new(); + cert.read_to_end(&mut cert_buffer)?; + match extension { + Some(ref der) if der.to_lowercase() == "der" => Ok(vec![X509::from_der(&cert_buffer)?]), + Some(ref pem) if pem.to_lowercase() == "pem" => Ok(X509::stack_from_pem(&cert_buffer)?), + _ => { + // Certificates can have various other extensions like .cer .crt .key (and .key can be both DER and PEM) + // Initial parsing dictated by `extension` is done because it would improper to parse .pem as a DER + Ok(X509::stack_from_pem(&cert_buffer) + .or_else(|_| X509::from_der(&cert_buffer).map(|cert| vec![cert]))?) + } + } +} + +fn get_file_extension(path: &Path) -> Option { + path.extension().map(os_str_to_string) +} + +fn get_file_name(path: &Path) -> Option { + path.file_name().map(os_str_to_string) +} + +fn get_file_stem(path: &Path) -> Option { + path.file_stem().map(os_str_to_string) +} + +fn os_str_to_string(os_str: &OsStr) -> String { + os_str.to_string_lossy().to_string() +} + +pub fn to_cert_data(certs: &Vec) -> anyhow::Result> { + let mut certs_data = Vec::new(); + for cert in certs { + let data = CertBasicData::try_from(cert.as_ref())?; + certs_data.push(data); + } + Ok(certs_data) +} + +/// Adds entries with given `nid` to given `subject` String. +fn add_cert_subject_entries( + subject: &mut BTreeMap, + cert: &X509Ref, + nid: Nid, + entry_short_name: &str, +) { + if let Some(entries) = cert_subject_entries(cert, nid) { + subject.insert(entry_short_name.to_string(), entries); + } +} + +/// Reads subject entries and returns them as comma separated `String`. +fn cert_subject_entries(cert: &X509Ref, nid: Nid) -> Option { + let entries = + cert.subject_name() + .entries_by_nid(nid) + .fold(String::from(""), |mut names, name| { + if !names.is_empty() { + names.push_str(", "); + } + let name = String::from_utf8_lossy(name.data().as_slice()); + names.push_str(&name); + names + }); + if entries.is_empty() { + return None; + } + Some(entries) +} + +pub fn cert_to_id(cert: &X509Ref) -> anyhow::Result { + let txt = cert.to_text()?; + Ok(str_to_short_hash(&txt)) +} + +pub fn visit_certificates( + cert_dir: &PathBuf, + visitor: T, +) -> anyhow::Result { + let keystore = Keystore::load(cert_dir)?; + let mut visitor = X509Visitor { visitor }; + keystore.visit_certs(&mut visitor)?; + Ok(visitor.visitor) +} + +pub struct CertBasicData { + pub id: String, + pub not_after: String, + pub subject: BTreeMap, +} + +impl TryFrom<&X509Ref> for CertBasicData { + type Error = anyhow::Error; + + fn try_from(cert: &X509Ref) -> Result { + let id = cert_to_id(cert)?; + let not_after = cert.not_after().to_string(); + let mut subject = BTreeMap::new(); + add_cert_subject_entries(&mut subject, cert, Nid::COMMONNAME, "CN"); + add_cert_subject_entries(&mut subject, cert, Nid::PKCS9_EMAILADDRESS, "E"); + add_cert_subject_entries(&mut subject, cert, Nid::ORGANIZATIONNAME, "O"); + add_cert_subject_entries(&mut subject, cert, Nid::ORGANIZATIONALUNITNAME, "OU"); + add_cert_subject_entries(&mut subject, cert, Nid::COUNTRYNAME, "C"); + add_cert_subject_entries(&mut subject, cert, Nid::STATEORPROVINCENAME, "ST"); + + Ok(CertBasicData { + id, + not_after, + subject, + }) + } +} + +pub trait CertBasicDataVisitor { + fn accept(&mut self, cert_data: CertBasicData); +} + +pub(crate) struct X509Visitor { + visitor: T, +} + +impl X509Visitor { + pub(crate) fn accept(&mut self, cert: &X509Ref) -> anyhow::Result<()> { + let cert_data = CertBasicData::try_from(cert)?; + self.visitor.accept(cert_data); + Ok(()) + } +} + +pub struct KeystoreManager { + ids: HashSet, + cert_dir: PathBuf, +} + +impl KeystoreManager { + pub fn try_new(cert_dir: &PathBuf) -> anyhow::Result { + let keystore = Keystore::load(cert_dir)?; + let ids = keystore.certs_ids()?; + let cert_dir = cert_dir.clone(); + Ok(Self { ids, cert_dir }) + } + + /// Copies certificates from given file to `cert_dir` and returns newly added certificates. + /// Certificates already existing in `cert_dir` are skipped. + pub fn load_certs(self, cert_paths: &Vec) -> anyhow::Result { + let mut loaded = HashMap::new(); + let mut skipped = HashMap::new(); + + for cert_path in cert_paths { + let mut new_certs = Vec::new(); + let file_certs = parse_cert_file(cert_path)?; + if file_certs.is_empty() { + continue; + } + let file_certs_len = file_certs.len(); + for file_cert in file_certs { + let id = cert_to_id(&file_cert)?; + if !self.ids.contains(&id) && !loaded.contains_key(&id) { + new_certs.push(file_cert.clone()); + loaded.insert(id, file_cert); + } else { + skipped.insert(id, file_cert); + } + } + if file_certs_len == new_certs.len() { + self.load_as_keychain_file(cert_path)?; + } else { + self.load_as_certificate_files(cert_path, new_certs)?; + } + } + + let loaded: Vec = loaded.into_values().collect(); + let skipped: Vec = skipped.into_values().collect(); + if loaded.is_empty() { + return Ok(KeystoreLoadResult::NothingNewToLoad { skipped }); + } + Ok(KeystoreLoadResult::Loaded { loaded, skipped }) + } + + pub fn remove_certs(self, ids: &HashSet) -> anyhow::Result { + if ids.difference(&self.ids).eq(ids) { + return Ok(KeystoreRemoveResult::NothingToRemove); + } + let mut removed = HashMap::new(); + + let cert_dir_entries: Vec> = + std::fs::read_dir(self.cert_dir.clone())?.collect(); + for dir_entry in cert_dir_entries { + let cert_file = dir_entry?; + let cert_file = cert_file.path(); + let certs = parse_cert_file(&cert_file)?; + if certs.is_empty() { + // No certificates in parsed file. + continue; + } + let mut ids_cert = certs.into_iter().fold(HashMap::new(), |mut certs, cert| { + if let Ok(id) = cert_to_id(&cert) { + certs.insert(id, cert); + } + certs + }); + let mut split_and_skip = false; + for id in ids.iter() { + if let Some(cert) = ids_cert.remove(id) { + removed.insert(id, cert); + split_and_skip = true; + } + } + if split_and_skip { + let file_stem = get_file_stem(&cert_file).expect("Cannot get file name stem"); + let dot_extension = get_file_extension(&cert_file) + .map_or_else(|| String::from(""), |ex| format!(".{ex}")); + for (id, cert) in ids_cert { + let cert = cert.to_pem()?; + let mut file_path = self.cert_dir.clone(); + let filename = format!("{file_stem}.{id}{dot_extension}"); + file_path.push(filename); + fs::write(file_path, cert)?; + } + fs::remove_file(cert_file)?; + } + } + + let removed: Vec = removed.into_values().collect(); + Ok(KeystoreRemoveResult::Removed { removed }) + } + + /// Loads keychain file to `cert_dir` + fn load_as_keychain_file(&self, cert_path: &PathBuf) -> anyhow::Result<()> { + let file_name = get_file_name(cert_path) + .ok_or_else(|| anyhow::anyhow!(format!("Cannot get filename of {cert_path:?}")))?; + let mut new_cert_path = self.cert_dir.clone(); + new_cert_path.push(file_name); + if new_cert_path.exists() { + let file_stem = get_file_stem(&new_cert_path).expect("Has to have stem"); + let dot_extension = get_file_extension(&new_cert_path) + .map(|ex| format!(".{ex}")) + .unwrap_or_else(|| String::from("")); + for i in 0..u32::MAX { + let numbered_filename = format!("{file_stem}.{i}{dot_extension}"); + new_cert_path = self.cert_dir.clone(); + new_cert_path.push(numbered_filename); + if !new_cert_path.exists() { + break; + } + } + if new_cert_path.exists() { + anyhow::bail!("Unable to load certificate"); + } + } + fs::copy(cert_path, new_cert_path)?; + Ok(()) + } + + /// Loads certificates as individual files to `cert_dir` + fn load_as_certificate_files(&self, cert_path: &Path, certs: Vec) -> anyhow::Result<()> { + let file_stem = get_file_stem(cert_path) + .ok_or_else(|| anyhow::anyhow!("Cannot get file name stem."))?; + let dot_extension = get_file_extension(cert_path) + .map(|ex| format!(".{ex}")) + .unwrap_or_else(|| String::from("")); + for cert in certs.into_iter() { + let id = cert_to_id(&cert)?; + let mut new_cert_path = self.cert_dir.clone(); + new_cert_path.push(format!("{file_stem}.{id}{dot_extension}")); + let cert = cert.to_pem()?; + fs::write(new_cert_path, cert)?; + } + Ok(()) + } +} + +pub enum KeystoreLoadResult { + NothingNewToLoad { + skipped: Vec, + }, + Loaded { + loaded: Vec, + skipped: Vec, + }, +} + +pub enum KeystoreRemoveResult { + NothingToRemove, + Removed { removed: Vec }, +} + +/// Calculates Md5 of `txt` and returns first 8 characters. +pub fn str_to_short_hash(txt: impl AsRef<[u8]>) -> String { + let digest = Md5::digest(txt); + let digest = format!("{digest:x}"); + let short_hash = &digest[..8]; // Md5 is 32 characters + short_hash.to_string() +} + +#[cfg(test)] +pub mod tests { + use super::*; + + #[test] + pub fn base64_wrapped_lines_test() { + let wrapped_base64 = " + VGhlIHF1aWNrIGJyb3du + IGZveCBqdW1wcyBvdmVy + IHRoZSBsYXp5IGRvZw=="; + let phrase = decode_data(wrapped_base64).expect("failed to decode base64 wrapped content"); + let phrase = String::from_utf8_lossy(&phrase).to_string(); + let expected = "The quick brown fox jumps over the lazy dog"; + assert_eq!( + &phrase, expected, + "Manifest related base64 payload may be encoded by the user, + and many tools wrap base64 output by default, + so we should try to filter out whitespace" + ) + } +} diff --git a/utils/manifest-utils/test-utils/Cargo.toml b/utils/manifest-utils/test-utils/Cargo.toml new file mode 100644 index 0000000000..901fe0c2f7 --- /dev/null +++ b/utils/manifest-utils/test-utils/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "ya-manifest-test-utils" +version = "0.1.0" +authors = ["Golem Factory "] +edition = "2018" +homepage = "https://github.com/golemfactory/yagna" +repository = "https://github.com/golemfactory/yagna" +license = "LGPL-3.0" +description = "Yagna manifest INTERNAL TEST utils" + +[dependencies] +base64 = "0.13" +tar = "0.4" +openssl = "0.10" + +ya-manifest-utils = "0.2" +ya-agreement-utils = "0.4" \ No newline at end of file diff --git a/utils/manifest-utils/test-utils/resources/test/certificates.tar b/utils/manifest-utils/test-utils/resources/test/certificates.tar new file mode 100644 index 0000000000..3e74f34421 Binary files /dev/null and b/utils/manifest-utils/test-utils/resources/test/certificates.tar differ diff --git a/utils/manifest-utils/test-utils/resources/test/data.json.base64 b/utils/manifest-utils/test-utils/resources/test/data.json.base64 new file mode 100644 index 0000000000..3285956213 --- /dev/null +++ b/utils/manifest-utils/test-utils/resources/test/data.json.base64 @@ -0,0 +1 @@ +ewogICAgImZvbyI6ICJiYXIiCn0K \ No newline at end of file diff --git a/utils/manifest-utils/test-utils/resources/test/data.json.base64.foo_req_sign.sha256.base64 b/utils/manifest-utils/test-utils/resources/test/data.json.base64.foo_req_sign.sha256.base64 new file mode 100644 index 0000000000..9da1198bdd --- /dev/null +++ b/utils/manifest-utils/test-utils/resources/test/data.json.base64.foo_req_sign.sha256.base64 @@ -0,0 +1 @@ +bcvrLgBybQr8g9SvEzvXnNNo9mnTgOc1VwTffEfeMVPDjeMx8L4WkoCf7QIP1n8E7YwcMGCGYHgCMPl4br3/PyN8H12FFEgf5fkC8+GyrY1dzTFVuxFb+ONshnv/k9zAeLWuYh6+jQnQjdvhVszTZQEF7WFaa/A1RHWDeOp/CiSptTkqFXe0TDnluDTatdMwap8XAWEbU0ejk6nSJfQWnWUetlGlTBwGZEk8TVdDkgHr0g4MC4vw6sIUaLBHOK5kJSCbIP55bYWDz8+8h3NONqFNDAprk6gJu6/XG33JHePnNsbn5jzn49LbObk3h3lX5wv4orGFDiDzK0SPvuDLBw== \ No newline at end of file diff --git a/utils/manifest-utils/test-utils/resources/test/data.json.base64.unknown_req_sign.sha256.base64 b/utils/manifest-utils/test-utils/resources/test/data.json.base64.unknown_req_sign.sha256.base64 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/utils/manifest-utils/test-utils/src/lib.rs b/utils/manifest-utils/test-utils/src/lib.rs new file mode 100644 index 0000000000..e549a39646 --- /dev/null +++ b/utils/manifest-utils/test-utils/src/lib.rs @@ -0,0 +1,164 @@ +use std::fs::{self, File}; +use std::str; +use std::sync::Once; +use std::{collections::HashSet, path::PathBuf}; + +use openssl::hash::MessageDigest; +use openssl::pkey::PKey; +use openssl::rsa::Rsa; +use openssl::sign::Signer; +use tar::Archive; + +use ya_manifest_utils::{ + util::{self, CertBasicData, CertBasicDataVisitor}, + KeystoreLoadResult, KeystoreRemoveResult, +}; + +static INIT: Once = Once::new(); + +#[allow(clippy::ptr_arg)] +pub fn load_certificates_from_dir( + resource_cert_dir: &PathBuf, + test_cert_dir: &PathBuf, + certfile_names: &[&str], +) -> KeystoreLoadResult { + let cert_paths: Vec = certfile_names + .iter() + .map(|certfile_name| { + let mut cert_path = resource_cert_dir.clone(); + cert_path.push(certfile_name); + cert_path + }) + .collect(); + let keystore_manager = + util::KeystoreManager::try_new(test_cert_dir).expect("Can create keystore manager"); + keystore_manager + .load_certs(&cert_paths) + .expect("Can load certificates") +} + +pub fn remove_certificates(test_cert_dir: &PathBuf, cert_ids: &[&str]) -> KeystoreRemoveResult { + let keystore_manager = + util::KeystoreManager::try_new(test_cert_dir).expect("Can create keystore manager"); + keystore_manager + .remove_certs(&slice_to_set(cert_ids)) + .expect("Can load certificates") +} + +#[derive(Default)] +pub struct TestCertDataVisitor { + expected: HashSet, + actual: HashSet, +} + +impl TestCertDataVisitor { + #[allow(dead_code)] + pub fn new(expected: &[&str]) -> Self { + Self { + expected: expected.iter().map(|s| s.to_string()).collect(), + ..Default::default() + } + } + + #[allow(dead_code)] + pub fn test(&self) { + assert_eq!(self.expected, self.actual) + } +} + +impl CertBasicDataVisitor for TestCertDataVisitor { + fn accept(&mut self, cert_data: CertBasicData) { + self.actual.insert(cert_data.id); + } +} + +pub struct TestResources { + pub temp_dir: &'static str, +} + +impl TestResources { + /// Creates new empty cert directory and resources directory with unpacked certificates. + pub fn init_cert_dirs(&self) -> (PathBuf, PathBuf) { + let resource_cert_dir = self.resource_cert_dir_path(); + INIT.call_once(|| { + if resource_cert_dir.exists() { + fs::remove_dir_all(&resource_cert_dir).expect("Can delete test cert resources dir"); + } + fs::create_dir_all(&resource_cert_dir).expect("Can create temp dir"); + self.unpack_cert_resources(&resource_cert_dir); + }); + let store_cert_dir = self.store_cert_dir_path(); + if store_cert_dir.exists() { + // we want to clean store cert dir before every test + fs::remove_dir_all(&store_cert_dir).expect("Can delete test temp dir"); + } + fs::create_dir_all(&store_cert_dir).expect("Can create temp dir"); + (resource_cert_dir, store_cert_dir) + } + + pub fn loaded_cert_files(&self) -> HashSet { + let store_cert_dir = self.store_cert_dir_path(); + let cert_dir = fs::read_dir(store_cert_dir).expect("Can read cert dir"); + cert_dir + .into_iter() + .map(|file| file.expect("Can list cert files")) + .map(|x| x.file_name().to_string_lossy().to_string()) + .collect() + } + + // Signs given `data_b64` using `signing_key` (filename) and returns base64 encoded signature. + pub fn sign_data(&self, data_b64: &[u8], signing_key: &str) -> String { + let mut signing_key_path = self.resource_cert_dir_path(); + signing_key_path.push(signing_key); + let signing_key = fs::read(signing_key_path).expect("Can read signing key"); + let mut password = self.resource_cert_dir_path(); + password.push("pass.txt"); + let password = fs::read(password).expect("Can read password file"); + let password = str::from_utf8(&password).unwrap().trim(); // just in case it got newline at the end + let keypair = Rsa::private_key_from_pem_passphrase(&signing_key, password.as_bytes()) + .expect("Can parse signing key"); + let keypair = PKey::from_rsa(keypair).unwrap(); + let mut signer = Signer::new(MessageDigest::sha256(), &keypair).unwrap(); + signer.update(data_b64).unwrap(); + let signature = signer.sign_to_vec().expect("Can sign manifest"); + base64::encode(signature) + } + + fn unpack_cert_resources(&self, cert_resources_dir: &PathBuf) { + let mut cert_archive = self.test_resources_dir_path(); + cert_archive.push("certificates.tar"); + let cert_archive = File::open(cert_archive).expect("Can open cert archive file"); + let mut cert_archive = Archive::new(cert_archive); + cert_archive + .unpack(cert_resources_dir) + .expect("Can unack cert archive"); + } + + pub fn test_resources_dir_path(&self) -> PathBuf { + let mut test_resources = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + test_resources.push("resources/test"); + test_resources + } + + fn resource_cert_dir_path(&self) -> PathBuf { + let mut cert_resources = self.temp_dir_path(); + cert_resources.push("cert_resources"); + cert_resources + } + + fn store_cert_dir_path(&self) -> PathBuf { + let mut cert_dir = self.temp_dir_path(); + cert_dir.push("cert_dir"); + cert_dir + } + + fn temp_dir_path(&self) -> PathBuf { + PathBuf::from(self.temp_dir) + } +} + +pub fn slice_to_set>(v: &[T]) -> HashSet { + v.iter() + .map(|s| s.as_ref().to_string()) + .collect::>() +} diff --git a/utils/manifest-utils/tests/certificate_expiration.rs b/utils/manifest-utils/tests/certificate_expiration.rs new file mode 100644 index 0000000000..85212edb9a --- /dev/null +++ b/utils/manifest-utils/tests/certificate_expiration.rs @@ -0,0 +1,281 @@ +use utils::*; + +use ya_manifest_utils::Keystore; + +#[test] +fn accept_not_expired_certificate() { + let test_cert_dir = tempfile::tempdir().unwrap(); + + let valid_from = chrono::Utc::now() + .checked_sub_signed(chrono::Duration::days(2)) + .unwrap(); + + let valid_to = chrono::Utc::now() + .checked_add_signed(chrono::Duration::days(2)) + .unwrap(); + + let (self_signed_cert, ca_key_pair) = + create_self_signed_certificate(valid_from, valid_to).unwrap(); + + write_cert_to_file( + &self_signed_cert, + &test_cert_dir.path().join("self_signed.pem"), + ); + + let sut = Keystore::load(&test_cert_dir).unwrap(); + + let (req, csr_key_pair) = create_csr().unwrap(); + let signed_cert = sign_csr(req, csr_key_pair.clone(), &self_signed_cert, ca_key_pair).unwrap(); + + let data = "DEADFACE"; + let sig_alg = "sha256".to_string(); + let sig = sign_data(data.as_bytes(), &csr_key_pair, &sig_alg); + + assert!(sut + .verify_signature( + base64::encode(signed_cert.to_pem().unwrap()), + base64::encode(sig), + sig_alg, + data + ) + .is_ok()); +} + +#[test] +fn not_accept_expired_certificate() { + let test_cert_dir = tempfile::tempdir().unwrap(); + + let valid_from = chrono::Utc::now() + .checked_sub_signed(chrono::Duration::days(2)) + .unwrap(); + + let valid_to = chrono::Utc::now() + .checked_sub_signed(chrono::Duration::days(1)) + .unwrap(); + + let (self_signed_cert, ca_key_pair) = + create_self_signed_certificate(valid_from, valid_to).unwrap(); + + write_cert_to_file( + &self_signed_cert, + &test_cert_dir.path().join("self_signed.pem"), + ); + + let sut = Keystore::load(&test_cert_dir).unwrap(); + + let (req, csr_key_pair) = create_csr().unwrap(); + let signed_cert = sign_csr(req, csr_key_pair.clone(), &self_signed_cert, ca_key_pair).unwrap(); + + let data = "DEADFACE"; + let sig_alg = "sha256".to_string(); + let sig = sign_data(data.as_bytes(), &csr_key_pair, &sig_alg); + + assert!(sut + .verify_signature( + base64::encode(signed_cert.to_pem().unwrap()), + base64::encode(sig), + sig_alg, + data + ) + .is_err()); +} + +#[test] +fn not_accept_not_ready_certificate() { + let test_cert_dir = tempfile::tempdir().unwrap(); + + let valid_from = chrono::Utc::now() + .checked_add_signed(chrono::Duration::days(1)) + .unwrap(); + + let valid_to = chrono::Utc::now() + .checked_add_signed(chrono::Duration::days(2)) + .unwrap(); + + let (self_signed_cert, ca_key_pair) = + create_self_signed_certificate(valid_from, valid_to).unwrap(); + + write_cert_to_file( + &self_signed_cert, + &test_cert_dir.path().join("self_signed.pem"), + ); + + let sut = Keystore::load(&test_cert_dir).unwrap(); + + let (req, csr_key_pair) = create_csr().unwrap(); + let signed_cert = sign_csr(req, csr_key_pair.clone(), &self_signed_cert, ca_key_pair).unwrap(); + + let data = "DEADFACE"; + let sig_alg = "sha256".to_string(); + let sig = sign_data(data.as_bytes(), &csr_key_pair, &sig_alg); + + assert!(sut + .verify_signature( + base64::encode(signed_cert.to_pem().unwrap()), + base64::encode(sig), + sig_alg, + data + ) + .is_err()); +} + +mod utils { + use std::fs::File; + use std::io::Write; + use std::path::Path; + + use chrono::{DateTime, Utc}; + use openssl::asn1::Asn1Time; + use openssl::bn::{BigNum, MsbOption}; + use openssl::error::ErrorStack; + use openssl::hash::MessageDigest; + use openssl::pkey::{PKey, Private}; + use openssl::rsa::Rsa; + use openssl::sign::Signer; + use openssl::x509::extension::{ + AuthorityKeyIdentifier, BasicConstraints, KeyUsage, SubjectAlternativeName, + SubjectKeyIdentifier, + }; + use openssl::x509::{X509NameBuilder, X509Ref, X509Req, X509ReqBuilder, X509}; + + pub fn sign_data( + data: &[u8], + csr_key_pair: &openssl::pkey::PKey, + sig_alg: &str, + ) -> Vec { + let mut signer = + Signer::new(MessageDigest::from_name(sig_alg).unwrap(), csr_key_pair).unwrap(); + signer.sign_oneshot_to_vec(data).unwrap() + } + + pub fn write_cert_to_file(cert: &X509Ref, file_path: &Path) { + let mut file = File::create(&file_path).unwrap(); + file.write_all(&cert.to_pem().unwrap()).unwrap(); + } + + pub fn create_self_signed_certificate( + valid_from: DateTime, + valid_to: DateTime, + ) -> Result<(X509, PKey), ErrorStack> { + let rsa = Rsa::generate(2048)?; + let key_pair = PKey::from_rsa(rsa)?; + + let mut x509_name = X509NameBuilder::new()?; + x509_name.append_entry_by_text("C", "US")?; + x509_name.append_entry_by_text("ST", "TX")?; + x509_name.append_entry_by_text("O", "Some CA organization")?; + x509_name.append_entry_by_text("CN", "ca test")?; + let x509_name = x509_name.build(); + + let mut cert_builder = X509::builder()?; + cert_builder.set_version(2)?; + let serial_number = { + let mut serial = BigNum::new()?; + serial.rand(159, MsbOption::MAYBE_ZERO, false)?; + serial.to_asn1_integer()? + }; + cert_builder.set_serial_number(&serial_number)?; + cert_builder.set_subject_name(&x509_name)?; + cert_builder.set_issuer_name(&x509_name)?; + cert_builder.set_pubkey(&key_pair)?; + + let not_before = Asn1Time::from_unix(valid_from.timestamp())?; + cert_builder.set_not_before(¬_before)?; + + let not_after = Asn1Time::from_unix(valid_to.timestamp())?; + cert_builder.set_not_after(¬_after)?; + + cert_builder.append_extension(BasicConstraints::new().critical().ca().build()?)?; + cert_builder.append_extension( + KeyUsage::new() + .critical() + .key_cert_sign() + .crl_sign() + .build()?, + )?; + + let subject_key_identifier = + SubjectKeyIdentifier::new().build(&cert_builder.x509v3_context(None, None))?; + cert_builder.append_extension(subject_key_identifier)?; + + cert_builder.sign(&key_pair, MessageDigest::sha256())?; + let cert = cert_builder.build(); + + Ok((cert, key_pair)) + } + + pub fn create_csr() -> Result<(X509Req, PKey), ErrorStack> { + let rsa = Rsa::generate(2048)?; + let key_pair = PKey::from_rsa(rsa)?; + + let mut req_builder = X509ReqBuilder::new()?; + req_builder.set_pubkey(&key_pair)?; + + let mut x509_name = X509NameBuilder::new()?; + x509_name.append_entry_by_text("C", "US")?; + x509_name.append_entry_by_text("ST", "TX")?; + x509_name.append_entry_by_text("O", "Some organization")?; + x509_name.append_entry_by_text("CN", "www.example.com")?; + let x509_name = x509_name.build(); + req_builder.set_subject_name(&x509_name)?; + + req_builder.sign(&key_pair, MessageDigest::sha256())?; + let req = req_builder.build(); + Ok((req, key_pair)) + } + + pub fn sign_csr( + csr: X509Req, + csr_keys: PKey, + signing_cert: &X509Ref, + signing_keys: PKey, + ) -> Result { + let mut cert_builder = X509::builder()?; + cert_builder.set_version(2)?; + let serial_number = { + let mut serial = BigNum::new()?; + serial.rand(159, MsbOption::MAYBE_ZERO, false)?; + serial.to_asn1_integer()? + }; + cert_builder.set_serial_number(&serial_number)?; + cert_builder.set_subject_name(csr.subject_name())?; + cert_builder.set_issuer_name(signing_cert.subject_name())?; + cert_builder.set_pubkey(&csr_keys)?; + let not_before = Asn1Time::days_from_now(0)?; + cert_builder.set_not_before(¬_before)?; + let not_after = Asn1Time::days_from_now(365)?; + cert_builder.set_not_after(¬_after)?; + + cert_builder.append_extension(BasicConstraints::new().build()?)?; + + cert_builder.append_extension( + KeyUsage::new() + .critical() + .non_repudiation() + .digital_signature() + .key_encipherment() + .build()?, + )?; + + let subject_key_identifier = SubjectKeyIdentifier::new() + .build(&cert_builder.x509v3_context(Some(signing_cert), None))?; + cert_builder.append_extension(subject_key_identifier)?; + + let auth_key_identifier = AuthorityKeyIdentifier::new() + .keyid(false) + .issuer(false) + .build(&cert_builder.x509v3_context(Some(signing_cert), None))?; + cert_builder.append_extension(auth_key_identifier)?; + + let subject_alt_name = SubjectAlternativeName::new() + .dns("*.example.com") + .dns("hello.com") + .build(&cert_builder.x509v3_context(Some(signing_cert), None))?; + cert_builder.append_extension(subject_alt_name)?; + + cert_builder.sign(&signing_keys, MessageDigest::sha256())?; + let cert = cert_builder.build(); + + Ok(cert) + } +} diff --git a/utils/manifest-utils/tests/certificate_store.rs b/utils/manifest-utils/tests/certificate_store.rs new file mode 100644 index 0000000000..58e716036a --- /dev/null +++ b/utils/manifest-utils/tests/certificate_store.rs @@ -0,0 +1,119 @@ +#[macro_use] +extern crate serial_test; + +use std::fs; + +use test_case::test_case; + +use ya_manifest_test_utils::*; +use ya_manifest_utils::util::visit_certificates; + +static TEST_RESOURCES: TestResources = TestResources { + temp_dir: env!("CARGO_TARGET_TMPDIR"), +}; + +/// Test utilities +#[test_case( + &[], + &[], + &[], + &[]; + "Does not fail when listing empty store" +)] +#[test_case( + &["foo_ca.cert.pem"], + &[], + &["c128af8c"], + &["foo_ca.cert.pem"]; + "Can load one certificate" +)] +#[test_case( + &["foo_ca.cert.pem", "foo_inter.cert.pem"], + &["c128af8c", "4e0df976"], + &[], + &[]; + "Can remove all certificates" +)] +#[test_case( + &["foo_ca-chain.cert.pem"], + &[], + &["4e0df976", "c128af8c"], + &["foo_ca-chain.cert.pem"]; + "Load keychain loads two certificates and stores them in received form (single keychain file)" +)] +#[test_case( + &["foo_ca-chain.cert.pem"], + &["c128af8c"], + &["4e0df976"], + &["foo_ca-chain.cert.4e0df976.pem"]; + "Load keychain and remove root CA results with intermediate cert and cert file with id in the name" +)] +#[test_case( + &["foo_ca-chain.cert.pem"], + &["4e0df976"], + &["c128af8c"], + &["foo_ca-chain.cert.c128af8c.pem"]; + "Load keychain and remove intermediate cert results with root CA and cert file with id in the name" +)] +#[test_case( + &["foo_ca.cert.pem", "foo_ca.cert.pem", "foo_ca.cert.pem"], + &[], + &["c128af8c"], + &["foo_ca.cert.pem"]; + "Adding duplicates results in a single certificate in the store" +)] +#[serial] +fn certificate_store_test( + certs_to_add: &[&str], + ids_to_remove: &[&str], + expected_ids: &[&str], + expected_files: &[&str], +) { + // Having + let (resource_cert_dir, test_cert_dir) = TEST_RESOURCES.init_cert_dirs(); + load_certificates_from_dir(&resource_cert_dir, &test_cert_dir, certs_to_add); + remove_certificates(&test_cert_dir, ids_to_remove); + let mut visitor = TestCertDataVisitor::new(expected_ids); + // When + visitor = visit_certificates(&test_cert_dir, visitor).expect("Can visit loaded certificates"); + // Then + visitor.test(); + let certs = TEST_RESOURCES.loaded_cert_files(); + assert_eq!(certs, slice_to_set(expected_files)); +} + +/// Name collision should be resolved +#[test] +#[serial] +fn certificate_name_collision_test() { + // Having + let (resource_cert_dir, test_cert_dir) = TEST_RESOURCES.init_cert_dirs(); + let colliding_name = "foo_inter.cert.pem"; + + let mut colliding_file_1 = resource_cert_dir.clone(); + colliding_file_1.push(colliding_name); + + let mut colliding_file_2 = resource_cert_dir.clone(); + colliding_file_2.push("copy"); + fs::create_dir_all(&colliding_file_2).expect("Can create dir"); + colliding_file_2.push(colliding_name); + let mut other = resource_cert_dir.clone(); + other.push("foo_req.cert.pem"); + fs::copy(other, colliding_file_2).expect("Can copy file"); + + load_certificates_from_dir( + &resource_cert_dir, + &test_cert_dir, + &[colliding_name, &format!("copy/{colliding_name}")], + ); + let mut visitor = TestCertDataVisitor::new(&["4e0df976", "0e136cb3"]); + // When + visitor = visit_certificates(&test_cert_dir, visitor).expect("Can visit loaded certificates"); + // Then + visitor.test(); + let certs = TEST_RESOURCES.loaded_cert_files(); + assert_eq!( + certs, + slice_to_set(&["foo_inter.cert.pem", "foo_inter.cert.0.pem"]) + ); +} diff --git a/utils/manifest-utils/tests/certificate_validation.rs b/utils/manifest-utils/tests/certificate_validation.rs new file mode 100644 index 0000000000..62dde40c04 --- /dev/null +++ b/utils/manifest-utils/tests/certificate_validation.rs @@ -0,0 +1,86 @@ +#[macro_use] +extern crate serial_test; + +use std::{fs, path::PathBuf}; + +use ya_manifest_test_utils::*; +use ya_manifest_utils::Keystore; + +static TEST_RESOURCES: TestResources = TestResources { + temp_dir: env!("CARGO_TARGET_TMPDIR"), +}; + +#[test] +#[serial] +fn valid_certificate_test() { + // Having + let (resource_cert_dir, test_cert_dir) = TEST_RESOURCES.init_cert_dirs(); + load_certificates_from_dir( + &resource_cert_dir, + &test_cert_dir, + &["foo_ca-chain.cert.pem"], + ); + + let request = prepare_request(resource_cert_dir); + + // Then + let keystore = Keystore::load(&test_cert_dir).expect("Can load certificates"); + keystore + .verify_signature(request.cert, request.sig, request.sig_alg, request.data) + .expect("Signature and cert can be validated") +} + +#[test] +#[serial] +fn invalid_certificate_test() { + // Having + let (resource_cert_dir, test_cert_dir) = TEST_RESOURCES.init_cert_dirs(); + load_certificates_from_dir(&resource_cert_dir, &test_cert_dir, &[]); + + let request = prepare_request(resource_cert_dir); + + // Then + let keystore = Keystore::load(&test_cert_dir).expect("Can load certificates"); + let result = + keystore.verify_signature(request.cert, request.sig, request.sig_alg, request.data); + assert!( + result.is_err(), + "Keystore has no intermediate cert - verification should fail" + ); + let err = result.expect_err("Error result"); + let msg = format!("{err:?}"); + assert_eq!(msg, "Invalid certificate"); +} + +struct SignedRequest { + cert: String, + sig: String, + sig_alg: String, + data: String, +} + +fn prepare_request(resource_cert_dir: PathBuf) -> SignedRequest { + let resource_dir = TEST_RESOURCES.test_resources_dir_path(); + + let mut cert = resource_cert_dir; + cert.push("foo_req.cert.pem"); + let mut cert = fs::read_to_string(cert).expect("Can read certificate file"); + cert = base64::encode(cert); + + let mut data = resource_dir.clone(); + data.push("data.json.base64"); + let data = fs::read_to_string(data).expect("Can read resource file"); + + let mut sig = resource_dir; + sig.push("data.json.base64.foo_req_sign.sha256.base64"); + let sig = fs::read_to_string(sig).expect("Can read resource file"); + + let sig_alg = "sha256".to_string(); + + SignedRequest { + cert, + sig, + sig_alg, + data, + } +} diff --git a/utils/networking/Cargo.toml b/utils/networking/Cargo.toml index f1dd79e6a0..5a5800bc3e 100644 --- a/utils/networking/Cargo.toml +++ b/utils/networking/Cargo.toml @@ -1,12 +1,12 @@ [package] name = "ya-utils-networking" -version = "0.1.1" +version = "0.2.0" authors = ["Golem Factory "] edition = "2018" [features] default = ["dns"] -dns = ["anyhow", "url", "trust-dns-resolver/mdns"] +dns = ["anyhow", "url", "trust-dns-resolver/tokio-openssl"] vpn = ["ya-relay-stack", "ipnet", "thiserror"] [dependencies] @@ -15,11 +15,10 @@ lazy_static = "1.4" log = "0.4" regex = "1" -ya-relay-stack = { git = "https://github.com/golemfactory/ya-relay.git", rev = "0e6863c24767a246531d038455921f12c9e75e94", optional = true } +ya-relay-stack = { git = "https://github.com/golemfactory/ya-relay.git", rev = "ec174a250fc014fba084088a90f9b1c95d613ed3", optional = true } anyhow = { version = "1.0", optional = true } -trust-dns-resolver = { version = "0.19", optional = true } -tokio-compat-02 = "0.2" +trust-dns-resolver = { version = "0.21", optional = true } url = { version = "2.2", optional = true } ipnet = { version = "2.3", optional = true } diff --git a/utils/networking/src/resolver.rs b/utils/networking/src/resolver.rs index b06f7f85fa..3da6e766be 100644 --- a/utils/networking/src/resolver.rs +++ b/utils/networking/src/resolver.rs @@ -3,13 +3,12 @@ use regex::Regex; use std::io::{Error as IoError, ErrorKind as IoErrorKind}; use std::iter::FromIterator; use std::net::IpAddr; -use tokio_compat_02::FutureExt; use trust_dns_resolver::config::{ResolverConfig, ResolverOpts}; use trust_dns_resolver::TokioAsyncResolver; use url::Url; -const DEFAULT_LOOKUP_DOMAIN: &'static str = "dev.golem.network"; +const DEFAULT_LOOKUP_DOMAIN: &str = "dev.golem.network"; /// Resolves prefixes in the `DEFAULT_LOOKUP_DOMAIN`, see also `resolve_record` pub async fn resolve_yagna_srv_record(prefix: &str) -> std::io::Result { @@ -25,10 +24,8 @@ pub async fn resolve_yagna_srv_record(prefix: &str) -> std::io::Result { /// If successful responds in the format of `hostname:port` pub async fn resolve_srv_record(record: &str) -> std::io::Result { let resolver: TokioAsyncResolver = - TokioAsyncResolver::tokio(ResolverConfig::google(), ResolverOpts::default()) - .compat() - .await?; - let lookup = resolver.srv_lookup(record).compat().await?; + TokioAsyncResolver::tokio(ResolverConfig::google(), ResolverOpts::default())?; + let lookup = resolver.srv_lookup(record).await?; let srv = lookup .iter() .next() @@ -49,7 +46,7 @@ pub async fn resolve_srv_record(record: &str) -> std::io::Result { pub async fn resolve_dns_record(request_url: &str) -> anyhow::Result { let request_host = Url::parse(request_url)? .host() - .ok_or(anyhow::anyhow!("Invalid url: {}", request_url))? + .ok_or_else(|| anyhow::anyhow!("Invalid url: {}", request_url))? .to_string(); let address = resolve_dns_record_host(&request_host).await?; @@ -57,15 +54,13 @@ pub async fn resolve_dns_record(request_url: &str) -> anyhow::Result { } pub async fn resolve_dns_record_host(host: &str) -> anyhow::Result { - let resolver = TokioAsyncResolver::tokio(ResolverConfig::google(), ResolverOpts::default()) - .compat() - .await?; + let resolver = TokioAsyncResolver::tokio(ResolverConfig::google(), ResolverOpts::default())?; - let response = resolver.lookup_ip(host).compat().await?; + let response = resolver.lookup_ip(host).await?; let address = response .iter() .next() - .ok_or(anyhow::anyhow!("DNS resolution failed for host: {}", host))? + .ok_or_else(|| anyhow::anyhow!("DNS resolution failed for host: {}", host))? .to_string(); Ok(address) } @@ -97,8 +92,7 @@ pub async fn try_resolve_dns_record(request_url_or_host: &str) -> String { /// Resolve all known IP addresses of a given domain pub async fn resolve_domain_name>(domain: &str) -> anyhow::Result { - let resolver = - TokioAsyncResolver::tokio(ResolverConfig::google(), ResolverOpts::default()).await?; + let resolver = TokioAsyncResolver::tokio(ResolverConfig::google(), ResolverOpts::default())?; let response = resolver.lookup_ip(domain).await?; Ok(T::from_iter(response.iter())) } diff --git a/utils/networking/src/vpn/common.rs b/utils/networking/src/vpn/common.rs index 0e5a9236bf..38c8a36b65 100644 --- a/utils/networking/src/vpn/common.rs +++ b/utils/networking/src/vpn/common.rs @@ -7,7 +7,7 @@ use ipnet::IpNet; use ya_relay_stack::Error; pub const DEFAULT_MAX_FRAME_SIZE: usize = 1500; -pub const DEFAULT_IPV4_NET_MASK: &'static str = "255.255.255.0"; +pub const DEFAULT_IPV4_NET_MASK: &str = "255.255.255.0"; #[inline(always)] pub fn hton(ip: IpAddr) -> Box<[u8]> { @@ -31,13 +31,13 @@ pub fn ntoh(data: &[u8]) -> Option { } pub fn to_ip(ip: &str) -> Result { - let ip = IpAddr::from_str(ip.as_ref()).map_err(Error::from)?; + let ip = IpAddr::from_str(ip).map_err(Error::from)?; if ip.is_loopback() || ip.is_unspecified() || ip.is_multicast() { - return Err(Error::IpAddrNotAllowed(ip).into()); + return Err(Error::IpAddrNotAllowed(ip)); } else if let IpAddr::V4(ip4) = &ip { if ip4.is_broadcast() { - return Err(Error::IpAddrNotAllowed(ip).into()); + return Err(Error::IpAddrNotAllowed(ip)); } } @@ -55,7 +55,7 @@ pub fn to_net>(ip: &str, mask: Option) -> Result let result = match ip.find('/') { Some(_) => IpNet::from_str(ip), None => { - let ip = IpAddr::from_str(&ip)?; + let ip = IpAddr::from_str(ip)?; let cidr = match &ip { IpAddr::V4(_) => { let mask = mask @@ -70,5 +70,5 @@ pub fn to_net>(ip: &str, mask: Option) -> Result IpNet::from_str(&format!("{}/{}", ip, cidr)) } }; - Ok(result.map_err(|_| Error::NetAddr(ip.to_string()))?) + result.map_err(|_| Error::NetAddr(ip.to_string())) } diff --git a/utils/networking/src/vpn/network.rs b/utils/networking/src/vpn/network.rs index b0e0714af1..3a7bd68818 100644 --- a/utils/networking/src/vpn/network.rs +++ b/utils/networking/src/vpn/network.rs @@ -24,7 +24,7 @@ impl Default for Networks { impl Networks { pub fn get_mut(&mut self, id: &str) -> Result<&mut Network, Error> { - self.networks.get_mut(id).ok_or_else(|| Error::NetNotFound) + self.networks.get_mut(id).ok_or(Error::NetNotFound) } pub fn endpoint>(&self, ip: B) -> Option { @@ -50,8 +50,7 @@ impl Networks { if self .networks .values() - .find(|n| n.as_ref() == &network || n.as_ref().contains(&network.addr())) - .is_some() + .any(|n| n.as_ref() == &network || n.as_ref().contains(&network.addr())) { return Err(Error::NetAddrTaken(network.addr())); } @@ -96,11 +95,7 @@ impl Network { } pub fn address(&self) -> Result { - self.addresses - .iter() - .next() - .cloned() - .ok_or_else(|| Error::NetEmpty) + self.addresses.iter().next().cloned().ok_or(Error::NetEmpty) } pub fn endpoints(&self) -> &HashMap, E> { @@ -112,7 +107,7 @@ impl Network { } pub fn add_address(&mut self, ip: &str) -> Result<(), Error> { - let ip = to_ip(ip.as_ref())?; + let ip = to_ip(ip)?; if !self.network.contains(&ip) { return Err(Error::NetAddr(ip.to_string())); } @@ -129,7 +124,7 @@ impl Network { } let node_id = id.to_string(); - let ip: Box<[u8]> = hton(ip_addr).into(); + let ip: Box<[u8]> = hton(ip_addr); if self.endpoints.contains_key(&ip) { return Err(Error::IpAddrTaken(ip_addr)); @@ -145,11 +140,11 @@ impl Network { } pub fn remove_node(&mut self, node_id: &str) { - self.nodes.remove(node_id).map(|addrs| { + if let Some(addrs) = self.nodes.remove(node_id) { addrs.into_iter().for_each(|a| { self.endpoints.remove(&to_octets(a)); }); - }); + } } } diff --git a/utils/path/src/data_dir.rs b/utils/path/src/data_dir.rs index 25282a07f8..2626071e6d 100644 --- a/utils/path/src/data_dir.rs +++ b/utils/path/src/data_dir.rs @@ -5,7 +5,7 @@ use std::{ops::Not, path::PathBuf, str::FromStr, string::ToString}; const ORGANIZATION: &str = "GolemFactory"; const QUALIFIER: &str = ""; -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct DataDir(PathBuf); impl DataDir { diff --git a/utils/path/src/lib.rs b/utils/path/src/lib.rs index ba829d8fc3..3f2066d902 100644 --- a/utils/path/src/lib.rs +++ b/utils/path/src/lib.rs @@ -78,6 +78,7 @@ pub fn normalize_path>(path: P) -> std::io::Result { Ok(path) } +#[allow(clippy::unnecessary_filter_map)] fn remove_insecure_chars>(path: PathRef) -> PathBuf { let path = path.as_ref().to_path_buf(); path.components() diff --git a/utils/process/Cargo.toml b/utils/process/Cargo.toml index 6e2f5fe8ac..37aa85d4bd 100644 --- a/utils/process/Cargo.toml +++ b/utils/process/Cargo.toml @@ -1,8 +1,13 @@ [package] name = "ya-utils-process" -version = "0.1.0" +version = "0.2.0" authors = ["Golem Factory "] edition = "2018" +homepage = "https://github.com/golemfactory/yagna" +repository = "https://github.com/golemfactory/yagna" +license = "LGPL-3.0" +description="Yagna utils for asynchronous process handling" +keywords=["golem", "yagna", "process"] [features] default = [] diff --git a/utils/process/src/lib.rs b/utils/process/src/lib.rs index 205da85dad..7608868a17 100644 --- a/utils/process/src/lib.rs +++ b/utils/process/src/lib.rs @@ -30,7 +30,7 @@ impl ProcessGroupExt for Command { unsafe { self.pre_exec(|| { - nix::unistd::setsid().map_err(|e| io::Error::from(e))?; + nix::unistd::setsid().map_err(io::Error::from)?; Ok(()) }); } @@ -50,7 +50,7 @@ impl ProcessGroupExt for tokio::process::Command { unsafe { self.pre_exec(|| { - nix::unistd::setsid().map_err(|e| io::Error::from(e))?; + nix::unistd::setsid().map_err(io::Error::from)?; Ok(()) }); } @@ -79,9 +79,9 @@ pub struct ProcessHandle { } impl ProcessHandle { - pub fn new(mut command: &mut Command) -> Result { + pub fn new(command: &mut Command) -> Result { Ok(ProcessHandle { - process: Arc::new(SharedChild::spawn(&mut command)?), + process: Arc::new(SharedChild::spawn(command)?), }) } @@ -96,7 +96,7 @@ impl ProcessHandle { #[cfg(unix)] pub async fn terminate(&self, timeout: Duration) -> Result<()> { let process = self.process.clone(); - if let Err(_) = process.send_signal(libc::SIGTERM) { + if process.send_signal(libc::SIGTERM).is_err() { // Error means, that probably process was already terminated, because: // - We have permissions to send signal, since we created this process. // - We specified correct signal SIGTERM. @@ -164,6 +164,6 @@ impl ProcessHandle { // Note: unwrap can't fail here. All sender, receiver and thread will // end their lifetime before await will return. There's no danger // that one of them will be dropped earlier. - return receiver.await.unwrap(); + receiver.await.unwrap() } } diff --git a/utils/process/src/lock.rs b/utils/process/src/lock.rs index 67cd15f629..759c49b584 100644 --- a/utils/process/src/lock.rs +++ b/utils/process/src/lock.rs @@ -4,8 +4,8 @@ use std::fs::File; use std::io::Write; use std::path::{Path, PathBuf}; -const LOCK_FILE_EXT: &'static str = "lock"; -const PID_FILE_EXT: &'static str = "pid"; +const LOCK_FILE_EXT: &str = "lock"; +const PID_FILE_EXT: &str = "pid"; pub struct ProcLock { dir: PathBuf, @@ -43,17 +43,15 @@ impl ProcLock { }) .unwrap_or(false) }) - .filter(|p| match File::open(&p) { + .any(|p| match File::open(&p) { Ok(f) => f.try_lock_exclusive().is_err(), _ => true, - }) - .next() - .is_some()) + })) } pub fn lock(mut self, pid: u32) -> Result { let (lock_file, lock_path) = self.lock_file(&self.name)?; - if let Err(_) = lock_file.try_lock_exclusive() { + if lock_file.try_lock_exclusive().is_err() { bail!("{} is already running", self.name); } @@ -81,7 +79,7 @@ impl ProcLock { pub fn read_pid(&self) -> Result { let (lock_file, _) = self.lock_file(&self.name)?; - if let Ok(_) = lock_file.try_lock_exclusive() { + if lock_file.try_lock_exclusive().is_ok() { let _ = lock_file.unlock(); bail!("{} is not running", self.name); } diff --git a/utils/scheduler/src/lib.rs b/utils/scheduler/src/lib.rs index 62255c9377..0f86f641f2 100644 --- a/utils/scheduler/src/lib.rs +++ b/utils/scheduler/src/lib.rs @@ -14,10 +14,7 @@ pub struct Job { impl Job { fn new(task: Task, trigger: Trigger) -> Job { - Job { - task: task, - trigger: trigger, - } + Job { task, trigger } } fn is_pending(&self) -> bool { @@ -36,14 +33,14 @@ pub struct Scheduler { jobs: Vec, } -impl<'a> Scheduler { +impl Scheduler { pub fn new(name: T, tick_time: u64) -> Scheduler where T: Into, { Scheduler { name: name.into(), - tick_time: tick_time, + tick_time, jobs: vec![], } } diff --git a/utils/scheduler/src/trigger.rs b/utils/scheduler/src/trigger.rs index f91cb94af6..491f676aa9 100644 --- a/utils/scheduler/src/trigger.rs +++ b/utils/scheduler/src/trigger.rs @@ -12,16 +12,15 @@ pub struct Interval { impl Interval { pub fn new(days: u32, hours: u32, minutes: u32, seconds: u32) -> Interval { Interval { - days: days, - hours: hours, - minutes: minutes, - seconds: seconds, + days, + hours, + minutes, + seconds, } } fn next(&self, from: DateTime) -> DateTime { - from.clone() - + Duration::days(i64::from(self.days)) + from + Duration::days(i64::from(self.days)) + Duration::hours(i64::from(self.hours)) + Duration::minutes(i64::from(self.minutes)) + Duration::seconds(i64::from(self.seconds)) @@ -42,7 +41,7 @@ impl Trigger { { Trigger { name: name.into(), - interval: interval, + interval, next_run: start_from, last_run: None, } @@ -124,7 +123,7 @@ mod tests { let days = 1; let now = Local::now(); let interval = Interval::new(days, 0, 0, 0); - let mut trigger = Trigger::new("trigger1", now.clone(), interval); + let mut trigger = Trigger::new("trigger1", now, interval); trigger.tick(); assert_ne!(trigger.last_run, None); assert_eq!(trigger.next_run, now + Duration::days(i64::from(days))); diff --git a/utils/std-utils/Cargo.toml b/utils/std-utils/Cargo.toml index 0facf24c46..83760e4d89 100644 --- a/utils/std-utils/Cargo.toml +++ b/utils/std-utils/Cargo.toml @@ -4,7 +4,6 @@ version = "0.1.0" authors = ["Golem Factory "] edition = "2018" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] log = "0.4" diff --git a/utils/std-utils/src/result.rs b/utils/std-utils/src/result.rs index 0b088c53be..c50acc8f91 100644 --- a/utils/std-utils/src/result.rs +++ b/utils/std-utils/src/result.rs @@ -37,10 +37,10 @@ impl LogErr for Result { if let Some(name) = symbol.name() { let module_path = name.to_string(); if module_path.starts_with(""] edition = "2018" [dependencies] -ya-client-model = "0.4" -ya-core-model = { version = "^0.6" } -ya-service-bus = "0.4" +ya-client-model = "0.5" +ya-core-model = { version = "^0.8" } +ya-service-bus = "0.6" ya-utils-path = "0.1" -gftp = { version = "^0.2"} +gftp = { version = "^0.3" } actix-http = "3" actix-web = "4" diff --git a/utils/transfer/examples/gftp.rs b/utils/transfer/examples/gftp.rs index 2c84e599f1..cb19d1c9dc 100644 --- a/utils/transfer/examples/gftp.rs +++ b/utils/transfer/examples/gftp.rs @@ -32,7 +32,7 @@ fn create_file(path: &Path, name: &str, chunk_size: usize, chunk_count: usize) - let input: Vec = (0..chunk_size).map(|_| rng.gen_range(0..=255u8)).collect(); hasher.input(&input); - file_src.write(&input).unwrap(); + let _ = file_src.write(&input).unwrap(); } file_src.flush().unwrap(); hasher.result() @@ -55,12 +55,15 @@ fn hash_file(path: &Path) -> HashOutput { #[actix_rt::main] async fn main() -> Result<(), Error> { - env::set_var("RUST_LOG", env::var("RUST_LOG").unwrap_or("info".into())); + env::set_var( + "RUST_LOG", + env::var("RUST_LOG").unwrap_or_else(|_| "info".into()), + ); env_logger::init(); let temp_dir = TempDir::new("transfer").unwrap(); - let chunk_size = 4096 as usize; - let chunk_count = 256 as usize; + let chunk_size = 4096_usize; + let chunk_count = 256_usize; log::info!( "Creating a random file of size {} * {}", diff --git a/utils/transfer/src/archive.rs b/utils/transfer/src/archive.rs index 2ab270d709..846f3d8810 100644 --- a/utils/transfer/src/archive.rs +++ b/utils/transfer/src/archive.rs @@ -77,7 +77,7 @@ impl<'s> TryFrom<&'s str> for ArchiveFormat { } } -impl<'s> TryFrom<&TransferArgs> for ArchiveFormat { +impl TryFrom<&TransferArgs> for ArchiveFormat { type Error = Error; fn try_from(args: &TransferArgs) -> Result { @@ -191,12 +191,12 @@ where let fut = async move { let mut path_iter = path_iter.peekable(); - if let None = path_iter.peek() { + if path_iter.peek().is_none() { return Err(io::Error::from(io::ErrorKind::InvalidData)); } let writer = TokioAsyncWrite( - tx.sink_map_err(|e| io_error(e)) + tx.sink_map_err(io_error) .with(|b| futures::future::ok::<_, io::Error>(Ok(b))), ); let mut builder = tokio_tar::Builder::new(writer); @@ -391,44 +391,40 @@ where let path = path.as_ref(); let mut reader = TokioAsyncRead(stream.into_async_read()); - loop { - if let Some(mut result) = read_zipfile_from_stream(&mut reader) - .await - .map_err(|e| io_error(e))? - { - let name = result.sanitized_name(); - let size = result.size() as usize; - let file_path = path.join(&name); - - let _ = evt_sender - .send(FileEvent::Processing { - name: name.clone(), - size, - is_dir: result.is_dir(), - }) - .await; - - if result.is_dir() { - create_dir_all(file_path)?; - } else { - if let Some(parent) = file_path.parent() { - create_dir_all(parent)?; - } - let mut file = OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .open(file_path) - .await?; - copy(&mut result, &mut file).await?; - file.flush().await?; - } - - let _ = evt_sender.send(FileEvent::Finished { name }).await; - result.exhaust().await; + while let Some(mut result) = read_zipfile_from_stream(&mut reader) + .await + .map_err(io_error)? + { + let name = result.sanitized_name(); + let size = result.size() as usize; + let file_path = path.join(&name); + + let _ = evt_sender + .send(FileEvent::Processing { + name: name.clone(), + size, + is_dir: result.is_dir(), + }) + .await; + + if result.is_dir() { + create_dir_all(file_path)?; } else { - break; + if let Some(parent) = file_path.parent() { + create_dir_all(parent)?; + } + let mut file = OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(file_path) + .await?; + copy(&mut result, &mut file).await?; + file.flush().await?; } + + let _ = evt_sender.send(FileEvent::Finished { name }).await; + result.exhaust().await; } Ok(()) @@ -456,10 +452,7 @@ where let evt = FileEvent::Processing { name: name.clone(), size: header.size().ok().unwrap_or(0) as usize, - is_dir: match header.entry_type() { - tokio_tar::EntryType::Directory => true, - _ => false, - }, + is_dir: header.entry_type() == tokio_tar::EntryType::Directory, }; let _ = evt_sender.send(evt).await; @@ -471,9 +464,7 @@ where Ok(()) } -fn codec_stream<'a, R>( - s: R, -) -> Pin> + Send + Sync + 'static>> +fn codec_stream(s: R) -> Pin> + Send + Sync + 'static>> where R: AsyncRead + Send + Sync + Unpin + 'static, { diff --git a/utils/transfer/src/file.rs b/utils/transfer/src/file.rs index 08e00475a3..47eb6efd48 100644 --- a/utils/transfer/src/file.rs +++ b/utils/transfer/src/file.rs @@ -13,17 +13,14 @@ use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt, BufReader, SeekFrom}; use tokio::task::spawn_local; use url::Url; +#[derive(Default)] pub struct FileTransferProvider; + +#[derive(Default)] pub struct DirTransferProvider; pub const DEFAULT_CHUNK_SIZE: usize = 40 * 1024; -impl Default for FileTransferProvider { - fn default() -> Self { - FileTransferProvider {} - } -} - impl TransferProvider for FileTransferProvider { fn schemes(&self) -> Vec<&'static str> { vec!["file"] @@ -55,7 +52,7 @@ impl TransferProvider for FileTransferProvider { reader.read_to_end(&mut vec).await?; vec }; - if vec.len() == 0 { + if vec.is_empty() { break; } @@ -74,7 +71,7 @@ impl TransferProvider for FileTransferProvider { fn destination(&self, url: &Url, ctx: &TransferContext) -> TransferSink { let (sink, mut rx, res_tx) = TransferSink::::create(1); - let path = PathBuf::from(extract_file_url(&url)); + let path = PathBuf::from(extract_file_url(url)); let path_c = path.clone(); let state = ctx.state.clone(); @@ -103,7 +100,7 @@ impl TransferProvider for FileTransferProvider { while let Some(result) = rx.next().await { let data = result?; let bytes = data.as_ref(); - if bytes.len() == 0 { + if bytes.is_empty() { break; } @@ -117,7 +114,7 @@ impl TransferProvider for FileTransferProvider { } .map_err(|error| { log::error!("Error writing to file [{}]: {}", path_c.display(), error); - Error::from(error) + error }); abortable_sink(fut, res_tx).await @@ -131,7 +128,7 @@ impl TransferProvider for FileTransferProvider { url: &Url, ctx: &TransferContext, ) -> LocalBoxFuture<'a, Result<(), Error>> { - let path = PathBuf::from(extract_file_url(&url)); + let path = PathBuf::from(extract_file_url(url)); let state = ctx.state.clone(); async move { state.set_offset(match tokio::fs::metadata(path).await { @@ -145,12 +142,6 @@ impl TransferProvider for FileTransferProvider { } } -impl Default for DirTransferProvider { - fn default() -> Self { - DirTransferProvider {} - } -} - impl TransferProvider for DirTransferProvider { fn schemes(&self) -> Vec<&'static str> { vec!["file"] @@ -178,10 +169,7 @@ impl TransferProvider for DirTransferProvider { archive(path_iter, dir, format, evt_tx) .await - .forward( - tx.sink_map_err(Error::from) - .with(|b| ready(Ok(Ok(TransferData::from(b))))), - ) + .forward(tx.sink_map_err(Error::from).with(|b| ready(Ok(Ok(b))))) .await }; diff --git a/utils/transfer/src/gftp.rs b/utils/transfer/src/gftp.rs index 020298882d..70499c852c 100644 --- a/utils/transfer/src/gftp.rs +++ b/utils/transfer/src/gftp.rs @@ -12,7 +12,7 @@ use url::Url; use ya_core_model::gftp as model; use ya_core_model::gftp::Error as GftpError; use ya_core_model::gftp::GftpChunk; -use ya_core_model::net::TryRemoteEndpoint; +use ya_core_model::net::RemoteEndpoint; use ya_service_bus::RpcEndpoint; pub struct GftpTransferProvider { @@ -43,7 +43,7 @@ impl TransferProvider for GftpTransferProvider { let (node_id, hash) = gftp::extract_url(&url) .map_err(|_| Error::InvalidUrlError("Invalid gftp URL".to_owned()))?; - let remote = node_id.try_service(&model::file_bus_id(&hash))?; + let remote = node_id.service_transfer(&model::file_bus_id(&hash)); let meta = remote.send(model::GetMetadata {}).await??; let n = (meta.file_size + chunk_size - 1) / chunk_size; @@ -89,7 +89,7 @@ impl TransferProvider for GftpTransferProvider { let fut = async move { let (node_id, random_filename) = gftp::extract_url(&url) .map_err(|_| Error::InvalidUrlError("invalid gftp URL".into()))?; - let remote = node_id.try_service(&model::file_bus_id(&random_filename))?; + let remote = node_id.service_transfer(&model::file_bus_id(&random_filename)); let digest_fut = async move { let mut digest = Sha3_256::default(); @@ -116,6 +116,11 @@ impl TransferProvider for GftpTransferProvider { }; let send_fut = chunk_rx.try_for_each_concurrent(concurrency, |chunk| async { + log::trace!( + "Sending chunk ({} B) at offset: {}", + chunk.content.len(), + chunk.offset + ); remote.call(model::UploadChunk { chunk }).await??; Ok(()) }); @@ -130,8 +135,12 @@ impl TransferProvider for GftpTransferProvider { Err(either) => return Err(either.factor_first().0), }; - let hash = Some(format!("{:x}", digest)); - remote.call(model::UploadFinished { hash }).await??; + let hash = format!("{:x}", digest); + log::debug!("Finishing transfer of {url}. Hash: {hash}. Sending UploadFinished.."); + + remote + .call(model::UploadFinished { hash: Some(hash) }) + .await??; Result::<(), Error>::Ok(()) } .map_err(Error::from); diff --git a/utils/transfer/src/http.rs b/utils/transfer/src/http.rs index b5a44d267a..97bb1b2026 100644 --- a/utils/transfer/src/http.rs +++ b/utils/transfer/src/http.rs @@ -135,8 +135,7 @@ impl TransferProvider for HttpTransferProvider { let size: Option = response .headers() .get(header::CONTENT_LENGTH) - .map(|v| v.to_str().ok().map(|s| u64::from_str(s).ok()).flatten()) - .flatten(); + .and_then(|v| v.to_str().ok().and_then(|s| u64::from_str(s).ok())); state.set_size(size); if !ranges { @@ -209,15 +208,14 @@ impl DownloadRequest { .await?; let is_redirect = resp.status().is_redirection(); - if (!is_redirect) || (is_redirect && redirects == 0) { + if !is_redirect || redirects == 0 { return Ok(resp); } match resp .headers() .get(header::LOCATION) - .map(|v| v.to_str().ok()) - .flatten() + .and_then(|v| v.to_str().ok()) { Some(location) => { url = location.to_string(); @@ -274,7 +272,7 @@ impl<'a> HttpErr>> for Se fn http_err(self) -> Result>, Error> { match self { SendClientRequest::Fut(fut, _, _) => { - Ok(async move { Ok(fut.await?.http_err()?) }.boxed_local()) + Ok(async move { fut.await?.http_err() }.boxed_local()) } SendClientRequest::Err(err) => Err(err .map(|e| e.into()) diff --git a/utils/transfer/src/lib.rs b/utils/transfer/src/lib.rs index cad62e89b4..b7e9ee51b6 100644 --- a/utils/transfer/src/lib.rs +++ b/utils/transfer/src/lib.rs @@ -41,7 +41,7 @@ where { let rx = sink.res_rx.take().unwrap(); stream.forward(sink).await?; - Ok(rx.await??) + rx.await? } /// Transfers data between `TransferProvider`s within current context @@ -66,7 +66,7 @@ where log::debug!("Transferring from offset: {}", ctx.state.offset()); - let stream = wrap_stream(src.source(&src_url.url, ctx), &src_url)?; + let stream = wrap_stream(src.source(&src_url.url, ctx), src_url)?; let sink = dst.destination(&dst_url.url, ctx); transfer(stream, sink).await?; @@ -191,6 +191,7 @@ pub struct TransferSink { res_rx: Option>>, } +#[allow(clippy::type_complexity)] impl TransferSink { pub fn create( channel_size: usize, @@ -330,7 +331,7 @@ impl TransferState { } pub fn size(&self) -> Option { - self.inner.borrow().size.clone() + self.inner.borrow().size } pub fn set_size(&self, size: Option) { @@ -351,8 +352,7 @@ impl TransferState { (*self.inner.borrow_mut()) .retry .as_mut() - .map(|r| r.delay(&err)) - .flatten() + .and_then(|r| r.delay(err)) } } diff --git a/utils/transfer/src/location.rs b/utils/transfer/src/location.rs index bb10ee96d9..6be55d91cd 100644 --- a/utils/transfer/src/location.rs +++ b/utils/transfer/src/location.rs @@ -18,13 +18,13 @@ impl UrlExt for url::Url { } } -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct TransferHash { pub alg: String, pub val: Vec, } -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct TransferUrl { pub hash: Option, pub url: Url, diff --git a/utils/transfer/src/retry.rs b/utils/transfer/src/retry.rs index 6dafeb25f6..e2c509f774 100644 --- a/utils/transfer/src/retry.rs +++ b/utils/transfer/src/retry.rs @@ -50,13 +50,13 @@ impl Iterator for Retry { self.backoff *= self.backoff_factor; let duration = Duration::from_secs_f32(self.backoff); - if self.count > 0 { - self.count -= 1; - Some(duration) - } else if self.count < 0 { - Some(duration) - } else { - None + match self.count.cmp(&0) { + std::cmp::Ordering::Less => Some(duration), + std::cmp::Ordering::Equal => None, + std::cmp::Ordering::Greater => { + self.count -= 1; + Some(duration) + } } } } @@ -65,26 +65,26 @@ fn can_retry(err: &Error) -> bool { match err { Error::HttpError(e) => match e { HttpError::Timeout(_) | HttpError::Connect(_) | HttpError::Server(_) => true, - HttpError::Io(kind) => match kind { + HttpError::Io(kind) => matches!( + kind, ErrorKind::ConnectionReset - | ErrorKind::ConnectionAborted - | ErrorKind::NotConnected - | ErrorKind::AddrNotAvailable - | ErrorKind::BrokenPipe - | ErrorKind::TimedOut - | ErrorKind::Interrupted => true, - _ => false, - }, + | ErrorKind::ConnectionAborted + | ErrorKind::NotConnected + | ErrorKind::AddrNotAvailable + | ErrorKind::BrokenPipe + | ErrorKind::TimedOut + | ErrorKind::Interrupted + ), _ => false, }, - Error::Gsb(e) => match e { + Error::Gsb(e) => matches!( + e, BusError::Timeout(_) - | BusError::Closed(_) - | BusError::ConnectionFail(_, _) - | BusError::ConnectionTimeout(_) - | BusError::NoEndpoint(_) => true, - _ => false, - }, + | BusError::Closed(_) + | BusError::ConnectionFail(_, _) + | BusError::ConnectionTimeout(_) + | BusError::NoEndpoint(_) + ), _ => false, } } diff --git a/utils/transfer/src/traverse.rs b/utils/transfer/src/traverse.rs index 74e8d9668c..9d6bf9e639 100644 --- a/utils/transfer/src/traverse.rs +++ b/utils/transfer/src/traverse.rs @@ -223,7 +223,7 @@ mod tests { fn include_in(set: &mut Option>, glob: S) { let glob = glob.to_string(); - let includes = match mem::replace(set, None) { + let includes = match set.take() { Some(entry) => match entry { SetEntry::Single(e) => Some(SetEntry::Multiple(vec![e, glob])), SetEntry::Multiple(mut v) => { @@ -261,21 +261,21 @@ mod tests { let dir = tempdir::TempDir::new("test-glob")?; create_files(dir.path()); - let items = Builder::new() + let items_count = Builder::new() .include("*.txt") .build() .traverse(dir.path())? - .collect::>(); + .count(); - assert_eq!(items.len(), 3); + assert_eq!(items_count, 3); - let items = Builder::new() + let items_count = Builder::new() .include("**/*.txt") .build() .traverse(dir.path())? - .collect::>(); + .count(); - assert_eq!(items.len(), 5); + assert_eq!(items_count, 5); Ok(()) } @@ -285,12 +285,9 @@ mod tests { let dir = tempdir::TempDir::new("test-glob")?; create_files(dir.path()); - let items = Builder::new() - .build() - .traverse(dir.path())? - .collect::>(); + let items_count = Builder::new().build().traverse(dir.path())?.count(); - assert_eq!(items.len(), 3 + 9); + assert_eq!(items_count, 3 + 9); Ok(()) } @@ -339,29 +336,29 @@ mod tests { let dir = tempdir::TempDir::new("test-glob").unwrap(); create_files(dir.path()); - let entries = Builder::new() + let entries_count = Builder::new() .include("**/*.txt") .depth(0) .build() .traverse(dir.path())? - .collect::>(); - assert_eq!(entries.len(), 3 as usize); + .count(); + assert_eq!(entries_count, 3_usize); - let entries = Builder::new() + let entries_count = Builder::new() .include("**/*.txt") .depth(1) .build() .traverse(dir.path())? - .collect::>(); - assert_eq!(entries.len(), 4 as usize); + .count(); + assert_eq!(entries_count, 4_usize); - let entries = Builder::new() + let entries_count = Builder::new() .include("**/*.txt") .depth(2) .build() .traverse(dir.path())? - .collect::>(); - assert_eq!(entries.len(), 5 as usize); + .count(); + assert_eq!(entries_count, 5_usize); Ok(()) }