From e1382b73b80d67023b247db88e141cc87d592054 Mon Sep 17 00:00:00 2001 From: Bohdan Khorolets Date: Sun, 20 Feb 2022 13:14:09 +0200 Subject: [PATCH 01/42] refactor: Switch from runing NEAR Indexer Framework to NEAR Lake Framework --- Cargo.lock | 1510 +++++++++++++++++++++++----- Cargo.toml | 13 +- src/configs.rs | 115 +-- src/db_adapters/account_changes.rs | 14 + src/main.rs | 150 +-- 5 files changed, 1294 insertions(+), 508 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9f210c0d..03f7af6d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -101,7 +101,7 @@ dependencies = [ "h2", "http", "httparse", - "itoa", + "itoa 0.4.8", "language-tags", "local-channel", "log", @@ -234,7 +234,7 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "itoa", + "itoa 0.4.8", "language-tags", "log", "mime", @@ -415,7 +415,7 @@ dependencies = [ "cookie", "derive_more", "futures-core", - "itoa", + "itoa 0.4.8", "log", "mime", "percent-encoding", @@ -426,6 +426,302 @@ dependencies = [ "serde_urlencoded", ] +[[package]] +name = "aws-config" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12b964849038df43a2b4e0c20e29b67451af5a93108d757dd58b9e82f41a0ee8" +dependencies = [ + "aws-http", + "aws-sdk-sso", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-types", + "aws-types", + "bytes", + "hex", + "http", + "hyper", + "ring", + "tokio", + "tower", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-endpoint" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06d059b181b25940b751e8efecc173ceb4fe65f45d8975f56b02e98db5c42fd6" +dependencies = [ + "aws-smithy-http", + "aws-types", + "http", + "regex", + "tracing", +] + +[[package]] +name = "aws-http" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3049066e3282c98bbf01e90459a1772ccf6c0b96cd1483c3dd5aa34bef9b9de1" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "aws-types", + "http", + "lazy_static", + "percent-encoding", + "tracing", +] + +[[package]] +name = "aws-sdk-s3" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d70be50ac07c3c2b5f37056271856ac00190e80c19c76c58bcbee5be0b63ec9" +dependencies = [ + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "http", + "md5", + "tokio-stream", + "tower", +] + +[[package]] +name = "aws-sdk-sso" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "222fcabbf95f1f13c4e28cab95c9a4bb02606f998b1ea2800713b6866be5701d" +dependencies = [ + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-types", + "aws-types", + "bytes", + "http", + "tokio-stream", + "tower", +] + +[[package]] +name = "aws-sdk-sts" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d85b9f081af2c73ee25642de1a35fa9ba7b2432e54f6bf42242e478ae53c3beb" +dependencies = [ + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-query", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "http", + "tower", +] + +[[package]] +name = "aws-sig-auth" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4012b5192350b5403aba19a01a5a3b1768158dab936c4269d89760970d4812bc" +dependencies = [ + "aws-sigv4", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-types", + "http", + "thiserror", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f4b9c0c3a34e5152a0cd5e43b8f2cfd780e3bd7a245948d8787e051095ac4c" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-http", + "bytes", + "form_urlencoded", + "hex", + "http", + "once_cell", + "percent-encoding", + "regex", + "ring", + "time 0.3.7", + "tracing", +] + +[[package]] +name = "aws-smithy-async" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b69dad0aefb1b64e63e0d3a1310dc50191608d8c9e226f2f241f344a7173642e" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", + "tokio-stream", +] + +[[package]] +name = "aws-smithy-client" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e47a8aca2194672518d6630936507d3b54598c482f13ffe53f9b7932724bbb" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-types", + "bytes", + "fastrand", + "http", + "http-body", + "hyper", + "hyper-rustls", + "lazy_static", + "pin-project", + "pin-project-lite", + "tokio", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98bcfcb063d29c7cc7bb0a64830afe606090de75533c10a11a05460d814e8d9" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c8bbe92ecdc4e39a612359b09994c45d000591d4951aa7343443f44b47e6696" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http", + "http-body", + "hyper", + "percent-encoding", + "pin-project", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "aws-smithy-http-tower" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f23fdf1253855af3bb4abb25e42ad3152a71241af89014eebf27c14c7a59b81d" +dependencies = [ + "aws-smithy-http", + "bytes", + "http", + "http-body", + "pin-project", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cc19c372b0a561aa6bfc5dfdd917da7c7b1641d3bc9049ca4d7b197bb616a09" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-query" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8254e49a237e9dc0301a4683c424a825f4220420b241ec4eb51e959a70626d8a" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-types" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cde96306a54777ec8781aa510830e242de614aa5746274713f5ecac0779f644f" +dependencies = [ + "itoa 1.0.1", + "num-integer", + "ryu", + "time 0.3.7", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b0466594a86074a6e96b11284f9a9ddc90c5c5b7d6144ab357a90be49d28c4" +dependencies = [ + "thiserror", + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "433fd128ea727e9b83b34c72c6d4db1b900f067760fa27b387694fe896633142" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "rustc_version 0.4.0", + "tracing", + "zeroize", +] + [[package]] name = "backtrace" version = "0.3.64" @@ -481,9 +777,9 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.57.0" +version = "0.59.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd4865004a46a0aafb2a0a5eb19d3c9fc46ee5f063a6cfc605c69ac9ecf5263d" +checksum = "2bd2a9a458e8f4304c52c43ebb0cfbd520289f8379a52e329a38afda99bf8eb8" dependencies = [ "bitflags", "cexpr", @@ -762,6 +1058,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" +[[package]] +name = "bytes-utils" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e314712951c43123e5920a446464929adc667a5eade7f8fb3997776c9df6e54" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "bytesize" version = "1.1.0" @@ -831,9 +1137,9 @@ dependencies = [ [[package]] name = "cexpr" -version = "0.4.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4aedb84272dbe89af497cf81375129abda4fc0a9e7c5d317498c15cc30c0d27" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" dependencies = [ "nom", ] @@ -875,9 +1181,9 @@ dependencies = [ [[package]] name = "clang-sys" -version = "1.2.2" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10612c0ec0e0a1ff0e97980647cb058a6e7aedb913d01d009c406b8b7d0b26ee" +checksum = "4cc00842eed744b858222c4c9faf7243aafc6d33f92f96935263ef4d8a41ce21" dependencies = [ "glob", "libc", @@ -992,9 +1298,9 @@ checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" [[package]] name = "cpp_demangle" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea47428dc9d2237f3c6bc134472edfd63ebba0af932e783506dcfd66f10d18a" +checksum = "eeaa953eaad386a53111e47172c2fedba671e5684c8dd601a5f474f4f118710f" dependencies = [ "cfg-if 1.0.0", ] @@ -1010,18 +1316,18 @@ dependencies = [ [[package]] name = "cranelift-bforest" -version = "0.80.0" +version = "0.80.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9516ba6b2ba47b4cbf63b713f75b432fafa0a0e0464ec8381ec76e6efe931ab3" +checksum = "62fc68cdb867b7d27b5f33cd65eb11376dfb41a2d09568a1a2c2bc1dc204f4ef" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-codegen" -version = "0.80.0" +version = "0.80.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "489e5d0081f7edff6be12d71282a8bf387b5df64d5592454b75d662397f2d642" +checksum = "31253a44ab62588f8235a996cc9b0636d98a299190069ced9628b8547329b47a" dependencies = [ "cranelift-bforest", "cranelift-codegen-meta", @@ -1031,61 +1337,61 @@ dependencies = [ "log", "regalloc", "smallvec", - "target-lexicon 0.12.2", + "target-lexicon 0.12.3", ] [[package]] name = "cranelift-codegen-meta" -version = "0.80.0" +version = "0.80.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d36ee1140371bb0f69100e734b30400157a4adf7b86148dee8b0a438763ead48" +checksum = "7a20ab4627d30b702fb1b8a399882726d216b8164d3b3fa6189e3bf901506afe" dependencies = [ "cranelift-codegen-shared", ] [[package]] name = "cranelift-codegen-shared" -version = "0.80.0" +version = "0.80.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "981da52d8f746af1feb96290c83977ff8d41071a7499e991d8abae0d4869f564" +checksum = "6687d9668dacfed4468361f7578d86bded8ca4db978f734d9b631494bebbb5b8" [[package]] name = "cranelift-entity" -version = "0.80.0" +version = "0.80.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2906740053dd3bcf95ce53df0fd9b5649c68ae4bd9adada92b406f059eae461" +checksum = "c77c5d72db97ba2cb36f69037a709edbae0d29cb25503775891e7151c5c874bf" dependencies = [ "serde", ] [[package]] name = "cranelift-frontend" -version = "0.80.0" +version = "0.80.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7cb156de1097f567d46bf57a0cd720a72c3e15e1a2bd8b1041ba2fc894471b7" +checksum = "426dca83f63c7c64ea459eb569aadc5e0c66536c0042ed5d693f91830e8750d0" dependencies = [ "cranelift-codegen", "log", "smallvec", - "target-lexicon 0.12.2", + "target-lexicon 0.12.3", ] [[package]] name = "cranelift-native" -version = "0.80.0" +version = "0.80.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "166028ca0343a6ee7bddac0e70084e142b23f99c701bd6f6ea9123afac1a7a46" +checksum = "8007864b5d0c49b026c861a15761785a2871124e401630c03ef1426e6d0d559e" dependencies = [ "cranelift-codegen", "libc", - "target-lexicon 0.12.2", + "target-lexicon 0.12.3", ] [[package]] name = "cranelift-wasm" -version = "0.80.0" +version = "0.80.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5012a1cde0c8b3898770b711490d803018ae9bec2d60674ba0e5b2058a874f80" +checksum = "94cf12c071415ba261d897387ae5350c4d83c238376c8c5a96514ecfa2ea66a3" dependencies = [ "cranelift-codegen", "cranelift-entity", @@ -1129,9 +1435,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.5" +version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd" +checksum = "c00d6d2ea26e8b151d99093005cb442fb9a37aeaca582a03ec70946f49ab5ed9" dependencies = [ "cfg-if 1.0.0", "crossbeam-utils", @@ -1166,6 +1472,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "ct-logs" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8" +dependencies = [ + "sct", +] + [[package]] name = "curve25519-dalek" version = "3.2.0" @@ -1324,9 +1639,9 @@ checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" [[package]] name = "dynasm" -version = "1.1.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdc2d9a5e44da60059bd38db2d05cbb478619541b8c79890547861ec1e3194f0" +checksum = "47b1801e630bd336d0bbbdbf814de6cc749c9a400c7e3d995e6adfd455d0c83c" dependencies = [ "bitflags", "byteorder", @@ -1339,13 +1654,13 @@ dependencies = [ [[package]] name = "dynasmrt" -version = "1.1.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42276e3f205fe63887cca255aa9a65a63fb72764c30b9a6252a7c7e46994f689" +checksum = "1d428afc93ad288f6dffc1fa5f4a78201ad2eec33c5a522e51c181009eb09061" dependencies = [ "byteorder", "dynasm", - "memmap2 0.2.3", + "memmap2", ] [[package]] @@ -1409,9 +1724,9 @@ dependencies = [ [[package]] name = "enumset" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e76129da36102af021b8e5000dab2c1c30dbef85c1e482beeff8da5dde0e0b0" +checksum = "6216d2c19a6fb5f29d1ada1dc7bc4367a8cbf0fa4af5cf12e07b5bbdde6b5b2c" dependencies = [ "enumset_derive", ] @@ -1483,6 +1798,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +[[package]] +name = "fastrand" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3fcf0cee53519c866c09b5de1f6c56ff9d647101f81c1964fa632e148896cdf" +dependencies = [ + "instant", +] + [[package]] name = "fixed-hash" version = "0.7.0" @@ -1743,6 +2067,15 @@ dependencies = [ "ahash 0.7.6", ] +[[package]] +name = "hashbrown" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c21d40587b92fa6a6c6e3c1bdbf87d75511db5672f9c93175574b3a00df1758" +dependencies = [ + "ahash 0.7.6", +] + [[package]] name = "heapsize" version = "0.4.2" @@ -1784,7 +2117,7 @@ checksum = "1323096b05d41827dadeaee54c9981958c0f94e670bc94ed80037d1a7b8b186b" dependencies = [ "bytes", "fnv", - "itoa", + "itoa 0.4.8", ] [[package]] @@ -1825,7 +2158,7 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa", + "itoa 0.4.8", "pin-project-lite", "socket2", "tokio", @@ -1834,6 +2167,23 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" +dependencies = [ + "ct-logs", + "futures-util", + "hyper", + "log", + "rustls", + "rustls-native-certs", + "tokio", + "tokio-rustls", + "webpki", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -1886,7 +2236,7 @@ dependencies = [ [[package]] name = "indexer-explorer" -version = "0.10.12" +version = "0.10.12-lake.1" dependencies = [ "actix", "actix-diesel", @@ -1908,9 +2258,12 @@ dependencies = [ "futures", "hex", "itertools", - "near-client", - "near-crypto", + "near-client 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "near-indexer", + "near-indexer-primitives", + "near-lake-framework", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "near-sdk", "num-traits", "openssl-probe", @@ -1980,6 +2333,12 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" +[[package]] +name = "itoa" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" + [[package]] name = "jobserver" version = "0.1.24" @@ -1989,6 +2348,15 @@ dependencies = [ "libc", ] +[[package]] +name = "js-sys" +version = "0.3.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a38fc24e30fd564ce974c02bf1d337caddff65be6cc4735a1f7eab22a7440f04" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "keccak" version = "0.1.0" @@ -2038,9 +2406,9 @@ checksum = "06e509672465a0504304aa87f9f176f2b2b716ed8fb105ebe5c02dc6dce96a94" [[package]] name = "libloading" -version = "0.7.1" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0cf036d15402bea3c5d4de17b3fce76b3e4a56ebc1f577be0e7a72f7c607cf0" +checksum = "efbc0f03f9a775e9f6aed295c6a1ba2253c5757a9e03d55c6caa46a681abcddd" dependencies = [ "cfg-if 1.0.0", "winapi", @@ -2048,9 +2416,9 @@ dependencies = [ [[package]] name = "librocksdb-sys" -version = "6.17.3" +version = "6.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5da125e1c0f22c7cae785982115523a0738728498547f415c9054cb17c7e89f9" +checksum = "c309a9d2470844aceb9a4a098cf5286154d20596868b75a6b36357d2bb9ca25d" dependencies = [ "bindgen", "cc", @@ -2193,6 +2561,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + [[package]] name = "memchr" version = "2.4.1" @@ -2211,27 +2585,18 @@ dependencies = [ [[package]] name = "memmap2" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "723e3ebdcdc5c023db1df315364573789f8857c11b631a2fdfad7c00f5c046b4" -dependencies = [ - "libc", -] - -[[package]] -name = "memmap2" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe3179b85e1fd8b14447cbebadb75e45a1002f541b925f0bfec366d56a81c56d" +checksum = "057a3db23999c867821a7a59feb06a578fcb03685e983dff90daf9e7d24ac08f" dependencies = [ "libc", ] [[package]] name = "memoffset" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" dependencies = [ "autocfg 1.0.1", ] @@ -2248,6 +2613,12 @@ version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.4.4" @@ -2282,9 +2653,9 @@ dependencies = [ [[package]] name = "more-asserts" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0debeb9fcf88823ea64d64e4a815ab1643f33127d995978e099942ce38f25238" +checksum = "7843ec2de400bcbc6a6328c958dc38e5359da6e93e72e37bc5246bf1ae776389" [[package]] name = "native-tls" @@ -2313,10 +2684,19 @@ dependencies = [ "serde", ] +[[package]] +name = "near-account-id" +version = "0.0.0" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" +dependencies = [ + "borsh 0.9.1", + "serde", +] + [[package]] name = "near-cache" version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" dependencies = [ "lru", ] @@ -2333,13 +2713,41 @@ dependencies = [ "failure_derive", "itertools", "lru", - "near-chain-configs", - "near-chain-primitives", - "near-crypto", - "near-metrics", - "near-pool", - "near-primitives", - "near-store", + "near-chain-configs 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-chain-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-pool 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-store 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "num-rational", + "once_cell", + "rand 0.7.3", + "rayon", + "strum", + "thiserror", + "tracing", +] + +[[package]] +name = "near-chain" +version = "0.0.0" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" +dependencies = [ + "actix", + "borsh 0.9.1", + "chrono", + "failure", + "failure_derive", + "itertools", + "lru", + "near-chain-configs 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-chain-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-pool 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-store 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "num-rational", "once_cell", "rand 0.7.3", @@ -2357,8 +2765,26 @@ dependencies = [ "anyhow", "chrono", "derive_more", - "near-crypto", - "near-primitives", + "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "num-rational", + "serde", + "serde_json", + "sha2", + "smart-default", + "tracing", +] + +[[package]] +name = "near-chain-configs" +version = "0.0.0" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" +dependencies = [ + "anyhow", + "chrono", + "derive_more", + "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "num-rational", "serde", "serde_json", @@ -2376,8 +2802,22 @@ dependencies = [ "failure", "failure_derive", "log", - "near-crypto", - "near-primitives", + "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "thiserror", +] + +[[package]] +name = "near-chain-primitives" +version = "0.0.0" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" +dependencies = [ + "chrono", + "failure", + "failure_derive", + "log", + "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "thiserror", ] @@ -2392,14 +2832,37 @@ dependencies = [ "futures", "log", "lru", - "near-chain", - "near-chunks-primitives", - "near-crypto", - "near-network", - "near-network-primitives", - "near-pool", - "near-primitives", - "near-store", + "near-chain 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-chunks-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-network 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-network-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-pool 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-store 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "rand 0.7.3", + "reed-solomon-erasure", +] + +[[package]] +name = "near-chunks" +version = "0.0.0" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" +dependencies = [ + "actix", + "borsh 0.9.1", + "chrono", + "futures", + "log", + "lru", + "near-chain 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-chunks-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-network 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-network-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-pool 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-store 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "rand 0.7.3", "reed-solomon-erasure", ] @@ -2409,7 +2872,15 @@ name = "near-chunks-primitives" version = "0.0.0" source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" dependencies = [ - "near-chain-primitives", + "near-chain-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", +] + +[[package]] +name = "near-chunks-primitives" +version = "0.0.0" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" +dependencies = [ + "near-chain-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", ] [[package]] @@ -2425,20 +2896,57 @@ dependencies = [ "futures", "log", "lru", - "near-chain", - "near-chain-configs", - "near-chunks", - "near-client-primitives", - "near-crypto", - "near-metrics", - "near-network", - "near-network-primitives", - "near-performance-metrics", - "near-performance-metrics-macros", - "near-pool", - "near-primitives", - "near-store", - "near-telemetry", + "near-chain 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-chain-configs 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-chunks 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-client-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-network 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-network-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-performance-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-performance-metrics-macros 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-pool 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-store 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-telemetry 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "num-rational", + "once_cell", + "rand 0.7.3", + "reed-solomon-erasure", + "serde_json", + "strum", + "sysinfo", + "thiserror", +] + +[[package]] +name = "near-client" +version = "0.0.0" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" +dependencies = [ + "actix", + "actix-rt", + "ansi_term", + "borsh 0.9.1", + "chrono", + "futures", + "log", + "lru", + "near-chain 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-chain-configs 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-chunks 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-client-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-network 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-network-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-performance-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-performance-metrics-macros 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-pool 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-store 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-telemetry 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "num-rational", "once_cell", "rand 0.7.3", @@ -2456,12 +2964,29 @@ source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b80 dependencies = [ "actix", "chrono", - "near-chain-configs", - "near-chain-primitives", - "near-chunks-primitives", - "near-crypto", - "near-network-primitives", - "near-primitives", + "near-chain-configs 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-chain-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-chunks-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-network-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "strum", + "thiserror", +] + +[[package]] +name = "near-client-primitives" +version = "0.0.0" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" +dependencies = [ + "actix", + "chrono", + "near-chain-configs 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-chain-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-chunks-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-network-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "strum", "thiserror", ] @@ -2480,7 +3005,33 @@ dependencies = [ "derive_more", "ed25519-dalek", "libc", - "near-account-id", + "near-account-id 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "once_cell", + "parity-secp256k1", + "primitive-types", + "rand 0.7.3", + "rand_core 0.5.1", + "serde", + "serde_json", + "subtle", + "thiserror", +] + +[[package]] +name = "near-crypto" +version = "0.0.0" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" +dependencies = [ + "arrayref", + "blake2", + "borsh 0.9.1", + "bs58", + "c2-chacha", + "curve25519-dalek", + "derive_more", + "ed25519-dalek", + "libc", + "near-account-id 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "once_cell", "parity-secp256k1", "primitive-types", @@ -2495,16 +3046,16 @@ dependencies = [ [[package]] name = "near-epoch-manager" version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" dependencies = [ "borsh 0.9.1", "log", "lru", - "near-chain", - "near-chain-configs", - "near-crypto", - "near-primitives", - "near-store", + "near-chain 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-chain-configs 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-store 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "num-rational", "primitive-types", "rand 0.6.5", @@ -2516,16 +3067,17 @@ dependencies = [ [[package]] name = "near-indexer" version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" dependencies = [ "actix", "anyhow", "async-recursion", "futures", - "near-chain-configs", - "near-client", - "near-crypto", - "near-primitives", + "near-chain-configs 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-client 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-indexer-primitives", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "nearcore", "node-runtime", "rocksdb", @@ -2535,25 +3087,35 @@ dependencies = [ "tracing", ] +[[package]] +name = "near-indexer-primitives" +version = "0.0.0" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" +dependencies = [ + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "serde", + "serde_json", +] + [[package]] name = "near-jsonrpc" version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" dependencies = [ "actix", "actix-cors", "actix-web", "easy-ext", "futures", - "near-chain-configs", - "near-client", + "near-chain-configs 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-client 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "near-jsonrpc-client", "near-jsonrpc-primitives", - "near-metrics", - "near-network", - "near-network-primitives", - "near-primitives", - "near-rpc-error-macro 0.0.0", + "near-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-network 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-network-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-rpc-error-macro 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "once_cell", "prometheus", "serde", @@ -2565,13 +3127,13 @@ dependencies = [ [[package]] name = "near-jsonrpc-client" version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" dependencies = [ "actix-http", "awc", "futures", "near-jsonrpc-primitives", - "near-primitives", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "serde", "serde_json", "uuid", @@ -2580,17 +3142,17 @@ dependencies = [ [[package]] name = "near-jsonrpc-primitives" version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" dependencies = [ "actix", - "near-chain-configs", - "near-client-primitives", - "near-crypto", - "near-metrics", - "near-network-primitives", - "near-primitives", - "near-primitives-core 0.0.0", - "near-rpc-error-macro 0.0.0", + "near-chain-configs 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-client-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-network-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-primitives-core 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-rpc-error-macro 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "once_cell", "serde", "serde_json", @@ -2599,6 +3161,27 @@ dependencies = [ "uuid", ] +[[package]] +name = "near-lake-framework" +version = "0.1.0" +source = "git+https://github.com/near/near-lake-framework?rev=34f77424051bf032bc5f1d21565ef6e021937b71#34f77424051bf032bc5f1d21565ef6e021937b71" +dependencies = [ + "anyhow", + "aws-config", + "aws-endpoint", + "aws-sdk-s3", + "aws-smithy-http", + "futures", + "itertools", + "near-indexer-primitives", + "serde", + "serde_json", + "tokio", + "tokio-stream", + "tracing", + "tracing-subscriber", +] + [[package]] name = "near-metrics" version = "0.0.0" @@ -2609,6 +3192,16 @@ dependencies = [ "prometheus", ] +[[package]] +name = "near-metrics" +version = "0.0.0" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" +dependencies = [ + "lazy_static", + "log", + "prometheus", +] + [[package]] name = "near-network" version = "0.0.0" @@ -2622,14 +3215,44 @@ dependencies = [ "futures", "itertools", "lru", - "near-crypto", - "near-metrics", - "near-network-primitives", - "near-performance-metrics", - "near-performance-metrics-macros", - "near-primitives", - "near-rate-limiter", - "near-store", + "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-network-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-performance-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-performance-metrics-macros 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-rate-limiter 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-store 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "once_cell", + "rand 0.7.3", + "strum", + "tokio", + "tokio-stream", + "tokio-util", + "tracing", +] + +[[package]] +name = "near-network" +version = "0.0.0" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" +dependencies = [ + "actix", + "borsh 0.9.1", + "bytes", + "bytesize", + "conqueue", + "futures", + "itertools", + "lru", + "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-network-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-performance-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-performance-metrics-macros 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-rate-limiter 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-store 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "once_cell", "rand 0.7.3", "strum", @@ -2649,17 +3272,52 @@ dependencies = [ "anyhow", "borsh 0.9.1", "chrono", - "near-crypto", - "near-primitives", + "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "strum", + "tokio", + "tracing", +] + +[[package]] +name = "near-network-primitives" +version = "0.0.0" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" +dependencies = [ + "actix", + "actix_derive", + "anyhow", + "borsh 0.9.1", + "chrono", + "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "strum", + "tokio", + "tracing", +] + +[[package]] +name = "near-performance-metrics" +version = "0.0.0" +source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" +dependencies = [ + "actix", + "bitflags", + "bytes", + "bytesize", + "futures", + "libc", + "log", + "once_cell", "strum", "tokio", - "tracing", + "tokio-util", ] [[package]] name = "near-performance-metrics" version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" dependencies = [ "actix", "bitflags", @@ -2683,15 +3341,37 @@ dependencies = [ "syn", ] +[[package]] +name = "near-performance-metrics-macros" +version = "0.0.0" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "near-pool" version = "0.0.0" source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" dependencies = [ "borsh 0.9.1", - "near-crypto", - "near-metrics", - "near-primitives", + "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "once_cell", + "rand 0.7.3", +] + +[[package]] +name = "near-pool" +version = "0.0.0" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" +dependencies = [ + "borsh 0.9.1", + "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "once_cell", "rand 0.7.3", ] @@ -2708,10 +3388,35 @@ dependencies = [ "derive_more", "easy-ext", "hex", - "near-crypto", - "near-primitives-core 0.0.0", - "near-rpc-error-macro 0.0.0", - "near-vm-errors 0.0.0", + "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-primitives-core 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-rpc-error-macro 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-vm-errors 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "num-rational", + "primitive-types", + "rand 0.7.3", + "reed-solomon-erasure", + "serde", + "serde_json", + "smart-default", +] + +[[package]] +name = "near-primitives" +version = "0.0.0" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" +dependencies = [ + "borsh 0.9.1", + "byteorder", + "bytesize", + "chrono", + "derive_more", + "easy-ext", + "hex", + "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-primitives-core 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-rpc-error-macro 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-vm-errors 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "num-rational", "primitive-types", "rand 0.7.3", @@ -2730,7 +3435,22 @@ dependencies = [ "borsh 0.9.1", "bs58", "derive_more", - "near-account-id", + "near-account-id 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "num-rational", + "serde", + "sha2", +] + +[[package]] +name = "near-primitives-core" +version = "0.0.0" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" +dependencies = [ + "base64 0.11.0", + "borsh 0.9.1", + "bs58", + "derive_more", + "near-account-id 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "num-rational", "serde", "sha2", @@ -2768,10 +3488,24 @@ dependencies = [ "tracing", ] +[[package]] +name = "near-rate-limiter" +version = "0.0.0" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" +dependencies = [ + "actix", + "bytes", + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "near-rosetta-rpc" version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" dependencies = [ "actix", "actix-cors", @@ -2782,13 +3516,13 @@ dependencies = [ "futures", "hex", "lazy_static", - "near-account-id", - "near-chain-configs", - "near-client", - "near-client-primitives", - "near-crypto", - "near-network", - "near-primitives", + "near-account-id 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-chain-configs 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-client 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-client-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-network 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "paperclip", "serde", "serde_json", @@ -2807,6 +3541,16 @@ dependencies = [ "syn", ] +[[package]] +name = "near-rpc-error-core" +version = "0.0.0" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" +dependencies = [ + "quote", + "serde", + "syn", +] + [[package]] name = "near-rpc-error-core" version = "0.1.0" @@ -2825,7 +3569,17 @@ name = "near-rpc-error-macro" version = "0.0.0" source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" dependencies = [ - "near-rpc-error-core 0.0.0", + "near-rpc-error-core 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "serde", + "syn", +] + +[[package]] +name = "near-rpc-error-macro" +version = "0.0.0" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" +dependencies = [ + "near-rpc-error-core 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "serde", "syn", ] @@ -2885,7 +3639,7 @@ dependencies = [ [[package]] name = "near-stable-hasher" version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" [[package]] name = "near-store" @@ -2899,8 +3653,31 @@ dependencies = [ "elastic-array", "fs2", "lru", - "near-crypto", - "near-primitives", + "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "num_cpus", + "rand 0.7.3", + "rocksdb", + "serde_json", + "strum", + "thiserror", + "tracing", +] + +[[package]] +name = "near-store" +version = "0.0.0" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" +dependencies = [ + "borsh 0.9.1", + "byteorder", + "bytesize", + "derive_more", + "elastic-array", + "fs2", + "lru", + "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "num_cpus", "rand 0.7.3", "rocksdb", @@ -2924,8 +3701,25 @@ dependencies = [ "actix-web", "awc", "futures", - "near-performance-metrics", - "near-performance-metrics-macros", + "near-performance-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-performance-metrics-macros 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "openssl", + "serde", + "serde_json", + "tracing", +] + +[[package]] +name = "near-telemetry" +version = "0.0.0" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" +dependencies = [ + "actix", + "actix-web", + "awc", + "futures", + "near-performance-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-performance-metrics-macros 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "openssl", "serde", "serde_json", @@ -2938,8 +3732,19 @@ version = "0.0.0" source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" dependencies = [ "borsh 0.9.1", - "near-account-id", - "near-rpc-error-macro 0.0.0", + "near-account-id 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "near-rpc-error-macro 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "serde", +] + +[[package]] +name = "near-vm-errors" +version = "0.0.0" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" +dependencies = [ + "borsh 0.9.1", + "near-account-id 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-rpc-error-macro 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "serde", ] @@ -2958,17 +3763,17 @@ dependencies = [ [[package]] name = "near-vm-logic" version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" dependencies = [ "base64 0.13.0", "borsh 0.9.1", "bs58", "byteorder", - "near-account-id", - "near-crypto", - "near-primitives", - "near-primitives-core 0.0.0", - "near-vm-errors 0.0.0", + "near-account-id 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-primitives-core 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-vm-errors 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "ripemd160", "serde", "sha2", @@ -2996,16 +3801,16 @@ dependencies = [ [[package]] name = "near-vm-runner" version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" dependencies = [ "anyhow", "borsh 0.9.1", "loupe", "memoffset", "near-cache", - "near-primitives", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "near-stable-hasher", - "near-vm-errors 0.0.0", + "near-vm-errors 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "near-vm-logic 0.0.0", "once_cell", "parity-wasm 0.41.0", @@ -3028,8 +3833,8 @@ dependencies = [ [[package]] name = "nearcore" -version = "1.25.0-rc.1" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" +version = "0.0.0" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" dependencies = [ "actix", "actix-rt", @@ -3047,21 +3852,21 @@ dependencies = [ "hyper-tls", "indicatif", "lazy-static-include", - "near-chain", - "near-chain-configs", - "near-chunks", - "near-client", - "near-crypto", + "near-chain 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-chain-configs 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-chunks 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-client 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "near-epoch-manager", "near-jsonrpc", - "near-network", - "near-network-primitives", - "near-performance-metrics", - "near-pool", - "near-primitives", + "near-network 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-network-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-performance-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-pool 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "near-rosetta-rpc", - "near-store", - "near-telemetry", + "near-store 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-telemetry 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "near-vm-runner", "node-runtime", "num-rational", @@ -3093,18 +3898,18 @@ dependencies = [ [[package]] name = "node-runtime" version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" +source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" dependencies = [ "borsh 0.9.1", "byteorder", "hex", "log", - "near-chain-configs", - "near-crypto", - "near-metrics", - "near-primitives", - "near-store", - "near-vm-errors 0.0.0", + "near-chain-configs 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-store 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-vm-errors 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", "near-vm-logic 0.0.0", "near-vm-runner", "num-bigint 0.3.3", @@ -3121,11 +3926,12 @@ dependencies = [ [[package]] name = "nom" -version = "5.1.2" +version = "7.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb4262d26ed83a1c0a33a38fe2bb15797329c85770da05e6b828ddb782627af" +checksum = "1b1d11e1ef389c76fe5b81bcaf2ea32cf88b62bc494e19f493d0b30e7a930109" dependencies = [ "memchr", + "minimal-lexical", "version_check", ] @@ -3202,6 +4008,15 @@ dependencies = [ "libc", ] +[[package]] +name = "num_threads" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97ba99ba6393e2c3734791401b66902d981cb03bf190af674ca69949b6d5fb15" +dependencies = [ + "libc", +] + [[package]] name = "number_prefix" version = "0.3.0" @@ -3233,9 +4048,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.36" +version = "0.10.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d9facdb76fec0b73c406f125d44d86fdad818d66fef0531eec9233ca425ff4a" +checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95" dependencies = [ "bitflags", "cfg-if 1.0.0", @@ -3253,18 +4068,18 @@ checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" [[package]] name = "openssl-src" -version = "111.16.0+1.1.1l" +version = "111.17.0+1.1.1m" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ab2173f69416cf3ec12debb5823d244127d23a9b127d5a5189aa97c5fa2859f" +checksum = "05d6a336abd10814198f66e2a91ccd7336611f30334119ca8ce300536666fcf4" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.67" +version = "0.9.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69df2d8dfc6ce3aaf44b40dec6f487d5a886516cf6879c49e98e0710f310a058" +checksum = "7e46109c383602735fa0a2e48dd2b7c892b048e1bf69e5c3b1d804b7d9c203cb" dependencies = [ "autocfg 1.0.1", "cc", @@ -3305,7 +4120,7 @@ dependencies = [ "paperclip-core", "paperclip-macros", "parking_lot 0.11.2", - "semver", + "semver 0.9.0", "serde", "serde_derive", "serde_json", @@ -3515,9 +4330,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.20" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c9b1041b4387893b91ee6746cddfc28516aff326a3519fb2adf820932c5e6cb" +checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe" [[package]] name = "ppv-lite86" @@ -3626,15 +4441,15 @@ dependencies = [ [[package]] name = "protobuf" -version = "2.25.1" +version = "2.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23129d50f2c9355ced935fce8a08bd706ee2e7ce2b3b33bf61dace0e379ac63a" +checksum = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96" [[package]] name = "psm" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd136ff4382c4753fc061cb9e4712ab2af263376b95bbd5bd8cd50c020b78e69" +checksum = "6eca0fa5dd7c4c96e184cec588f0b1db1ee3165e678db21c09793105acb17e6f" dependencies = [ "cc", ] @@ -4017,6 +4832,21 @@ dependencies = [ "bytecheck", ] +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", +] + [[package]] name = "ripemd160" version = "0.9.1" @@ -4030,12 +4860,12 @@ dependencies = [ [[package]] name = "rkyv" -version = "0.7.29" +version = "0.7.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49a37de5dfc60bae2d94961dacd03c7b80e426b66a99fa1b17799570dbdd8f96" +checksum = "bf98e3e6c7ed44e474b454b1ebded3193ee5aba3428e29c55d59b1d65e49945e" dependencies = [ "bytecheck", - "hashbrown 0.11.2", + "hashbrown 0.12.0", "ptr_meta", "rend", "rkyv_derive", @@ -4044,9 +4874,9 @@ dependencies = [ [[package]] name = "rkyv_derive" -version = "0.7.29" +version = "0.7.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "719d447dd0e84b23cee6cb5b32d97e21efb112a3e3c636c8da36647b938475a1" +checksum = "cc9940ec6a7c62b1d1f476f607c6caf0d7fbf74e43f77bc022143b878fcd3266" dependencies = [ "proc-macro2", "quote", @@ -4087,7 +4917,16 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" dependencies = [ - "semver", + "semver 0.9.0", +] + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver 1.0.5", ] [[package]] @@ -4104,11 +4943,36 @@ dependencies = [ "winapi", ] +[[package]] +name = "rustls" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" +dependencies = [ + "base64 0.13.0", + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "rustls-native-certs" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092" +dependencies = [ + "openssl-probe", + "rustls", + "schannel", + "security-framework", +] + [[package]] name = "rustversion" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61b3909d758bb75c79f23d4736fac9433868679d3ad2ea7a61e3c25cfda9a088" +checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f" [[package]] name = "ryu" @@ -4141,6 +5005,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sct" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "seahash" version = "4.1.0" @@ -4179,6 +5053,12 @@ dependencies = [ "semver-parser", ] +[[package]] +name = "semver" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0486718e92ec9a68fbed73bb5ef687d71103b142595b406835649bebd33f72c7" + [[package]] name = "semver-parser" version = "0.7.0" @@ -4231,7 +5111,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f690853975602e1bfe1ccbf50504d67174e3bcf340f23b5ea9992e0587a52d8" dependencies = [ "indexmap", - "itoa", + "itoa 0.4.8", "ryu", "serde", ] @@ -4243,7 +5123,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" dependencies = [ "form_urlencoded", - "itoa", + "itoa 0.4.8", "ryu", "serde", ] @@ -4315,9 +5195,9 @@ dependencies = [ [[package]] name = "shlex" -version = "0.1.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" +checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" [[package]] name = "signal-hook-registry" @@ -4367,6 +5247,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -4395,7 +5281,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" dependencies = [ "discard", - "rustc_version", + "rustc_version 0.2.3", "stdweb-derive", "stdweb-internal-macros", "stdweb-internal-runtime", @@ -4521,19 +5407,19 @@ checksum = "ab0e7238dcc7b40a7be719a25365910f6807bd864f4cce6b2e6b873658e2b19d" [[package]] name = "target-lexicon" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9bffcddbc2458fa3e6058414599e3c838a022abae82e5c67b4f7f80298d5bff" +checksum = "d7fa7e55043acb85fca6b3c01485a2eeb6b69c5d21002e273c79e465f43b7ac1" [[package]] name = "tempfile" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" dependencies = [ "cfg-if 1.0.0", + "fastrand", "libc", - "rand 0.8.4", "redox_syscall 0.2.10", "remove_dir_all", "winapi", @@ -4630,6 +5516,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "time" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "004cbc98f30fa233c61a38bc77e96a9106e65c88f2d3bef182ae952027e5753d" +dependencies = [ + "libc", + "num_threads", +] + [[package]] name = "time-macros" version = "0.1.1" @@ -4711,21 +5607,32 @@ dependencies = [ [[package]] name = "tokio-openssl" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f24cddc8445a4dc8359cdd9e91c19d544fc95f672e32afe8945852b9381a09fe" +checksum = "c08f9ffb7809f1b20c1b398d92acf4cc719874b3b2b2d9ea2f09b4a80350878a" dependencies = [ - "futures", + "futures-util", "openssl", "openssl-sys", "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" +dependencies = [ + "rustls", + "tokio", + "webpki", +] + [[package]] name = "tokio-stream" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b2f3f698253f03119ac0102beaa64f67a67e08074d03a22d18784104543727f" +checksum = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3" dependencies = [ "futures-core", "pin-project-lite", @@ -4755,6 +5662,28 @@ dependencies = [ "serde", ] +[[package]] +name = "tower" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a89fd63ad6adf737582df5db40d286574513c69a11dac5214dc3b5603d6713e" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62" + [[package]] name = "tower-service" version = "0.3.1" @@ -4768,6 +5697,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "375a639232caf30edfc78e8d89b2d4c375515393e7af7e16f01cd96917fb2105" dependencies = [ "cfg-if 1.0.0", + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -4914,6 +5844,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "url" version = "2.2.2" @@ -4926,6 +5862,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a1f0175e03a0973cf4afd476bef05c26e228520400eb1fd473ad417b1c00ffb" + [[package]] name = "uuid" version = "0.8.2" @@ -4999,9 +5941,9 @@ checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" [[package]] name = "wasm-bindgen" -version = "0.2.78" +version = "0.2.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "632f73e236b219150ea279196e54e610f5dbafa5d61786303d4da54f84e47fce" +checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06" dependencies = [ "cfg-if 1.0.0", "wasm-bindgen-macro", @@ -5009,9 +5951,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.78" +version = "0.2.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a317bf8f9fba2476b4b2c85ef4c4af8ff39c3c7f0cdfeed4f82c34a880aa837b" +checksum = "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca" dependencies = [ "bumpalo", "lazy_static", @@ -5024,9 +5966,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.78" +version = "0.2.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56146e7c495528bf6587663bea13a8eb588d39b36b679d83972e1a2dbbdacf9" +checksum = "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5034,9 +5976,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.78" +version = "0.2.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7803e0eea25835f8abdc585cd3021b3deb11543c6fe226dcd30b228857c5c5ab" +checksum = "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc" dependencies = [ "proc-macro2", "quote", @@ -5047,9 +5989,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.78" +version = "0.2.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0237232789cf037d5480773fe568aac745bfe2afbc11a863e97901780a6b47cc" +checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2" [[package]] name = "wasmer-compiler-near" @@ -5063,7 +6005,7 @@ dependencies = [ "serde", "serde_bytes", "smallvec", - "target-lexicon 0.12.2", + "target-lexicon 0.12.3", "thiserror", "wasmer-types-near", "wasmer-vm-near", @@ -5100,12 +6042,12 @@ dependencies = [ "enumset", "lazy_static", "loupe", - "memmap2 0.5.2", + "memmap2", "more-asserts", "rustc-demangle", "serde", "serde_bytes", - "target-lexicon 0.12.2", + "target-lexicon 0.12.3", "thiserror", "wasmer-compiler-near", "wasmer-types-near", @@ -5133,9 +6075,9 @@ dependencies = [ [[package]] name = "wasmer-runtime-core-near" -version = "0.18.2" +version = "0.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cf26de331c8ef4257bef33649b4665535b105c927ab2e00715f17891362efe8" +checksum = "5a3fac37da3c625e98708c5dd92d3f642aaf700fd077168d3d0fff277ec6a165" dependencies = [ "bincode", "blake3", @@ -5150,7 +6092,7 @@ dependencies = [ "nix", "page_size", "parking_lot 0.10.2", - "rustc_version", + "rustc_version 0.2.3", "serde", "serde-bench", "serde_bytes", @@ -5250,9 +6192,9 @@ checksum = "98930446519f63d00a836efdc22f67766ceae8dbcc1571379f2bcabc6b2b9abc" [[package]] name = "wasmtime" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414be1bc5ca12e755ffd3ff7acc3a6d1979922f8237fc34068b2156cebcc3270" +checksum = "4c9c724da92e39a85d2231d4c2a942c8be295211441dbca581c6c3f3f45a9f00" dependencies = [ "anyhow", "backtrace", @@ -5269,7 +6211,7 @@ dependencies = [ "region 2.2.0", "rustc-demangle", "serde", - "target-lexicon 0.12.2", + "target-lexicon 0.12.3", "wasmparser 0.81.0", "wasmtime-cranelift", "wasmtime-environ", @@ -5280,9 +6222,9 @@ dependencies = [ [[package]] name = "wasmtime-cranelift" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4693d33725773615a4c9957e4aa731af57b27dca579702d1d8ed5750760f1a9" +checksum = "1762765dd69245f00e5d9783b695039e449a7be0f9c5383e4c78465dd6131aeb" dependencies = [ "anyhow", "cranelift-codegen", @@ -5294,7 +6236,7 @@ dependencies = [ "log", "more-asserts", "object", - "target-lexicon 0.12.2", + "target-lexicon 0.12.3", "thiserror", "wasmparser 0.81.0", "wasmtime-environ", @@ -5302,9 +6244,9 @@ dependencies = [ [[package]] name = "wasmtime-environ" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b17e47116a078b9770e6fb86cff8b9a660826623cebcfff251b047c8d8993ef" +checksum = "c4468301d95ec71710bb6261382efe27d1296447711645e3dbabaea6e4de3504" dependencies = [ "anyhow", "cranelift-entity", @@ -5314,7 +6256,7 @@ dependencies = [ "more-asserts", "object", "serde", - "target-lexicon 0.12.2", + "target-lexicon 0.12.3", "thiserror", "wasmparser 0.81.0", "wasmtime-types", @@ -5322,9 +6264,9 @@ dependencies = [ [[package]] name = "wasmtime-jit" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60ea5b380bdf92e32911400375aeefb900ac9d3f8e350bb6ba555a39315f2ee7" +checksum = "ab0ae6e581ff014b470ec35847ea3c0b4c3ace89a55df5a04c802a11f4574e7d" dependencies = [ "addr2line", "anyhow", @@ -5334,7 +6276,7 @@ dependencies = [ "object", "region 2.2.0", "serde", - "target-lexicon 0.12.2", + "target-lexicon 0.12.3", "thiserror", "wasmtime-environ", "wasmtime-runtime", @@ -5343,9 +6285,9 @@ dependencies = [ [[package]] name = "wasmtime-runtime" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abc7cd79937edd6e238b337608ebbcaf9c086a8457f01dfd598324f7fa56d81a" +checksum = "6d9c28877ae37a367cda7b52b8887589816152e95dde9b7c80cc686f52761961" dependencies = [ "anyhow", "backtrace", @@ -5368,9 +6310,9 @@ dependencies = [ [[package]] name = "wasmtime-types" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9e5e51a461a2cf2b69e1fc48f325b17d78a8582816e18479e8ead58844b23f8" +checksum = "395726e8f5dd8c57cb0db445627b842343f7e29ed7489467fdf7953ed9d3cd4f" dependencies = [ "cranelift-entity", "serde", @@ -5378,6 +6320,26 @@ dependencies = [ "wasmparser 0.81.0", ] +[[package]] +name = "web-sys" +version = "0.3.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c060b319f29dd25724f09a2ba1418f142f539b2be99fbf4d2d5a8f7330afb8eb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "wee_alloc" version = "0.4.5" @@ -5427,6 +6389,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214" +[[package]] +name = "xmlparser" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "114ba2b24d2167ef6d67d7d04c8cc86522b87f490025f39f0303b7db5bf5e3d8" + [[package]] name = "xz2" version = "0.1.6" diff --git a/Cargo.toml b/Cargo.toml index 54bb754d..32a1ea91 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "indexer-explorer" -version = "0.10.12" +version = "0.10.12-lake.1" authors = ["Near Inc "] edition = "2021" @@ -32,13 +32,16 @@ openssl-probe = { version = "0.1.2" } r2d2 = "0.8.8" serde = { version = "1", features = ["derive"] } serde_json = "1.0.55" -tokio = { version = "1.1", features = ["sync", "time"] } -tokio-stream = { version = "0.1" } +tokio = { version = "1.1", features = ["sync", "time", "macros", "rt-multi-thread"] } +tokio-stream = { version = "0.1.8" } tracing = "0.1.13" tracing-subscriber = "0.2.4" uint = { version = "0.8.3", default-features = false } actix-diesel = { git = "https://github.com/frol/actix-diesel", branch = "actix-0.11-beta.2" } -near-indexer = { git = "https://github.com/near/nearcore", rev = "539f254f793a3324a29e6e97e3b804b9fa4f27a2" } -near-crypto = { git = "https://github.com/near/nearcore", rev = "539f254f793a3324a29e6e97e3b804b9fa4f27a2" } +near-indexer = { git = "https://github.com/near/nearcore", rev = "5f09a3bf042b32d1ff26554433ad6449199ea02a" } +near-crypto = { git = "https://github.com/near/nearcore", rev = "5f09a3bf042b32d1ff26554433ad6449199ea02a" } +near-primitives = { git = "https://github.com/near/nearcore", rev = "5f09a3bf042b32d1ff26554433ad6449199ea02a" } +near-indexer-primitives = { git = "https://github.com/near/nearcore", rev = "5f09a3bf042b32d1ff26554433ad6449199ea02a" } near-client = { git = "https://github.com/near/nearcore", rev = "539f254f793a3324a29e6e97e3b804b9fa4f27a2" } +near-lake-framework = { git = "https://github.com/near/near-lake-framework", rev = "34f77424051bf032bc5f1d21565ef6e021937b71" } diff --git a/src/configs.rs b/src/configs.rs index ac664c7b..70ef675c 100644 --- a/src/configs.rs +++ b/src/configs.rs @@ -1,5 +1,3 @@ -use std::convert::TryFrom; - use clap::Parser; /// NEAR Indexer for Explorer @@ -14,120 +12,25 @@ use clap::Parser; setting(clap::AppSettings::NextLineHelp) )] pub(crate) struct Opts { - /// Sets a custom config dir. Defaults to ~/.near/ - #[clap(short, long)] - pub home_dir: Option, /// Enabled Indexer for Explorer debug level of logs #[clap(long)] pub debug: bool, - #[clap(subcommand)] - pub subcmd: SubCommand, -} - -#[derive(Parser, Debug)] -pub(crate) enum SubCommand { - /// Run NEAR Indexer Example. Start observe the network - Run(RunArgs), - /// Initialize necessary configs - Init(InitConfigArgs), -} - -#[derive(Parser, Debug, Clone)] -pub(crate) struct RunArgs { - /// Store initial data from genesis like Accounts, AccessKeys - #[clap(long)] - pub store_genesis: bool, - /// Force streaming while node is syncing - #[clap(long)] - pub stream_while_syncing: bool, /// Switches indexer to non-strict mode (skips Receipts without parent Transaction hash, stops storing AccountChanges and AccessKeys) #[clap(long)] pub non_strict_mode: bool, - /// Stops indexer completely after indexing the provided number of blocks - #[clap(long, short)] - pub stop_after_number_of_blocks: Option, /// Sets the concurrency for indexing. Note: concurrency (set to 2+) may lead to warnings due to tight constraints between transactions and receipts (those will get resolved eventually, but unless it is the second pass of indexing, concurrency won't help at the moment). #[clap(long, default_value = "1")] pub concurrency: std::num::NonZeroU16, - #[clap(subcommand)] - pub sync_mode: SyncModeSubCommand, -} - -#[allow(clippy::enum_variant_names)] // we want commands to be more explicit -#[derive(Parser, Debug, Clone)] -pub(crate) enum SyncModeSubCommand { - /// continue from the block Indexer was interrupted - SyncFromInterruption(InterruptionArgs), - /// start from the newest block after node finishes syncing - SyncFromLatest, - /// start from specified block height - SyncFromBlock(BlockArgs), -} - -#[derive(Parser, Debug, Clone)] -pub(crate) struct InterruptionArgs { - /// start indexing this number of blocks earlier than the actual interruption happened - #[clap(long, default_value = "0")] - pub delta: u64, -} - -#[derive(Parser, Debug, Clone)] -pub(crate) struct BlockArgs { - /// block height for block sync mode - #[clap(long)] - pub height: u64, -} - -impl TryFrom for near_indexer::SyncModeEnum { - type Error = &'static str; - - fn try_from(sync_mode: SyncModeSubCommand) -> Result { - match sync_mode { - SyncModeSubCommand::SyncFromInterruption(_) => Err("Unable to convert SyncFromInterruption variant because it has additional parameter which is not acceptable by near_indexer::SyncModeEnum::SyncFromInterruption"), - SyncModeSubCommand::SyncFromLatest => Ok(Self::LatestSynced), - SyncModeSubCommand::SyncFromBlock(args) => Ok(Self::BlockHeight(args.height)), - } - } -} - -#[derive(Parser, Debug)] -pub(crate) struct InitConfigArgs { - /// chain/network id (localnet, testnet, devnet, betanet) - #[clap(short, long)] - pub chain_id: Option, - /// Account ID for the validator key - #[clap(long)] - pub account_id: Option, - /// Specify private key generated from seed (TESTING ONLY) - #[clap(long)] - pub test_seed: Option, - /// Number of shards to initialize the chain with - #[clap(short, long, default_value = "1")] - pub num_shards: u64, - /// Makes block production fast (TESTING ONLY) - #[clap(short, long)] - pub fast: bool, - /// Genesis file to use when initialize testnet (including downloading) - #[clap(short, long)] - pub genesis: Option, - #[clap(short, long)] - /// Download the verified NEAR config file automatically. - #[clap(long)] - pub download_config: bool, - #[clap(long)] - pub download_config_url: Option, - /// Download the verified NEAR genesis file automatically. - #[clap(long)] - pub download_genesis: bool, - /// Specify a custom download URL for the genesis-file. + /// Store initial data from genesis like Accounts, AccessKeys #[clap(long)] - pub download_genesis_url: Option, - /// Customize max_gas_burnt_view runtime limit. If not specified, value - /// from genesis configuration will be taken. + pub store_genesis: bool, + /// AWS S3 bucket name to get the stream from #[clap(long)] - pub max_gas_burnt_view: Option, - /// Initialize boots nodes in @ format seperated by commas - /// to bootstrap the network and store them in config.json + pub s3_bucket_name: String, + /// AWS S3 bucket region #[clap(long)] - pub boot_nodes: Option, + pub s3_region_name: String, + /// Block height to start the stream from + #[clap(long, short)] + pub start_block_height: u64, } diff --git a/src/db_adapters/account_changes.rs b/src/db_adapters/account_changes.rs index 8e0018f5..a971195e 100644 --- a/src/db_adapters/account_changes.rs +++ b/src/db_adapters/account_changes.rs @@ -1,11 +1,25 @@ use actix_diesel::dsl::AsyncRunQueryDsl; use diesel::PgConnection; +use futures::future::try_join_all; use crate::models; use crate::schema; /// Saves state change related to account to database pub(crate) async fn store_account_changes( + pool: &actix_diesel::Database, + shards: &[near_indexer::IndexerShard], + block_hash: &near_indexer::near_primitives::hash::CryptoHash, + block_timestamp: u64, +) -> anyhow::Result<()> { + let futures = shards.iter().map(|shard| { + store_account_changes_for_chunk(pool, &shard.state_changes, block_hash, block_timestamp) + }); + + try_join_all(futures).await.map(|_| ()) +} + +async fn store_account_changes_for_chunk( pool: &actix_diesel::Database, state_changes: &[near_indexer::near_primitives::views::StateChangeWithCauseView], block_hash: &near_indexer::near_primitives::hash::CryptoHash, diff --git a/src/main.rs b/src/main.rs index 1f8c9901..380eef38 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,9 +12,9 @@ use tokio::sync::{mpsc, Mutex}; use tracing::{debug, info, warn}; use tracing_subscriber::EnvFilter; -use crate::configs::{Opts, SubCommand}; +use crate::configs::Opts; -mod aggregated; +// mod aggregated; mod configs; mod db_adapters; mod models; @@ -31,8 +31,8 @@ const MAX_DELAY_TIME: std::time::Duration = std::time::Duration::from_secs(120); #[derive(Clone, Hash, PartialEq, Eq, Debug)] pub enum ReceiptOrDataId { - ReceiptId(near_indexer::near_primitives::hash::CryptoHash), - DataId(near_indexer::near_primitives::hash::CryptoHash), + ReceiptId(near_lake_framework::near_indexer_primitives::CryptoHash), + DataId(near_lake_framework::near_indexer_primitives::CryptoHash), } // Creating type aliases to make HashMap types for cache more explicit pub type ParentTransactionHashString = String; @@ -131,7 +131,7 @@ async fn handle_message( // StateChange related to Account let account_changes_future = db_adapters::account_changes::store_account_changes( pool, - &streamer_message.state_changes, + &streamer_message.shards, &streamer_message.block.header.hash, streamer_message.block.header.timestamp, ); @@ -154,7 +154,7 @@ async fn handle_message( } async fn listen_blocks( - stream: mpsc::Receiver, + stream: mpsc::Receiver, pool: Database, concurrency: std::num::NonZeroU16, strict_mode: bool, @@ -214,65 +214,8 @@ async fn listen_blocks( tokio::time::sleep(std::time::Duration::from_secs(7)).await; } -/// Takes `home_dir` and `RunArgs` to build proper IndexerConfig and returns it -async fn construct_near_indexer_config( - pool: &Database, - home_dir: std::path::PathBuf, - args: configs::RunArgs, -) -> near_indexer::IndexerConfig { - // Extract await mode to avoid duplication - info!( - target: crate::INDEXER_FOR_EXPLORER, - "construct_near_indexer_config" - ); - let sync_mode: near_indexer::SyncModeEnum = match args.sync_mode { - configs::SyncModeSubCommand::SyncFromInterruption(interruption_args) - if interruption_args.delta == 1 => - { - info!(target: crate::INDEXER_FOR_EXPLORER, "got from interruption"); - // If delta is 0 we just return IndexerConfig with sync_mode FromInterruption - // without any changes - near_indexer::SyncModeEnum::FromInterruption - } - configs::SyncModeSubCommand::SyncFromInterruption(interruption_args) => { - info!(target: crate::INDEXER_FOR_EXPLORER, "got from interruption"); - info!( - target: crate::INDEXER_FOR_EXPLORER, - "delta is non zero, calculating..." - ); - - db_adapters::blocks::latest_block_height(pool) - .await - .ok() - .map(|latest_block_height| { - if let Some(height) = latest_block_height { - near_indexer::SyncModeEnum::BlockHeight( - height.saturating_sub(interruption_args.delta), - ) - } else { - near_indexer::SyncModeEnum::FromInterruption - } - }) - .unwrap_or_else(|| near_indexer::SyncModeEnum::FromInterruption) - } - configs::SyncModeSubCommand::SyncFromBlock(block_args) => { - near_indexer::SyncModeEnum::BlockHeight(block_args.height) - } - configs::SyncModeSubCommand::SyncFromLatest => near_indexer::SyncModeEnum::LatestSynced, - }; - - near_indexer::IndexerConfig { - home_dir, - sync_mode, - await_for_node_synced: if args.stream_while_syncing { - near_indexer::AwaitForNodeSyncedEnum::StreamWhileSyncing - } else { - near_indexer::AwaitForNodeSyncedEnum::WaitForFullSync - }, - } -} - -fn main() { +#[tokio::main] +async fn main() -> anyhow::Result<()> { // We use it to automatically search the for root certificates to perform HTTPS calls // (sending telemetry and downloading genesis) openssl_probe::init_ssl_cert_env_vars(); @@ -320,66 +263,21 @@ fn main() { .with_writer(std::io::stderr) .init(); - let home_dir = opts.home_dir.unwrap_or_else(near_indexer::get_default_home); - - match opts.subcmd { - SubCommand::Run(args) => { - tracing::info!( - target: crate::INDEXER_FOR_EXPLORER, - "NEAR Indexer for Explorer v{} starting...", - env!("CARGO_PKG_VERSION") - ); - - let system = actix::System::new(); - system.block_on(async move { - let indexer_config = - construct_near_indexer_config(&pool, home_dir, args.clone()).await; - let indexer = - near_indexer::Indexer::new(indexer_config).expect("Failed to initiate Indexer"); - if args.store_genesis { - let near_config = indexer.near_config().clone(); - db_adapters::genesis::store_genesis_records(pool.clone(), near_config.clone()) - .await - .expect("Failed to store the records from the genesis"); - } - - // Regular indexer process starts here - let stream = indexer.streamer(); - - // Spawning the computation of aggregated data - aggregated::spawn_aggregated_computations(pool.clone(), &indexer); - - listen_blocks( - stream, - pool, - args.concurrency, - !args.non_strict_mode, - args.stop_after_number_of_blocks, - ) - .await; + let config = near_lake_framework::LakeConfig { + s3_bucket_name: opts.s3_bucket_name.clone(), + s3_region_name: opts.s3_region_name.clone(), + start_block_height: opts.start_block_height, // want to start from the freshest + }; + let stream = near_lake_framework::streamer(config); + + listen_blocks( + stream, + pool.clone(), + opts.concurrency, + !opts.non_strict_mode, + None, + ) + .await; - actix::System::current().stop(); - }); - system.run().unwrap(); - } - SubCommand::Init(config) => near_indexer::init_configs( - &home_dir, - config.chain_id.as_ref().map(AsRef::as_ref), - config.account_id.map(|account_id_string| { - near_indexer::near_primitives::types::AccountId::try_from(account_id_string) - .expect("Received accound_id is not valid") - }), - config.test_seed.as_ref().map(AsRef::as_ref), - config.num_shards, - config.fast, - config.genesis.as_ref().map(AsRef::as_ref), - config.download_genesis, - config.download_genesis_url.as_ref().map(AsRef::as_ref), - config.download_config, - config.download_config_url.as_ref().map(AsRef::as_ref), - config.boot_nodes.as_ref().map(AsRef::as_ref), - config.max_gas_burnt_view, - ) - .expect("Failed to initiate config"), - } + Ok(()) } From 3f42ccc0cc8f06721b9d3e6fdce17cba273b9221 Mon Sep 17 00:00:00 2001 From: Bohdan Khorolets Date: Sun, 20 Feb 2022 17:41:50 +0200 Subject: [PATCH 02/42] start streamer with actix instead of tokio (diesel-actix) is used --- src/main.rs | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/main.rs b/src/main.rs index 380eef38..9b5fd36e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -214,8 +214,7 @@ async fn listen_blocks( tokio::time::sleep(std::time::Duration::from_secs(7)).await; } -#[tokio::main] -async fn main() -> anyhow::Result<()> { +fn main() { // We use it to automatically search the for root certificates to perform HTTPS calls // (sending telemetry and downloading genesis) openssl_probe::init_ssl_cert_env_vars(); @@ -227,7 +226,7 @@ async fn main() -> anyhow::Result<()> { let opts: Opts = Opts::parse(); let mut env_filter = EnvFilter::new( - "tokio_reactor=info,near=info,stats=info,telemetry=info,indexer=info,aggregated=info", + "tokio_reactor=info,near=info,stats=info,telemetry=info,indexer=info,aggregated=info,near_lake_framework=info", ); if opts.debug { @@ -268,16 +267,21 @@ async fn main() -> anyhow::Result<()> { s3_region_name: opts.s3_region_name.clone(), start_block_height: opts.start_block_height, // want to start from the freshest }; - let stream = near_lake_framework::streamer(config); - - listen_blocks( - stream, - pool.clone(), - opts.concurrency, - !opts.non_strict_mode, - None, - ) - .await; - - Ok(()) + let system = actix::System::new(); + system.block_on(async move { + let stream = near_lake_framework::streamer(config); + + listen_blocks( + stream, + pool.clone(), + opts.concurrency, + !opts.non_strict_mode, + None, + ) + .await; + + actix::System::current().stop(); + }); + + system.run().unwrap(); } From e113dd2205eca8baadf40bd52c003f2ce28e6e3d Mon Sep 17 00:00:00 2001 From: Bohdan Khorolets Date: Thu, 17 Mar 2022 20:29:38 +0200 Subject: [PATCH 03/42] fix find parent tx for receipt logic --- src/db_adapters/receipts.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db_adapters/receipts.rs b/src/db_adapters/receipts.rs index e22c6188..8bd80ced 100644 --- a/src/db_adapters/receipts.rs +++ b/src/db_adapters/receipts.rs @@ -261,7 +261,7 @@ async fn find_tx_hashes_for_receipts( .map( |(receipt_id_string, transaction_hash_string): (String, String)| { ( - crate::ReceiptOrDataId::ReceiptId( + crate::ReceiptOrDataId::DataId( near_primitives::hash::CryptoHash::from_str( &receipt_id_string, ) From 55f17c6f179885508b875b9682ec29c3e03de0e2 Mon Sep 17 00:00:00 2001 From: Vlad Frolov Date: Thu, 18 Aug 2022 15:09:32 +0200 Subject: [PATCH 04/42] refactor: upgraded to near-lake-framework 0.5.0 (#1) --- CONTRIBUTING.md | 4 +- Cargo.lock | 4598 +++-------------- Cargo.toml | 27 +- rust-toolchain | 2 +- src/aggregated/account_details.rs | 8 +- src/aggregated/circulating_supply/lockup.rs | 6 +- src/db_adapters/access_keys.rs | 24 +- src/db_adapters/account_changes.rs | 8 +- src/db_adapters/accounts.rs | 26 +- src/db_adapters/assets/events.rs | 6 +- .../assets/fungible_token_events.rs | 8 +- .../assets/non_fungible_token_events.rs | 8 +- src/db_adapters/blocks.rs | 4 +- src/db_adapters/chunks.rs | 4 +- src/db_adapters/execution_outcomes.rs | 6 +- src/db_adapters/genesis.rs | 4 +- src/db_adapters/mod.rs | 2 +- src/db_adapters/receipts.rs | 64 +- src/db_adapters/transactions.rs | 20 +- src/main.rs | 30 +- src/models/access_keys.rs | 14 +- src/models/account_changes.rs | 22 +- src/models/accounts.rs | 10 +- src/models/blocks.rs | 6 +- src/models/chunks.rs | 4 +- src/models/enums.rs | 60 +- src/models/execution_outcomes.rs | 2 +- src/models/receipts.rs | 22 +- src/models/serializers.rs | 22 +- src/models/transactions.rs | 8 +- 30 files changed, 901 insertions(+), 4128 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c39613eb..02a9d10f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -56,7 +56,7 @@ We use the following order to group our imports (use statements): 1. standard Rust library imports (e.g. `std::sync::Arc`) 2. external crates (e.g. `tokio::time`) -3. near-* crates (e.g. `near_indexer::near_primitives::types`) +3. near-* crates (e.g. `near_lake_framework::near_indexer_primitives::types`) 4. local crate modules (e.g. `crate::db`) 5. local modules (e.g. `self::access_keys`) 6. `mod` statements @@ -104,7 +104,7 @@ fn my_func() { over: ```rust -use near_indexer::near_primitives::types::Account; +use near_lake_framework::near_indexer_primitives::types::Account; fn my_func() { let my_account = Account::from("test.near"); diff --git a/Cargo.lock b/Cargo.lock index 03f7af6d..765e7f78 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,9 +10,9 @@ checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" [[package]] name = "actix" -version = "0.11.0-beta.2" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eb78f9871feb3519e06b947c2becbf2cc7f67ce786e510e6bd3f9a27da3dbf1" +checksum = "f728064aca1c318585bf4bb04ffcfac9e75e508ab4e8b1bd9ba5dfe04e2cbed5" dependencies = [ "actix-rt", "actix_derive", @@ -22,52 +22,22 @@ dependencies = [ "futures-core", "futures-sink", "futures-task", + "futures-util", "log", "once_cell", - "parking_lot 0.11.2", + "parking_lot", "pin-project-lite", "smallvec", "tokio", "tokio-util", ] -[[package]] -name = "actix-codec" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d5dbeb2d9e51344cb83ca7cc170f1217f9fe25bfc50160e6e200b5c31c1019a" -dependencies = [ - "bitflags", - "bytes", - "futures-core", - "futures-sink", - "log", - "pin-project-lite", - "tokio", - "tokio-util", -] - -[[package]] -name = "actix-cors" -version = "0.6.0-beta.2" -source = "git+https://github.com/near/actix-extras.git?branch=actix-web-4-beta.6#eb38fbccd2df7846f83f2d9ace3b4c2a593614f0" -dependencies = [ - "actix-service", - "actix-web", - "derive_more", - "futures-util", - "log", - "once_cell", - "tinyvec", -] - [[package]] name = "actix-diesel" version = "0.3.1-alpha.0" -source = "git+https://github.com/frol/actix-diesel?branch=actix-0.11-beta.2#7066a04ccce15b20940d926758837c99954765cc" +source = "git+https://github.com/frol/actix-diesel?rev=3a001986c89dfabfc3c448d8bae28525101b4992#3a001986c89dfabfc3c448d8bae28525101b4992" dependencies = [ "actix", - "actix-rt", "derive_more", "diesel", "futures", @@ -76,50 +46,6 @@ dependencies = [ "r2d2", ] -[[package]] -name = "actix-http" -version = "3.0.0-beta.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59d51c2ba06062e698a5d212d860e9fb2afc931c285ede687aaae896c8150347" -dependencies = [ - "actix-codec", - "actix-rt", - "actix-service", - "actix-tls", - "actix-utils", - "ahash 0.7.6", - "base64 0.13.0", - "bitflags", - "brotli2", - "bytes", - "bytestring", - "derive_more", - "encoding_rs", - "flate2", - "futures-core", - "futures-util", - "h2", - "http", - "httparse", - "itoa 0.4.8", - "language-tags", - "local-channel", - "log", - "mime", - "once_cell", - "paste", - "percent-encoding", - "pin-project", - "pin-project-lite", - "rand 0.8.4", - "regex", - "serde", - "sha-1", - "smallvec", - "time 0.2.27", - "tokio", -] - [[package]] name = "actix-macros" version = "0.2.3" @@ -130,163 +56,28 @@ dependencies = [ "syn", ] -[[package]] -name = "actix-router" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ad299af73649e1fc893e333ccf86f377751eb95ff875d095131574c6f43452c" -dependencies = [ - "bytestring", - "http", - "log", - "regex", - "serde", -] - [[package]] name = "actix-rt" -version = "2.2.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc7d7cd957c9ed92288a7c3c96af81fa5291f65247a76a34dac7b6af74e52ba0" +checksum = "7ea16c295198e958ef31930a6ef37d0fb64e9ca3b6116e6b93a8bdae96ee1000" dependencies = [ "actix-macros", "futures-core", "tokio", ] -[[package]] -name = "actix-server" -version = "2.0.0-beta.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7367665785765b066ad16e1086d26a087f696bc7c42b6f93004ced6cfcf1eeca" -dependencies = [ - "actix-rt", - "actix-service", - "actix-utils", - "futures-core", - "log", - "mio", - "num_cpus", - "tokio", -] - -[[package]] -name = "actix-service" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d3dc6a618b082974a08d7a4781d24d4691cba51500059bfebe6656a61ebfe1e" -dependencies = [ - "futures-core", - "paste", - "pin-project-lite", -] - -[[package]] -name = "actix-tls" -version = "3.0.0-beta.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65b7bb60840962ef0332f7ea01a57d73a24d2cb663708511ff800250bbfef569" -dependencies = [ - "actix-codec", - "actix-rt", - "actix-service", - "actix-utils", - "derive_more", - "futures-core", - "http", - "log", - "openssl", - "tokio-openssl", - "tokio-util", -] - -[[package]] -name = "actix-utils" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e491cbaac2e7fc788dfff99ff48ef317e23b3cf63dbaf7aaab6418f40f92aa94" -dependencies = [ - "local-waker", - "pin-project-lite", -] - -[[package]] -name = "actix-web" -version = "4.0.0-beta.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff12e933051557d700b0fcad20fe25b9ca38395cc87bbc5aeaddaef17b937a2f" -dependencies = [ - "actix-codec", - "actix-http", - "actix-macros", - "actix-router", - "actix-rt", - "actix-server", - "actix-service", - "actix-tls", - "actix-utils", - "actix-web-codegen", - "ahash 0.7.6", - "bytes", - "cookie", - "derive_more", - "either", - "encoding_rs", - "futures-core", - "futures-util", - "itoa 0.4.8", - "language-tags", - "log", - "mime", - "once_cell", - "pin-project", - "regex", - "serde", - "serde_json", - "serde_urlencoded", - "smallvec", - "socket2", - "time 0.2.27", - "url", -] - -[[package]] -name = "actix-web-codegen" -version = "0.5.0-beta.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f138ac357a674c3b480ddb7bbd894b13c1b6e8927d728bc9ea5e17eee2f8fc9" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "actix_derive" -version = "0.6.0-beta.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae749cf2582eb83efd288edd4e9704600fdce1bc4f69aa0c86ca1368a3e4c13f" +checksum = "6d44b8fee1ced9671ba043476deddef739dd0959bf77030b26b738cc591737a7" dependencies = [ "proc-macro2", "quote", "syn", ] -[[package]] -name = "addr2line" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" - [[package]] name = "ahash" version = "0.4.7" @@ -299,7 +90,7 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" dependencies = [ - "getrandom 0.2.3", + "getrandom 0.2.6", "once_cell", "version_check", ] @@ -324,9 +115,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.52" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84450d0b4a8bd1ba4144ce8ce718fbc5d071358b1e5384bace6536b3d1f2d5b3" +checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc" [[package]] name = "arrayref" @@ -342,9 +133,9 @@ checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" [[package]] name = "arrayvec" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4dc07131ffa69b8072d35f5007352af944213cde02545e2103680baed38fcd" +checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" [[package]] name = "async-mutex" @@ -355,22 +146,11 @@ dependencies = [ "event-listener", ] -[[package]] -name = "async-recursion" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7d78656ba01f1b93024b7c3a0467f1608e4be67d725749fdcd7d2c7678fd7a2" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "async-trait" -version = "0.1.51" +version = "0.1.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44318e776df68115a881de9a8fd1b9e53368d7a4a5ce4cc48517da3393233a5e" +checksum = "ed6aa3524a2dfcf9fe180c51eae2b58738348d819517ceadf95789c51fff7600" dependencies = [ "proc-macro2", "quote", @@ -390,47 +170,15 @@ dependencies = [ [[package]] name = "autocfg" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" - -[[package]] -name = "autocfg" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" - -[[package]] -name = "awc" -version = "3.0.0-beta.5" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8bea03ceedb5bc5d1696f0d800dfcb81cf3d1991eed1c84ae00306eb74b2d70" -dependencies = [ - "actix-codec", - "actix-http", - "actix-rt", - "actix-service", - "base64 0.13.0", - "bytes", - "cookie", - "derive_more", - "futures-core", - "itoa 0.4.8", - "log", - "mime", - "percent-encoding", - "pin-project-lite", - "rand 0.8.4", - "serde", - "serde_json", - "serde_urlencoded", -] +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "aws-config" -version = "0.6.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12b964849038df43a2b4e0c20e29b67451af5a93108d757dd58b9e82f41a0ee8" +checksum = "588c761aa9f0587106d77c6523b59b49532ebb6c048052fe878f9f8a3c830599" dependencies = [ "aws-http", "aws-sdk-sso", @@ -455,9 +203,9 @@ dependencies = [ [[package]] name = "aws-endpoint" -version = "0.6.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06d059b181b25940b751e8efecc173ceb4fe65f45d8975f56b02e98db5c42fd6" +checksum = "04619a4c0339675fdb7082a3c2a1799bf2b6398fbe0889d69a14f2bd130feb67" dependencies = [ "aws-smithy-http", "aws-types", @@ -468,9 +216,9 @@ dependencies = [ [[package]] name = "aws-http" -version = "0.6.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3049066e3282c98bbf01e90459a1772ccf6c0b96cd1483c3dd5aa34bef9b9de1" +checksum = "d023c3e17c572f3c071699763ad5035ea6e94f3ecf5712c301355beef0c80893" dependencies = [ "aws-smithy-http", "aws-smithy-types", @@ -483,9 +231,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "0.6.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d70be50ac07c3c2b5f37056271856ac00190e80c19c76c58bcbee5be0b63ec9" +checksum = "2e4421aa343a7eb324446d839c4744272190f396c787056f587f5258c1bc3f98" dependencies = [ "aws-endpoint", "aws-http", @@ -501,16 +249,16 @@ dependencies = [ "aws-types", "bytes", "http", - "md5", + "md-5", "tokio-stream", "tower", ] [[package]] name = "aws-sdk-sso" -version = "0.6.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222fcabbf95f1f13c4e28cab95c9a4bb02606f998b1ea2800713b6866be5701d" +checksum = "06d3a467637b986708c9299a4031ea120680b43056ee9013f79aa8e949467fa8" dependencies = [ "aws-endpoint", "aws-http", @@ -530,9 +278,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "0.6.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d85b9f081af2c73ee25642de1a35fa9ba7b2432e54f6bf42242e478ae53c3beb" +checksum = "9fa1ba5309db9d1c988c53a57eb80f6ccf9af4626b11068ba48e457fdab3aacf" dependencies = [ "aws-endpoint", "aws-http", @@ -552,24 +300,23 @@ dependencies = [ [[package]] name = "aws-sig-auth" -version = "0.6.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4012b5192350b5403aba19a01a5a3b1768158dab936c4269d89760970d4812bc" +checksum = "1c9d556c7432b15b180c4d7015dfe16050ce86fd82602efb7aa415cd7929b4ea" dependencies = [ "aws-sigv4", "aws-smithy-eventstream", "aws-smithy-http", "aws-types", "http", - "thiserror", "tracing", ] [[package]] name = "aws-sigv4" -version = "0.6.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f4b9c0c3a34e5152a0cd5e43b8f2cfd780e3bd7a245948d8787e051095ac4c" +checksum = "47f44f6f9aecc1ccbf3d410d6f00552046a4c808c320b51bdd72df0baa6cbbc1" dependencies = [ "aws-smithy-eventstream", "aws-smithy-http", @@ -581,15 +328,15 @@ dependencies = [ "percent-encoding", "regex", "ring", - "time 0.3.7", + "time 0.3.9", "tracing", ] [[package]] name = "aws-smithy-async" -version = "0.36.0" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b69dad0aefb1b64e63e0d3a1310dc50191608d8c9e226f2f241f344a7173642e" +checksum = "dd107a426047cff81691b598ce5b9488bf23d8b8fd803564c8aad4046521d069" dependencies = [ "futures-util", "pin-project-lite", @@ -599,9 +346,9 @@ dependencies = [ [[package]] name = "aws-smithy-client" -version = "0.36.0" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93e47a8aca2194672518d6630936507d3b54598c482f13ffe53f9b7932724bbb" +checksum = "14afb75139d67d3e076afbdc25110d409573e941e69ce1434d7d85107a2886f4" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -623,9 +370,9 @@ dependencies = [ [[package]] name = "aws-smithy-eventstream" -version = "0.36.0" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98bcfcb063d29c7cc7bb0a64830afe606090de75533c10a11a05460d814e8d9" +checksum = "33b2f9543079950a3864a62fc8559a89572daab429e5777b5a1aa6c06378e51c" dependencies = [ "aws-smithy-types", "bytes", @@ -634,9 +381,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.36.0" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c8bbe92ecdc4e39a612359b09994c45d000591d4951aa7343443f44b47e6696" +checksum = "1976aaf680a01d5ca920cc5834bb808576e803845107907b991d6a441747098b" dependencies = [ "aws-smithy-eventstream", "aws-smithy-types", @@ -646,6 +393,7 @@ dependencies = [ "http", "http-body", "hyper", + "once_cell", "percent-encoding", "pin-project", "tokio", @@ -655,9 +403,9 @@ dependencies = [ [[package]] name = "aws-smithy-http-tower" -version = "0.36.0" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f23fdf1253855af3bb4abb25e42ad3152a71241af89014eebf27c14c7a59b81d" +checksum = "ba6a7a52b67bab2277e677e656970ed38e06b9f74e42c37a54f7f61550d3264d" dependencies = [ "aws-smithy-http", "bytes", @@ -670,18 +418,18 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.36.0" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cc19c372b0a561aa6bfc5dfdd917da7c7b1641d3bc9049ca4d7b197bb616a09" +checksum = "168c5afb872e8d11f086bfa0157833293d2cac0ab66176690a887ad1367dd4a7" dependencies = [ "aws-smithy-types", ] [[package]] name = "aws-smithy-query" -version = "0.36.0" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8254e49a237e9dc0301a4683c424a825f4220420b241ec4eb51e959a70626d8a" +checksum = "306dc0963ee08df39ba50969c20926718a3efa5061bb15e54082c1f74335bf86" dependencies = [ "aws-smithy-types", "urlencoding", @@ -689,60 +437,41 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "0.36.0" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde96306a54777ec8781aa510830e242de614aa5746274713f5ecac0779f644f" +checksum = "72afd4e14f068f773d53123b28cabbfd44cfb1f69418f83fa16f2fca65959daf" dependencies = [ - "itoa 1.0.1", + "itoa", "num-integer", "ryu", - "time 0.3.7", + "time 0.3.9", ] [[package]] name = "aws-smithy-xml" -version = "0.36.0" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3b0466594a86074a6e96b11284f9a9ddc90c5c5b7d6144ab357a90be49d28c4" +checksum = "b606d35369febb651b1911039731625a23102cf775bde10295a2cf9a722de55a" dependencies = [ - "thiserror", "xmlparser", ] [[package]] name = "aws-types" -version = "0.6.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "433fd128ea727e9b83b34c72c6d4db1b900f067760fa27b387694fe896633142" +checksum = "23b500c108f8aa03ff2a401373abcc20e7752795ecd6fa6d4628e606d0d26a23" dependencies = [ "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", "aws-smithy-types", - "rustc_version 0.4.0", + "http", + "rustc_version", "tracing", "zeroize", ] -[[package]] -name = "backtrace" -version = "0.3.64" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e121dee8023ce33ab248d9ce1493df03c3b38a659b240096fcbd7048ff9c31f" -dependencies = [ - "addr2line", - "cc", - "cfg-if 1.0.0", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", -] - -[[package]] -name = "base-x" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4521f3e3d031370679b3b140beb36dfe4801b09ac77e30c61941f97df3ef28b" - [[package]] name = "base64" version = "0.11.0" @@ -766,34 +495,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "bincode" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] - -[[package]] -name = "bindgen" -version = "0.59.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bd2a9a458e8f4304c52c43ebb0cfbd520289f8379a52e329a38afda99bf8eb8" -dependencies = [ - "bitflags", - "cexpr", - "clang-sys", - "lazy_static", - "lazycell", - "peeking_take_while", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex", -] - [[package]] name = "bitflags" version = "1.3.2" @@ -824,28 +525,22 @@ dependencies = [ ] [[package]] -name = "blake3" -version = "0.3.8" +name = "block-buffer" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b64485778c4f16a6a5a9d335e80d449ac6c70cdd6a06d2af18a6f6f775a125b3" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" dependencies = [ - "arrayref", - "arrayvec 0.5.2", - "cc", - "cfg-if 0.1.10", - "constant_time_eq", - "crypto-mac", - "digest 0.9.0", + "block-padding", + "generic-array", ] [[package]] name = "block-buffer" -version = "0.9.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324" dependencies = [ - "block-padding", - "generic-array 0.14.4", + "generic-array", ] [[package]] @@ -875,12 +570,12 @@ dependencies = [ [[package]] name = "borsh" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18dda7dc709193c0d86a1a51050a926dc3df1cf262ec46a23a25dba421ea1924" +checksum = "15bf3650200d8bffa99015595e10f1fbd17de07abbc25bb067da79e769939bfa" dependencies = [ - "borsh-derive 0.9.1", - "hashbrown 0.9.1", + "borsh-derive 0.9.3", + "hashbrown 0.11.2", ] [[package]] @@ -910,12 +605,12 @@ dependencies = [ [[package]] name = "borsh-derive" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "684155372435f578c0fa1acd13ebbb182cc19d6b38b64ae7901da4393217d264" +checksum = "6441c552f230375d18e3cc377677914d2ca2b0d36e52129fe15450a2dce46775" dependencies = [ - "borsh-derive-internal 0.9.1", - "borsh-schema-derive-internal 0.9.1", + "borsh-derive-internal 0.9.3", + "borsh-schema-derive-internal 0.9.3", "proc-macro-crate 0.1.5", "proc-macro2", "syn", @@ -945,9 +640,9 @@ dependencies = [ [[package]] name = "borsh-derive-internal" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2102f62f8b6d3edeab871830782285b64cc1830168094db05c8e458f209bc5c3" +checksum = "5449c28a7b352f2d1e592a8a28bf139bc71afb0764a14f3c02500935d8c44065" dependencies = [ "proc-macro2", "quote", @@ -978,35 +673,15 @@ dependencies = [ [[package]] name = "borsh-schema-derive-internal" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196c978c4c9b0b142d446ef3240690bf5a8a33497074a113ff9a337ccb750483" +checksum = "cdbd5696d8bfa21d53d9fe39a714a18538bad11492a42d066dbbc395fb1951c0" dependencies = [ "proc-macro2", "quote", "syn", ] -[[package]] -name = "brotli-sys" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4445dea95f4c2b41cde57cc9fee236ae4dbae88d8fcbdb4750fc1bb5d86aaecd" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "brotli2" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cb036c3eade309815c15ddbacec5b22c4d1f3983a774ab2eac2e3e9ea85568e" -dependencies = [ - "brotli-sys", - "libc", -] - [[package]] name = "bs58" version = "0.4.0" @@ -1015,36 +690,15 @@ checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" [[package]] name = "bumpalo" -version = "3.8.0" +version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f1e260c3a9040a7c19a12468758f4c16f31a81a1fe087482be9570ec864bb6c" +checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" [[package]] name = "byte-slice-cast" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d30c751592b77c499e7bce34d99d67c2c11bdc0574e9a488ddade14150a4698" - -[[package]] -name = "bytecheck" -version = "0.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "314889ea31cda264cb7c3d6e6e5c9415a987ecb0e72c17c00d36fbb881d34abe" -dependencies = [ - "bytecheck_derive", - "ptr_meta", -] - -[[package]] -name = "bytecheck_derive" -version = "0.6.7" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a2b3b92c135dae665a6f760205b89187638e83bed17ef3e44e83c712cf30600" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] +checksum = "87c5fdd0166095e1d463fc6cc01aa8ce547ad77a4e84d42eb6762b084e28067e" [[package]] name = "byteorder" @@ -1060,9 +714,9 @@ checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" [[package]] name = "bytes-utils" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e314712951c43123e5920a446464929adc667a5eade7f8fb3997776c9df6e54" +checksum = "1934a3ef9cac8efde4966a92781e77713e1ba329f1d42e446c7d7eba340d8ef1" dependencies = [ "bytes", "either", @@ -1074,20 +728,11 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c58ec36aac5066d5ca17df51b3e70279f5670a72102f5752cb7e7c856adfc70" -[[package]] -name = "bytestring" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90706ba19e97b90786e19dc0d5e2abd80008d99d4c0c5d1ad0b5e72cec7c494d" -dependencies = [ - "bytes", -] - [[package]] name = "c2-chacha" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15f1a1e56adbcfb7a96c51ec42e37a22ee5cda66c0eae80f9b94ff68a71d4759" +checksum = "d27dae93fe7b1e0424dc57179ac396908c26b035a87234809f5c4dfd1b47dc80" dependencies = [ "cipher", "ppv-lite86", @@ -1115,7 +760,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4230b8d9f5db741004bfaef172c5b2dbf0eb94f105204cc6147a220080daaa85" dependencies = [ "cached_proc_macro_types", - "darling", + "darling 0.13.4", "quote", "syn", ] @@ -1128,21 +773,9 @@ checksum = "3a4f925191b4367301851c6d99b09890311d74b0d43f274c0b34c86d308a3663" [[package]] name = "cc" -version = "1.0.71" +version = "1.0.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79c2681d6594606957bbb8631c4b90a7fcaaa72cdb714743a437b156d6a7eedd" -dependencies = [ - "jobserver", -] - -[[package]] -name = "cexpr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" -dependencies = [ - "nom", -] +checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" [[package]] name = "cfg-if" @@ -1166,7 +799,7 @@ dependencies = [ "num-integer", "num-traits", "serde", - "time 0.1.43", + "time 0.1.44", "winapi", ] @@ -1176,45 +809,33 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12f8e7987cbd042a63249497f41aed09f8e65add917ea6566effbc56578d6801" dependencies = [ - "generic-array 0.14.4", -] - -[[package]] -name = "clang-sys" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cc00842eed744b858222c4c9faf7243aafc6d33f92f96935263ef4d8a41ce21" -dependencies = [ - "glob", - "libc", - "libloading", + "generic-array", ] [[package]] name = "clap" -version = "3.0.0-beta.5" +version = "3.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "feff3878564edb93745d58cf63e17b63f24142506e7a20c87a5521ed7bfb1d63" +checksum = "d2dbdf4bdacb33466e854ce889eee8dfd5729abf7ccd7664d0a2d60cd384440b" dependencies = [ "atty", "bitflags", "clap_derive", + "clap_lex", "indexmap", "lazy_static", - "os_str_bytes", "strsim", "termcolor", "textwrap", - "unicase", ] [[package]] name = "clap_derive" -version = "3.0.0-beta.5" +version = "3.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b15c6b4f786ffb6192ffe65a36855bc1fc2444bcd0945ae16748dcd6ed7d0d3" +checksum = "25320346e922cffe59c0bbc5410c8d8784509efb321488971081313cb1e1a33c" dependencies = [ - "heck", + "heck 0.4.0", "proc-macro-error", "proc-macro2", "quote", @@ -1222,69 +843,25 @@ dependencies = [ ] [[package]] -name = "cloudabi" -version = "0.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" -dependencies = [ - "bitflags", -] - -[[package]] -name = "conqueue" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac4306c796b95d3964b94fa65018a57daee08b45a54b86a4f64910426427b66" - -[[package]] -name = "console" -version = "0.15.0" +name = "clap_lex" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28b32d32ca44b70c3e4acd7db1babf555fa026e385fb95f18028f88848b3c31" +checksum = "a37c35f1112dad5e6e0b1adaff798507497a18fceeb30cceb3bae7d1427b9213" dependencies = [ - "encode_unicode", - "libc", - "once_cell", - "regex", - "terminal_size", - "unicode-width", - "winapi", + "os_str_bytes", ] -[[package]] -name = "const_fn" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f92cfa0fd5690b3cf8c1ef2cabbd9b7ef22fa53cf5e1f92b05103f6d5d1cf6e7" - -[[package]] -name = "constant_time_eq" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" - [[package]] name = "convert_case" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" -[[package]] -name = "cookie" -version = "0.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f1c7727e460397e56abc4bddc1d49e07a1ad78fc98eb2e1c8f032a58a2f80d" -dependencies = [ - "percent-encoding", - "time 0.2.27", - "version_check", -] - [[package]] name = "core-foundation" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6888e10551bb93e424d8df1d07f1a8b4fceb0001a3a4b048bfc47554946f47b3" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" dependencies = [ "core-foundation-sys", "libc", @@ -1296,248 +873,203 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" -[[package]] -name = "cpp_demangle" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeaa953eaad386a53111e47172c2fedba671e5684c8dd601a5f474f4f118710f" -dependencies = [ - "cfg-if 1.0.0", -] - [[package]] name = "cpufeatures" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469" +checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b" dependencies = [ "libc", ] [[package]] -name = "cranelift-bforest" -version = "0.80.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62fc68cdb867b7d27b5f33cd65eb11376dfb41a2d09568a1a2c2bc1dc204f4ef" -dependencies = [ - "cranelift-entity", -] - -[[package]] -name = "cranelift-codegen" -version = "0.80.1" +name = "crc32fast" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31253a44ab62588f8235a996cc9b0636d98a299190069ced9628b8547329b47a" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ - "cranelift-bforest", - "cranelift-codegen-meta", - "cranelift-codegen-shared", - "cranelift-entity", - "gimli", - "log", - "regalloc", - "smallvec", - "target-lexicon 0.12.3", + "cfg-if 1.0.0", ] [[package]] -name = "cranelift-codegen-meta" -version = "0.80.1" +name = "crossbeam-channel" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a20ab4627d30b702fb1b8a399882726d216b8164d3b3fa6189e3bf901506afe" +checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53" dependencies = [ - "cranelift-codegen-shared", + "cfg-if 1.0.0", + "crossbeam-utils", ] [[package]] -name = "cranelift-codegen-shared" -version = "0.80.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6687d9668dacfed4468361f7578d86bded8ca4db978f734d9b631494bebbb5b8" - -[[package]] -name = "cranelift-entity" -version = "0.80.1" +name = "crossbeam-utils" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c77c5d72db97ba2cb36f69037a709edbae0d29cb25503775891e7151c5c874bf" +checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" dependencies = [ - "serde", + "cfg-if 1.0.0", + "lazy_static", ] [[package]] -name = "cranelift-frontend" -version = "0.80.1" +name = "crunchy" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "426dca83f63c7c64ea459eb569aadc5e0c66536c0042ed5d693f91830e8750d0" -dependencies = [ - "cranelift-codegen", - "log", - "smallvec", - "target-lexicon 0.12.3", -] +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" [[package]] -name = "cranelift-native" -version = "0.80.1" +name = "crypto-common" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8007864b5d0c49b026c861a15761785a2871124e401630c03ef1426e6d0d559e" +checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8" dependencies = [ - "cranelift-codegen", - "libc", - "target-lexicon 0.12.3", + "generic-array", + "typenum", ] [[package]] -name = "cranelift-wasm" -version = "0.80.1" +name = "crypto-mac" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94cf12c071415ba261d897387ae5350c4d83c238376c8c5a96514ecfa2ea66a3" +checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" dependencies = [ - "cranelift-codegen", - "cranelift-entity", - "cranelift-frontend", - "itertools", - "log", - "smallvec", - "wasmparser 0.81.0", - "wasmtime-types", + "generic-array", + "subtle", ] [[package]] -name = "crc32fast" -version = "1.2.1" +name = "ct-logs" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8" dependencies = [ - "cfg-if 1.0.0", + "sct", ] [[package]] -name = "crossbeam-channel" -version = "0.5.1" +name = "curve25519-dalek" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4" +checksum = "0b9fdf9972b2bd6af2d913799d9ebc165ea4d2e65878e329d9c6b372c4491b61" dependencies = [ - "cfg-if 1.0.0", - "crossbeam-utils", + "byteorder", + "digest 0.9.0", + "rand_core 0.5.1", + "subtle", + "zeroize", ] [[package]] -name = "crossbeam-deque" -version = "0.8.1" +name = "darling" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" +checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" dependencies = [ - "cfg-if 1.0.0", - "crossbeam-epoch", - "crossbeam-utils", + "darling_core 0.13.4", + "darling_macro 0.13.4", ] [[package]] -name = "crossbeam-epoch" -version = "0.9.7" +name = "darling" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c00d6d2ea26e8b151d99093005cb442fb9a37aeaca582a03ec70946f49ab5ed9" +checksum = "4529658bdda7fd6769b8614be250cdcfc3aeb0ee72fe66f9e41e5e5eb73eac02" dependencies = [ - "cfg-if 1.0.0", - "crossbeam-utils", - "lazy_static", - "memoffset", - "scopeguard", + "darling_core 0.14.1", + "darling_macro 0.14.1", ] [[package]] -name = "crossbeam-utils" -version = "0.8.5" +name = "darling_core" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db" +checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" dependencies = [ - "cfg-if 1.0.0", - "lazy_static", + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", ] [[package]] -name = "crunchy" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" - -[[package]] -name = "crypto-mac" -version = "0.8.0" +name = "darling_core" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" +checksum = "649c91bc01e8b1eac09fb91e8dbc7d517684ca6be8ebc75bb9cafc894f9fdb6f" dependencies = [ - "generic-array 0.14.4", - "subtle", + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", ] [[package]] -name = "ct-logs" -version = "0.8.0" +name = "darling_macro" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8" +checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" dependencies = [ - "sct", + "darling_core 0.13.4", + "quote", + "syn", ] [[package]] -name = "curve25519-dalek" -version = "3.2.0" +name = "darling_macro" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9fdf9972b2bd6af2d913799d9ebc165ea4d2e65878e329d9c6b372c4491b61" +checksum = "ddfc69c5bfcbd2fc09a0f38451d2daf0e372e367986a83906d1b0dbc88134fb5" dependencies = [ - "byteorder", - "digest 0.9.0", - "rand_core 0.5.1", - "subtle", - "zeroize", + "darling_core 0.14.1", + "quote", + "syn", ] [[package]] -name = "darling" -version = "0.13.0" +name = "derive_builder" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "757c0ded2af11d8e739c4daea1ac623dd1624b06c844cf3f5a39f1bdbd99bb12" +checksum = "d07adf7be193b71cc36b193d0f5fe60b918a3a9db4dad0449f57bcfd519704a3" dependencies = [ - "darling_core", - "darling_macro", + "derive_builder_macro", ] [[package]] -name = "darling_core" -version = "0.13.0" +name = "derive_builder_core" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c34d8efb62d0c2d7f60ece80f75e5c63c1588ba68032740494b0b9a996466e3" +checksum = "1f91d4cfa921f1c05904dc3c57b4a32c38aed3340cce209f3a6fd1478babafc4" dependencies = [ - "fnv", - "ident_case", + "darling 0.14.1", "proc-macro2", "quote", - "strsim", "syn", ] [[package]] -name = "darling_macro" -version = "0.13.0" +name = "derive_builder_macro" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ade7bff147130fe5e6d39f089c6bd49ec0250f35d70b2eebf72afdfc919f15cc" +checksum = "8f0314b72bed045f3a68671b3c86328386762c93f82d98c65c3cb5e5f573dd68" dependencies = [ - "darling_core", - "quote", + "derive_builder_core", "syn", ] [[package]] name = "derive_more" -version = "0.99.14" +version = "0.99.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc7b9cef1e351660e5443924e4f43ab25fbbed3e9a5f052df3677deb4d6b320" +checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case", "proc-macro2", "quote", + "rustc_version", "syn", ] @@ -1564,7 +1096,7 @@ name = "diesel-derive-enum" version = "1.1.0" source = "git+https://github.com/khorolets/diesel-derive-enum.git?branch=lookup-hack#f15b35927291d1e3be725fb2321da6edb91704aa" dependencies = [ - "heck", + "heck 0.3.3", "proc-macro2", "quote", "syn", @@ -1581,88 +1113,31 @@ dependencies = [ "syn", ] -[[package]] -name = "digest" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" -dependencies = [ - "generic-array 0.12.4", -] - [[package]] name = "digest" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" dependencies = [ - "generic-array 0.14.4", -] - -[[package]] -name = "dirs" -version = "3.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30baa043103c9d0c2a57cf537cc2f35623889dc0d405e6c3cccfadbc81c71309" -dependencies = [ - "dirs-sys", + "generic-array", ] [[package]] -name = "dirs-sys" -version = "0.3.6" +name = "digest" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03d86534ed367a67548dc68113a0f5db55432fdfbb6e6f9d77704397d95d5780" +checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" dependencies = [ - "libc", - "redox_users", - "winapi", + "block-buffer 0.10.2", + "crypto-common", ] -[[package]] -name = "discard" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" - -[[package]] -name = "doc-comment" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" - [[package]] name = "dotenv" version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" -[[package]] -name = "dynasm" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b1801e630bd336d0bbbdbf814de6cc749c9a400c7e3d995e6adfd455d0c83c" -dependencies = [ - "bitflags", - "byteorder", - "lazy_static", - "proc-macro-error", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "dynasmrt" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d428afc93ad288f6dffc1fa5f4a78201ad2eec33c5a522e51c181009eb09061" -dependencies = [ - "byteorder", - "dynasm", - "memmap2", -] - [[package]] name = "easy-ext" version = "0.2.9" @@ -1671,9 +1146,9 @@ checksum = "53aff6fdc1b181225acdcb5b14c47106726fd8e486707315b1b138baed68ee31" [[package]] name = "ed25519" -version = "1.2.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4620d40f6d2601794401d6dd95a5cf69b6c157852539470eeda433a99b3c0efc" +checksum = "f9cb2103c580a9f8732121f755eccb51312f7db26314664314c119298107064b" dependencies = [ "signature", ] @@ -1688,7 +1163,7 @@ dependencies = [ "ed25519", "rand 0.7.3", "serde", - "sha2", + "sha2 0.9.9", "zeroize", ] @@ -1699,108 +1174,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" [[package]] -name = "elastic-array" -version = "0.11.0" +name = "event-listener" +version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d63720ea2bc2e1b79f7aa044d9dc0b825f9ccb6930b32120f8fb9e873aa84bc" -dependencies = [ - "heapsize", -] +checksum = "77f3309417938f28bf8228fcff79a4a37103981e3e186d2ccd19c74b38f4eb71" [[package]] -name = "encode_unicode" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" - -[[package]] -name = "encoding_rs" -version = "0.8.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a74ea89a0a1b98f6332de42c95baff457ada66d1cb4030f9ff151b2041a1c746" -dependencies = [ - "cfg-if 1.0.0", -] - -[[package]] -name = "enumset" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6216d2c19a6fb5f29d1ada1dc7bc4367a8cbf0fa4af5cf12e07b5bbdde6b5b2c" -dependencies = [ - "enumset_derive", -] - -[[package]] -name = "enumset_derive" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6451128aa6655d880755345d085494cf7561a6bee7c8dc821e5d77e6d267ecd4" -dependencies = [ - "darling", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "errno" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" -dependencies = [ - "errno-dragonfly", - "libc", - "winapi", -] - -[[package]] -name = "errno-dragonfly" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "event-listener" -version = "2.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7531096570974c3a9dcf9e4b8e1cede1ec26cf5046219fb3b9d897503b9be59" - -[[package]] -name = "failure" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" -dependencies = [ - "backtrace", - "failure_derive", -] - -[[package]] -name = "failure_derive" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - -[[package]] -name = "fallible-iterator" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" - -[[package]] -name = "fastrand" -version = "1.7.0" +name = "fastrand" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3fcf0cee53519c866c09b5de1f6c56ff9d647101f81c1964fa632e148896cdf" dependencies = [ @@ -1814,44 +1195,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfcf0ed7fe52a17a03854ec54a9f76d6d84508d1c0e66bc1793301c73fc8493c" dependencies = [ "byteorder", - "rand 0.8.4", + "rand 0.8.5", "rustc-hex", "static_assertions", ] -[[package]] -name = "flate2" -version = "1.0.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f" -dependencies = [ - "cfg-if 1.0.0", - "crc32fast", - "libc", - "miniz_oxide", -] - [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.0.1" @@ -1862,16 +1216,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fs2" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "funty" version = "1.1.0" @@ -1880,9 +1224,9 @@ checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" [[package]] name = "futures" -version = "0.3.17" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12aa0eb539080d55c3f2d45a67c3b58b6b0773c1a3ca2dfec66d58c97fd66ca" +checksum = "f73fe65f54d1e12b726f517d3e2135ca3125a437b6d998caf1962961f7172d9e" dependencies = [ "futures-channel", "futures-core", @@ -1895,9 +1239,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.17" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5da6ba8c3bb3c165d3c7319fc1cc8304facf1fb8db99c5de877183c08a273888" +checksum = "c3083ce4b914124575708913bca19bfe887522d6e2e6d0952943f5eac4a74010" dependencies = [ "futures-core", "futures-sink", @@ -1905,15 +1249,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.17" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d1c26957f23603395cd326b0ffe64124b818f4449552f960d815cfba83a53d" +checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3" [[package]] name = "futures-executor" -version = "0.3.17" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45025be030969d763025784f7f355043dc6bc74093e4ecc5000ca4dc50d8745c" +checksum = "9420b90cfa29e327d0429f19be13e7ddb68fa1cccb09d65e5706b8c7a749b8a6" dependencies = [ "futures-core", "futures-task", @@ -1922,18 +1266,16 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.17" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "522de2a0fe3e380f1bc577ba0474108faf3f6b18321dbf60b3b9c39a75073377" +checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b" [[package]] name = "futures-macro" -version = "0.3.17" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18e4a4b95cea4b4ccbcf1c5675ca7c4ee4e9e75eb79944d07defde18068f79bb" +checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512" dependencies = [ - "autocfg 1.0.1", - "proc-macro-hack", "proc-macro2", "quote", "syn", @@ -1941,23 +1283,22 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.17" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36ea153c13024fe480590b3e3d4cad89a0cfacecc24577b68f86c6ced9c2bc11" +checksum = "21163e139fa306126e6eedaf49ecdb4588f939600f0b1e770f4205ee4b7fa868" [[package]] name = "futures-task" -version = "0.3.17" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d3d00f4eddb73e498a54394f228cd55853bdf059259e8e7bc6e69d408892e99" +checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a" [[package]] name = "futures-util" -version = "0.3.17" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36568465210a3a6ee45e1f165136d68671471a501e632e9a98d96872222b5481" +checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a" dependencies = [ - "autocfg 1.0.1", "futures-channel", "futures-core", "futures-io", @@ -1967,25 +1308,14 @@ dependencies = [ "memchr", "pin-project-lite", "pin-utils", - "proc-macro-hack", - "proc-macro-nested", "slab", ] [[package]] name = "generic-array" -version = "0.12.4" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd" -dependencies = [ - "typenum", -] - -[[package]] -name = "generic-array" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +checksum = "fd48d33ec7f05fbfa152300fdad764757cbded343c1aa1cff2fbaf4134851803" dependencies = [ "typenum", "version_check", @@ -2004,37 +1334,20 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.3" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" +checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" dependencies = [ "cfg-if 1.0.0", "libc", - "wasi 0.10.2+wasi-snapshot-preview1", -] - -[[package]] -name = "gimli" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4" -dependencies = [ - "fallible-iterator", - "indexmap", - "stable_deref_trait", + "wasi 0.10.0+wasi-snapshot-preview1", ] -[[package]] -name = "glob" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" - [[package]] name = "h2" -version = "0.3.6" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c06815895acec637cd6ed6e9662c935b866d20a106f8361892893a7d9234964" +checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57" dependencies = [ "bytes", "fnv", @@ -2067,24 +1380,6 @@ dependencies = [ "ahash 0.7.6", ] -[[package]] -name = "hashbrown" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c21d40587b92fa6a6c6e3c1bdbf87d75511db5672f9c93175574b3a00df1758" -dependencies = [ - "ahash 0.7.6", -] - -[[package]] -name = "heapsize" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1679e6ea370dee694f91f1dc469bf94cf8f52051d147aec3e1f9497c6fc22461" -dependencies = [ - "winapi", -] - [[package]] name = "heck" version = "0.3.3" @@ -2094,6 +1389,12 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "heck" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" + [[package]] name = "hermit-abi" version = "0.1.19" @@ -2111,13 +1412,13 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "0.2.5" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1323096b05d41827dadeaee54c9981958c0f94e670bc94ed80037d1a7b8b186b" +checksum = "ff8670570af52249509a86f5e3e18a08c60b177071826898fde8997cf5f6bfbb" dependencies = [ "bytes", "fnv", - "itoa 0.4.8", + "itoa", ] [[package]] @@ -2133,21 +1434,21 @@ dependencies = [ [[package]] name = "httparse" -version = "1.5.1" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acd94fdbe1d4ff688b67b04eee2e17bd50995534a61539e45adfefb45e5e5503" +checksum = "496ce29bb5a52785b44e0f7ca2847ae0bb839c9bd28f69acac9b99d461c0c04c" [[package]] name = "httpdate" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6456b8a6c8f33fee7d958fcd1b60d55b11940a79e63ae87013e6d22e26034440" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" -version = "0.14.14" +version = "0.14.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b91bb1f221b6ea1f1e4371216b70f40748774c2fb5971b450c07773fb92d26b" +checksum = "b26ae0a80afebe130861d90abf98e3814a4f28a4c6ffeb5ab8ebb2be311e0ef2" dependencies = [ "bytes", "futures-channel", @@ -2158,7 +1459,7 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa 0.4.8", + "itoa", "pin-project-lite", "socket2", "tokio", @@ -2184,36 +1485,12 @@ dependencies = [ "webpki", ] -[[package]] -name = "hyper-tls" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" -dependencies = [ - "bytes", - "hyper", - "native-tls", - "tokio", - "tokio-native-tls", -] - [[package]] name = "ident_case" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" -[[package]] -name = "idna" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" -dependencies = [ - "matches", - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "impl-codec" version = "0.5.1" @@ -2225,9 +1502,9 @@ dependencies = [ [[package]] name = "impl-trait-for-tuples" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5dacb10c5b3bb92d46ba347505a9041e676bb20ad220101326bffb0c93031ee" +checksum = "11d7a9f6330b71fea57921c9b61c47ee6e84f72d394754eff6163ae67e7395eb" dependencies = [ "proc-macro2", "quote", @@ -2236,15 +1513,11 @@ dependencies = [ [[package]] name = "indexer-explorer" -version = "0.10.12-lake.1" +version = "0.10.19" dependencies = [ "actix", "actix-diesel", - "actix-http", "actix-rt", - "actix-tls", - "actix-web", - "actix_derive", "anyhow", "base64 0.11.0", "bigdecimal", @@ -2258,12 +1531,9 @@ dependencies = [ "futures", "hex", "itertools", - "near-client 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-indexer", - "near-indexer-primitives", + "near-crypto", "near-lake-framework", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-primitives", "near-sdk", "num-traits", "openssl-probe", @@ -2279,25 +1549,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb1fa934250de4de8aef298d81c729a7d33d8c239daa3a7575e6b92bfc7313b" -dependencies = [ - "autocfg 1.0.1", - "hashbrown 0.9.1", - "serde", -] - -[[package]] -name = "indicatif" -version = "0.15.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4" +checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" dependencies = [ - "console", - "lazy_static", - "number_prefix", - "regex", + "autocfg", + "hashbrown 0.11.2", ] [[package]] @@ -2309,15 +1566,6 @@ dependencies = [ "cfg-if 1.0.0", ] -[[package]] -name = "io-lifetimes" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ef6787e7f0faedc040f95716bdd0e62bcfcf4ba93da053b62dea2691c13864" -dependencies = [ - "winapi", -] - [[package]] name = "itertools" version = "0.10.3" @@ -2329,30 +1577,15 @@ dependencies = [ [[package]] name = "itoa" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" - -[[package]] -name = "itoa" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" - -[[package]] -name = "jobserver" -version = "0.1.24" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa" -dependencies = [ - "libc", -] +checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" [[package]] name = "js-sys" -version = "0.3.56" +version = "0.3.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a38fc24e30fd564ce974c02bf1d337caddff65be6cc4735a1f7eab22a7440f04" +checksum = "671a26f820db17c2a2750743f1dd03bafd15b98c9f30c7c2628c024c05d73397" dependencies = [ "wasm-bindgen", ] @@ -2363,1097 +1596,179 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67c21572b4949434e4fc1e1978b99c5f77064153c59d998bf13ecd96fb5ecba7" -[[package]] -name = "language-tags" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a" - -[[package]] -name = "lazy-static-include" -version = "3.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6002fe04202bdaf9e8d82929a7c9ebfcf47d027d87f671818e8cf9ccb4029908" -dependencies = [ - "lazy_static", - "manifest-dir-macros", - "syn", -] - [[package]] name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - -[[package]] -name = "leb128" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" - [[package]] name = "libc" -version = "0.2.118" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06e509672465a0504304aa87f9f176f2b2b716ed8fb105ebe5c02dc6dce96a94" +checksum = "5916d2ae698f6de9bfb891ad7a8d65c09d232dc58cc4ac433c7da3b2fd84bc2b" [[package]] -name = "libloading" -version = "0.7.3" +name = "lock_api" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efbc0f03f9a775e9f6aed295c6a1ba2253c5757a9e03d55c6caa46a681abcddd" +checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" dependencies = [ - "cfg-if 1.0.0", - "winapi", + "autocfg", + "scopeguard", ] [[package]] -name = "librocksdb-sys" -version = "6.20.3" +name = "log" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c309a9d2470844aceb9a4a098cf5286154d20596868b75a6b36357d2bb9ca25d" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ - "bindgen", - "cc", - "glob", - "libc", + "cfg-if 1.0.0", ] [[package]] -name = "linked-hash-map" -version = "0.5.4" +name = "matchers" +version = "0.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" +checksum = "f099785f7595cc4b4553a174ce30dd7589ef93391ff414dbb67f62392b9e0ce1" +dependencies = [ + "regex-automata", +] [[package]] -name = "linux-raw-sys" -version = "0.0.36" +name = "matches" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a261afc61b7a5e323933b402ca6a1765183687c614789b1e4db7762ed4230bca" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" [[package]] -name = "local-channel" -version = "0.1.2" +name = "md-5" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6246c68cf195087205a0512559c97e15eaf95198bf0e206d662092cdcb03fe9f" +checksum = "658646b21e0b72f7866c7038ab086d3d5e1cd6271f060fd37defb241949d0582" dependencies = [ - "futures-core", - "futures-sink", - "futures-util", - "local-waker", + "digest 0.10.3", ] [[package]] -name = "local-waker" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84f9a2d3e27ce99ce2c3aad0b09b1a7b916293ea9b2bf624c13fe646fadd8da4" - -[[package]] -name = "lock_api" -version = "0.3.4" +name = "memchr" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" -dependencies = [ - "scopeguard", -] +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] -name = "lock_api" -version = "0.4.5" +name = "memory_units" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712a4d093c9976e24e7dbca41db895dabcbac38eb5f4045393d17a95bdfb1109" -dependencies = [ - "scopeguard", - "serde", -] +checksum = "8452105ba047068f40ff7093dd1d9da90898e63dd61736462e9cdda6a90ad3c3" [[package]] -name = "log" -version = "0.4.14" +name = "mio" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +checksum = "713d550d9b44d89174e066b7a6217ae06234c10cb47819a88290d2b353c31799" dependencies = [ - "cfg-if 1.0.0", + "libc", + "log", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys", ] [[package]] -name = "loupe" -version = "0.1.3" +name = "near-account-id" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b6a72dfa44fe15b5e76b94307eeb2ff995a8c5b283b55008940c02e0c5b634d" +checksum = "71d258582a1878e6db67400b0504a5099db85718d22c2e07f747fe1706ae7150" dependencies = [ - "indexmap", - "loupe-derive", - "rustversion", + "borsh 0.9.3", + "serde", ] [[package]] -name = "loupe-derive" -version = "0.1.3" +name = "near-crypto" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0fbfc88337168279f2e9ae06e157cfed4efd3316e14dc96ed074d4f2e6c5952" +checksum = "1e75673d69fd7365508f3d32483669fe45b03bfb34e4d9363e90adae9dfb416c" dependencies = [ - "quote", - "syn", -] - -[[package]] -name = "lru" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "274353858935c992b13c0ca408752e2121da852d07dec7ce5f108c77dfa14d1f" -dependencies = [ - "hashbrown 0.11.2", -] - -[[package]] -name = "lzma-sys" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdb4b7c3eddad11d3af9e86c487607d2d2442d185d848575365c4856ba96d619" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - -[[package]] -name = "mach" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" -dependencies = [ - "libc", -] - -[[package]] -name = "manifest-dir-macros" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca7bbc41d799583acd24ed05a9c3db3c9275c93491b4e7cde0e609bb9598f2f0" -dependencies = [ - "once_cell", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "matchers" -version = "0.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f099785f7595cc4b4553a174ce30dd7589ef93391ff414dbb67f62392b9e0ce1" -dependencies = [ - "regex-automata", -] - -[[package]] -name = "matches" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" - -[[package]] -name = "md5" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" - -[[package]] -name = "memchr" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" - -[[package]] -name = "memmap" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "memmap2" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057a3db23999c867821a7a59feb06a578fcb03685e983dff90daf9e7d24ac08f" -dependencies = [ - "libc", -] - -[[package]] -name = "memoffset" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" -dependencies = [ - "autocfg 1.0.1", -] - -[[package]] -name = "memory_units" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8452105ba047068f40ff7093dd1d9da90898e63dd61736462e9cdda6a90ad3c3" - -[[package]] -name = "mime" -version = "0.3.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" - -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - -[[package]] -name = "miniz_oxide" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" -dependencies = [ - "adler", - "autocfg 1.0.1", -] - -[[package]] -name = "mio" -version = "0.7.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8067b404fe97c70829f082dec8bcf4f71225d7eaea1d8645349cb76fa06205cc" -dependencies = [ - "libc", - "log", - "miow", - "ntapi", - "winapi", -] - -[[package]] -name = "miow" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" -dependencies = [ - "winapi", -] - -[[package]] -name = "more-asserts" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7843ec2de400bcbc6a6328c958dc38e5359da6e93e72e37bc5246bf1ae776389" - -[[package]] -name = "native-tls" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d" -dependencies = [ - "lazy_static", - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - -[[package]] -name = "near-account-id" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" -dependencies = [ - "borsh 0.9.1", - "serde", -] - -[[package]] -name = "near-account-id" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" -dependencies = [ - "borsh 0.9.1", - "serde", -] - -[[package]] -name = "near-cache" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" -dependencies = [ - "lru", -] - -[[package]] -name = "near-chain" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" -dependencies = [ - "actix", - "borsh 0.9.1", - "chrono", - "failure", - "failure_derive", - "itertools", - "lru", - "near-chain-configs 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-chain-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-pool 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-store 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "num-rational", - "once_cell", - "rand 0.7.3", - "rayon", - "strum", - "thiserror", - "tracing", -] - -[[package]] -name = "near-chain" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" -dependencies = [ - "actix", - "borsh 0.9.1", - "chrono", - "failure", - "failure_derive", - "itertools", - "lru", - "near-chain-configs 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-chain-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-pool 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-store 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "num-rational", - "once_cell", - "rand 0.7.3", - "rayon", - "strum", - "thiserror", - "tracing", -] - -[[package]] -name = "near-chain-configs" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" -dependencies = [ - "anyhow", - "chrono", - "derive_more", - "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "num-rational", - "serde", - "serde_json", - "sha2", - "smart-default", - "tracing", -] - -[[package]] -name = "near-chain-configs" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" -dependencies = [ - "anyhow", - "chrono", - "derive_more", - "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "num-rational", - "serde", - "serde_json", - "sha2", - "smart-default", - "tracing", -] - -[[package]] -name = "near-chain-primitives" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" -dependencies = [ - "chrono", - "failure", - "failure_derive", - "log", - "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "thiserror", -] - -[[package]] -name = "near-chain-primitives" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" -dependencies = [ - "chrono", - "failure", - "failure_derive", - "log", - "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "thiserror", -] - -[[package]] -name = "near-chunks" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" -dependencies = [ - "actix", - "borsh 0.9.1", - "chrono", - "futures", - "log", - "lru", - "near-chain 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-chunks-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-network 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-network-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-pool 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-store 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "rand 0.7.3", - "reed-solomon-erasure", -] - -[[package]] -name = "near-chunks" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" -dependencies = [ - "actix", - "borsh 0.9.1", - "chrono", - "futures", - "log", - "lru", - "near-chain 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-chunks-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-network 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-network-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-pool 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-store 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "rand 0.7.3", - "reed-solomon-erasure", -] - -[[package]] -name = "near-chunks-primitives" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" -dependencies = [ - "near-chain-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", -] - -[[package]] -name = "near-chunks-primitives" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" -dependencies = [ - "near-chain-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", -] - -[[package]] -name = "near-client" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" -dependencies = [ - "actix", - "actix-rt", - "ansi_term", - "borsh 0.9.1", - "chrono", - "futures", - "log", - "lru", - "near-chain 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-chain-configs 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-chunks 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-client-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-network 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-network-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-performance-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-performance-metrics-macros 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-pool 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-store 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-telemetry 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "num-rational", - "once_cell", - "rand 0.7.3", - "reed-solomon-erasure", - "serde_json", - "strum", - "sysinfo", - "thiserror", -] - -[[package]] -name = "near-client" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" -dependencies = [ - "actix", - "actix-rt", - "ansi_term", - "borsh 0.9.1", - "chrono", - "futures", - "log", - "lru", - "near-chain 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-chain-configs 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-chunks 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-client-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-network 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-network-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-performance-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-performance-metrics-macros 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-pool 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-store 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-telemetry 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "num-rational", - "once_cell", - "rand 0.7.3", - "reed-solomon-erasure", - "serde_json", - "strum", - "sysinfo", - "thiserror", -] - -[[package]] -name = "near-client-primitives" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" -dependencies = [ - "actix", - "chrono", - "near-chain-configs 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-chain-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-chunks-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-network-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "strum", - "thiserror", -] - -[[package]] -name = "near-client-primitives" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" -dependencies = [ - "actix", - "chrono", - "near-chain-configs 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-chain-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-chunks-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-network-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "strum", - "thiserror", -] - -[[package]] -name = "near-crypto" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" -dependencies = [ - "arrayref", - "blake2", - "borsh 0.9.1", - "bs58", - "c2-chacha", - "curve25519-dalek", - "derive_more", - "ed25519-dalek", - "libc", - "near-account-id 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "once_cell", - "parity-secp256k1", - "primitive-types", - "rand 0.7.3", - "rand_core 0.5.1", - "serde", - "serde_json", - "subtle", - "thiserror", -] - -[[package]] -name = "near-crypto" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" -dependencies = [ - "arrayref", - "blake2", - "borsh 0.9.1", - "bs58", - "c2-chacha", - "curve25519-dalek", - "derive_more", - "ed25519-dalek", - "libc", - "near-account-id 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "once_cell", - "parity-secp256k1", - "primitive-types", - "rand 0.7.3", - "rand_core 0.5.1", - "serde", - "serde_json", - "subtle", - "thiserror", -] - -[[package]] -name = "near-epoch-manager" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" -dependencies = [ - "borsh 0.9.1", - "log", - "lru", - "near-chain 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-chain-configs 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-store 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "num-rational", - "primitive-types", - "rand 0.6.5", - "rand 0.7.3", - "serde_json", - "smart-default", -] - -[[package]] -name = "near-indexer" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" -dependencies = [ - "actix", - "anyhow", - "async-recursion", - "futures", - "near-chain-configs 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-client 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-indexer-primitives", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "nearcore", - "node-runtime", - "rocksdb", - "serde", - "serde_json", - "tokio", - "tracing", -] - -[[package]] -name = "near-indexer-primitives" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" -dependencies = [ - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "serde", - "serde_json", -] - -[[package]] -name = "near-jsonrpc" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" -dependencies = [ - "actix", - "actix-cors", - "actix-web", - "easy-ext", - "futures", - "near-chain-configs 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-client 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-jsonrpc-client", - "near-jsonrpc-primitives", - "near-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-network 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-network-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-rpc-error-macro 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "once_cell", - "prometheus", - "serde", - "serde_json", - "tokio", - "tracing", -] - -[[package]] -name = "near-jsonrpc-client" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" -dependencies = [ - "actix-http", - "awc", - "futures", - "near-jsonrpc-primitives", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "serde", - "serde_json", - "uuid", -] - -[[package]] -name = "near-jsonrpc-primitives" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" -dependencies = [ - "actix", - "near-chain-configs 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-client-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-network-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-primitives-core 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-rpc-error-macro 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "once_cell", - "serde", - "serde_json", - "thiserror", - "tracing", - "uuid", -] - -[[package]] -name = "near-lake-framework" -version = "0.1.0" -source = "git+https://github.com/near/near-lake-framework?rev=34f77424051bf032bc5f1d21565ef6e021937b71#34f77424051bf032bc5f1d21565ef6e021937b71" -dependencies = [ - "anyhow", - "aws-config", - "aws-endpoint", - "aws-sdk-s3", - "aws-smithy-http", - "futures", - "itertools", - "near-indexer-primitives", - "serde", - "serde_json", - "tokio", - "tokio-stream", - "tracing", - "tracing-subscriber", -] - -[[package]] -name = "near-metrics" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" -dependencies = [ - "lazy_static", - "log", - "prometheus", -] - -[[package]] -name = "near-metrics" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" -dependencies = [ - "lazy_static", - "log", - "prometheus", -] - -[[package]] -name = "near-network" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" -dependencies = [ - "actix", - "borsh 0.9.1", - "bytes", - "bytesize", - "conqueue", - "futures", - "itertools", - "lru", - "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-network-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-performance-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-performance-metrics-macros 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-rate-limiter 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-store 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "once_cell", - "rand 0.7.3", - "strum", - "tokio", - "tokio-stream", - "tokio-util", - "tracing", -] - -[[package]] -name = "near-network" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" -dependencies = [ - "actix", - "borsh 0.9.1", - "bytes", - "bytesize", - "conqueue", - "futures", - "itertools", - "lru", - "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-network-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-performance-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-performance-metrics-macros 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-rate-limiter 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-store 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "once_cell", - "rand 0.7.3", - "strum", - "tokio", - "tokio-stream", - "tokio-util", - "tracing", -] - -[[package]] -name = "near-network-primitives" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" -dependencies = [ - "actix", - "actix_derive", - "anyhow", - "borsh 0.9.1", - "chrono", - "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "strum", - "tokio", - "tracing", -] - -[[package]] -name = "near-network-primitives" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" -dependencies = [ - "actix", - "actix_derive", - "anyhow", - "borsh 0.9.1", - "chrono", - "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "strum", - "tokio", - "tracing", -] - -[[package]] -name = "near-performance-metrics" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" -dependencies = [ - "actix", - "bitflags", - "bytes", - "bytesize", - "futures", - "libc", - "log", - "once_cell", - "strum", - "tokio", - "tokio-util", -] - -[[package]] -name = "near-performance-metrics" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" -dependencies = [ - "actix", - "bitflags", - "bytes", - "bytesize", - "futures", - "libc", - "log", - "once_cell", - "strum", - "tokio", - "tokio-util", -] - -[[package]] -name = "near-performance-metrics-macros" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" -dependencies = [ - "quote", - "syn", -] - -[[package]] -name = "near-performance-metrics-macros" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" -dependencies = [ - "quote", - "syn", -] - -[[package]] -name = "near-pool" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" -dependencies = [ - "borsh 0.9.1", - "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "arrayref", + "blake2", + "borsh 0.9.3", + "bs58", + "c2-chacha", + "curve25519-dalek", + "derive_more", + "ed25519-dalek", + "near-account-id", "once_cell", + "parity-secp256k1", + "primitive-types", "rand 0.7.3", + "rand_core 0.5.1", + "serde", + "serde_json", + "subtle", + "thiserror", ] [[package]] -name = "near-pool" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" +name = "near-indexer-primitives" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1c089edfb295575737e94f8bad60d125f632919a742d6648644939b1b847665" dependencies = [ - "borsh 0.9.1", - "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "once_cell", - "rand 0.7.3", + "near-primitives", + "serde", + "serde_json", ] [[package]] -name = "near-primitives" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" +name = "near-lake-framework" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0316ee594f99e37604a0ec839e0130519e8e42110cfe4049d64d8f4b021c49e" dependencies = [ - "borsh 0.9.1", - "byteorder", - "bytesize", - "chrono", - "derive_more", - "easy-ext", - "hex", - "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-primitives-core 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-rpc-error-macro 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-vm-errors 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "num-rational", - "primitive-types", - "rand 0.7.3", - "reed-solomon-erasure", + "anyhow", + "aws-config", + "aws-sdk-s3", + "aws-types", + "derive_builder", + "futures", + "near-indexer-primitives", "serde", "serde_json", - "smart-default", + "tokio", + "tokio-stream", + "tracing", ] [[package]] name = "near-primitives" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ad1a9a1640539c81f065425c31bffcfbf6b31ef1aeaade59ce905f5df6ac860" dependencies = [ - "borsh 0.9.1", + "borsh 0.9.3", "byteorder", "bytesize", "chrono", "derive_more", "easy-ext", "hex", - "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-primitives-core 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-rpc-error-macro 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-vm-errors 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-crypto", + "near-primitives-core 0.14.0", + "near-rpc-error-macro 0.14.0", + "near-vm-errors 0.14.0", "num-rational", + "once_cell", "primitive-types", "rand 0.7.3", "reed-solomon-erasure", "serde", "serde_json", "smart-default", -] - -[[package]] -name = "near-primitives-core" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" -dependencies = [ - "base64 0.11.0", - "borsh 0.9.1", - "bs58", - "derive_more", - "near-account-id 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "num-rational", - "serde", - "sha2", -] - -[[package]] -name = "near-primitives-core" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" -dependencies = [ - "base64 0.11.0", - "borsh 0.9.1", - "bs58", - "derive_more", - "near-account-id 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "num-rational", - "serde", - "sha2", + "strum", + "thiserror", ] [[package]] @@ -3471,84 +1786,24 @@ dependencies = [ "num-rational", "serde", "serde_json", - "sha2", -] - -[[package]] -name = "near-rate-limiter" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" -dependencies = [ - "actix", - "bytes", - "futures-core", - "pin-project-lite", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "near-rate-limiter" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" -dependencies = [ - "actix", - "bytes", - "futures-core", - "pin-project-lite", - "tokio", - "tokio-util", - "tracing", + "sha2 0.9.9", ] [[package]] -name = "near-rosetta-rpc" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" +name = "near-primitives-core" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91d508f0fc340f6461e4e256417685720d3c4c00bb5a939b105160e49137caba" dependencies = [ - "actix", - "actix-cors", - "actix-http", - "actix-web", - "awc", + "base64 0.11.0", + "borsh 0.9.3", + "bs58", "derive_more", - "futures", - "hex", - "lazy_static", - "near-account-id 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-chain-configs 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-client 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-client-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-network 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "paperclip", + "near-account-id", + "num-rational", "serde", - "serde_json", + "sha2 0.10.2", "strum", - "tokio", - "validator", -] - -[[package]] -name = "near-rpc-error-core" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" -dependencies = [ - "quote", - "serde", - "syn", -] - -[[package]] -name = "near-rpc-error-core" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" -dependencies = [ - "quote", - "serde", - "syn", ] [[package]] @@ -3558,390 +1813,131 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffa8dbf8437a28ac40fcb85859ab0d0b8385013935b000c7a51ae79631dd74d9" dependencies = [ "proc-macro2", - "quote", - "serde", - "serde_json", - "syn", -] - -[[package]] -name = "near-rpc-error-macro" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" -dependencies = [ - "near-rpc-error-core 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "serde", - "syn", -] - -[[package]] -name = "near-rpc-error-macro" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" -dependencies = [ - "near-rpc-error-core 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "serde", - "syn", -] - -[[package]] -name = "near-rpc-error-macro" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6111d713e90c7c551dee937f4a06cb9ea2672243455a4454cc7566387ba2d9" -dependencies = [ - "near-rpc-error-core 0.1.0", - "proc-macro2", - "quote", - "serde", - "serde_json", - "syn", -] - -[[package]] -name = "near-runtime-utils" -version = "4.0.0-pre.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a48d80c4ca1d4cf99bc16490e1e3d49826c150dfc4410ac498918e45c7d98e07" -dependencies = [ - "lazy_static", - "regex", -] - -[[package]] -name = "near-sdk" -version = "4.0.0-pre.3" -source = "git+https://github.com/near/near-sdk-rs?rev=03487c184d37b0382dd9bd41c57466acad58fc1f#03487c184d37b0382dd9bd41c57466acad58fc1f" -dependencies = [ - "base64 0.13.0", - "borsh 0.8.2", - "bs58", - "near-primitives-core 0.4.0", - "near-sdk-macros", - "near-sys", - "near-vm-logic 4.0.0-pre.1", - "serde", - "serde_json", - "wee_alloc", -] - -[[package]] -name = "near-sdk-macros" -version = "4.0.0-pre.3" -source = "git+https://github.com/near/near-sdk-rs?rev=03487c184d37b0382dd9bd41c57466acad58fc1f#03487c184d37b0382dd9bd41c57466acad58fc1f" -dependencies = [ - "Inflector", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "near-stable-hasher" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" - -[[package]] -name = "near-store" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" -dependencies = [ - "borsh 0.9.1", - "byteorder", - "bytesize", - "derive_more", - "elastic-array", - "fs2", - "lru", - "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "num_cpus", - "rand 0.7.3", - "rocksdb", - "serde_json", - "strum", - "thiserror", - "tracing", -] - -[[package]] -name = "near-store" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" -dependencies = [ - "borsh 0.9.1", - "byteorder", - "bytesize", - "derive_more", - "elastic-array", - "fs2", - "lru", - "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "num_cpus", - "rand 0.7.3", - "rocksdb", - "serde_json", - "strum", - "thiserror", - "tracing", -] - -[[package]] -name = "near-sys" -version = "0.1.0" -source = "git+https://github.com/near/near-sdk-rs?rev=03487c184d37b0382dd9bd41c57466acad58fc1f#03487c184d37b0382dd9bd41c57466acad58fc1f" - -[[package]] -name = "near-telemetry" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" -dependencies = [ - "actix", - "actix-web", - "awc", - "futures", - "near-performance-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-performance-metrics-macros 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "openssl", - "serde", - "serde_json", - "tracing", -] - -[[package]] -name = "near-telemetry" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" -dependencies = [ - "actix", - "actix-web", - "awc", - "futures", - "near-performance-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-performance-metrics-macros 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "openssl", + "quote", "serde", "serde_json", - "tracing", + "syn", ] [[package]] -name = "near-vm-errors" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2#539f254f793a3324a29e6e97e3b804b9fa4f27a2" +name = "near-rpc-error-core" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93ee0b41c75ef859c193a8ff1dadfa0c8207bc0ac447cc22259721ad769a1408" dependencies = [ - "borsh 0.9.1", - "near-account-id 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", - "near-rpc-error-macro 0.0.0 (git+https://github.com/near/nearcore?rev=539f254f793a3324a29e6e97e3b804b9fa4f27a2)", + "quote", "serde", + "syn", ] [[package]] -name = "near-vm-errors" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" +name = "near-rpc-error-macro" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6111d713e90c7c551dee937f4a06cb9ea2672243455a4454cc7566387ba2d9" dependencies = [ - "borsh 0.9.1", - "near-account-id 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-rpc-error-macro 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", + "near-rpc-error-core 0.1.0", + "proc-macro2", + "quote", "serde", + "serde_json", + "syn", ] [[package]] -name = "near-vm-errors" -version = "4.0.0-pre.1" +name = "near-rpc-error-macro" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e281d8730ed8cb0e3e69fb689acee6b93cdb43824cd69a8ffd7e1bfcbd1177d7" +checksum = "8e837bd4bacd807073ec5ceb85708da7f721b46a4c2a978de86027fb0034ce31" dependencies = [ - "borsh 0.8.2", - "hex", - "near-rpc-error-macro 0.1.0", + "near-rpc-error-core 0.14.0", "serde", + "syn", ] [[package]] -name = "near-vm-logic" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" +name = "near-runtime-utils" +version = "4.0.0-pre.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a48d80c4ca1d4cf99bc16490e1e3d49826c150dfc4410ac498918e45c7d98e07" dependencies = [ - "base64 0.13.0", - "borsh 0.9.1", - "bs58", - "byteorder", - "near-account-id 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-primitives-core 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-vm-errors 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "ripemd160", - "serde", - "sha2", - "sha3", + "lazy_static", + "regex", ] [[package]] -name = "near-vm-logic" -version = "4.0.0-pre.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e11cb28a2d07f37680efdaf860f4c9802828c44fc50c08009e7884de75d982c5" +name = "near-sdk" +version = "4.0.0-pre.3" +source = "git+https://github.com/near/near-sdk-rs?rev=03487c184d37b0382dd9bd41c57466acad58fc1f#03487c184d37b0382dd9bd41c57466acad58fc1f" dependencies = [ "base64 0.13.0", "borsh 0.8.2", "bs58", - "byteorder", "near-primitives-core 0.4.0", - "near-runtime-utils", - "near-vm-errors 4.0.0-pre.1", + "near-sdk-macros", + "near-sys", + "near-vm-logic", "serde", - "sha2", - "sha3", + "serde_json", + "wee_alloc", ] [[package]] -name = "near-vm-runner" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" +name = "near-sdk-macros" +version = "4.0.0-pre.3" +source = "git+https://github.com/near/near-sdk-rs?rev=03487c184d37b0382dd9bd41c57466acad58fc1f#03487c184d37b0382dd9bd41c57466acad58fc1f" dependencies = [ - "anyhow", - "borsh 0.9.1", - "loupe", - "memoffset", - "near-cache", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-stable-hasher", - "near-vm-errors 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-vm-logic 0.0.0", - "once_cell", - "parity-wasm 0.41.0", - "pwasm-utils 0.12.0", - "pwasm-utils 0.18.2", - "serde", - "threadpool", - "tracing", - "wasmer-compiler-near", - "wasmer-compiler-singlepass-near", - "wasmer-engine-near", - "wasmer-engine-universal-near", - "wasmer-runtime-core-near", - "wasmer-runtime-near", - "wasmer-types-near", - "wasmer-vm-near", - "wasmparser 0.78.2", - "wasmtime", + "Inflector", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "nearcore" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" -dependencies = [ - "actix", - "actix-rt", - "actix-web", - "actix_derive", - "anyhow", - "awc", - "borsh 0.9.1", - "byteorder", - "chrono", - "dirs", - "easy-ext", - "futures", - "hyper", - "hyper-tls", - "indicatif", - "lazy-static-include", - "near-chain 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-chain-configs 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-chunks 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-client 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-epoch-manager", - "near-jsonrpc", - "near-network 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-network-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-performance-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-pool 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-rosetta-rpc", - "near-store 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-telemetry 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-vm-runner", - "node-runtime", - "num-rational", - "rand 0.7.3", - "rayon", - "serde", - "serde_json", - "smart-default", - "tempfile", - "thiserror", - "tokio", - "tracing", - "xz2", -] +name = "near-sys" +version = "0.1.0" +source = "git+https://github.com/near/near-sdk-rs?rev=03487c184d37b0382dd9bd41c57466acad58fc1f#03487c184d37b0382dd9bd41c57466acad58fc1f" [[package]] -name = "nix" -version = "0.15.0" +name = "near-vm-errors" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b2e0b4f3320ed72aaedb9a5ac838690a8047c7b275da22711fddff4f8a14229" -dependencies = [ - "bitflags", - "cc", - "cfg-if 0.1.10", - "libc", - "void", -] - -[[package]] -name = "node-runtime" -version = "0.0.0" -source = "git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a#5f09a3bf042b32d1ff26554433ad6449199ea02a" +checksum = "d0da466a30f0446639cbd788c30865086fac3e8dcb07a79e51d2b0775ed4261e" dependencies = [ - "borsh 0.9.1", - "byteorder", - "hex", - "log", - "near-chain-configs 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-crypto 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-metrics 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-primitives 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-store 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-vm-errors 0.0.0 (git+https://github.com/near/nearcore?rev=5f09a3bf042b32d1ff26554433ad6449199ea02a)", - "near-vm-logic 0.0.0", - "near-vm-runner", - "num-bigint 0.3.3", - "num-rational", - "num-traits", - "once_cell", - "rand 0.7.3", - "rayon", + "borsh 0.9.3", + "near-account-id", + "near-rpc-error-macro 0.14.0", "serde", - "serde_json", - "thiserror", - "tracing", ] [[package]] -name = "nom" -version = "7.1.0" +name = "near-vm-errors" +version = "4.0.0-pre.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1d11e1ef389c76fe5b81bcaf2ea32cf88b62bc494e19f493d0b30e7a930109" +checksum = "e281d8730ed8cb0e3e69fb689acee6b93cdb43824cd69a8ffd7e1bfcbd1177d7" dependencies = [ - "memchr", - "minimal-lexical", - "version_check", + "borsh 0.8.2", + "hex", + "near-rpc-error-macro 0.1.0", + "serde", ] [[package]] -name = "ntapi" -version = "0.3.6" +name = "near-vm-logic" +version = "4.0.0-pre.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" +checksum = "e11cb28a2d07f37680efdaf860f4c9802828c44fc50c08009e7884de75d982c5" dependencies = [ - "winapi", + "base64 0.13.0", + "borsh 0.8.2", + "bs58", + "byteorder", + "near-primitives-core 0.4.0", + "near-runtime-utils", + "near-vm-errors 4.0.0-pre.1", + "serde", + "sha2 0.9.9", + "sha3", ] [[package]] @@ -3950,7 +1946,7 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" dependencies = [ - "autocfg 1.0.1", + "autocfg", "num-integer", "num-traits", ] @@ -3961,18 +1957,18 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f6f7833f2cbf2360a6cfd58cd41a53aa7a90bd4c202f5b1c7dd2ed73c57b2c3" dependencies = [ - "autocfg 1.0.1", + "autocfg", "num-integer", "num-traits", ] [[package]] name = "num-integer" -version = "0.1.44" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ - "autocfg 1.0.1", + "autocfg", "num-traits", ] @@ -3982,7 +1978,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12ac428b1cb17fce6f731001d307d351ec70a6d202fc2e60f7d4c5e42d8f4f07" dependencies = [ - "autocfg 1.0.1", + "autocfg", "num-bigint 0.3.3", "num-integer", "num-traits", @@ -3991,18 +1987,18 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ - "autocfg 1.0.1", + "autocfg", ] [[package]] name = "num_cpus" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" dependencies = [ "hermit-abi", "libc", @@ -4010,35 +2006,18 @@ dependencies = [ [[package]] name = "num_threads" -version = "0.1.3" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97ba99ba6393e2c3734791401b66902d981cb03bf190af674ca69949b6d5fb15" +checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" dependencies = [ "libc", ] -[[package]] -name = "number_prefix" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" - -[[package]] -name = "object" -version = "0.27.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67ac1d3f9a1d3616fd9a60c8d74296f22406a238b6a72f5cc1e6f314df4ffbf9" -dependencies = [ - "crc32fast", - "indexmap", - "memchr", -] - [[package]] name = "once_cell" -version = "1.8.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" +checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225" [[package]] name = "opaque-debug" @@ -4046,138 +2025,17 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" -[[package]] -name = "openssl" -version = "0.10.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95" -dependencies = [ - "bitflags", - "cfg-if 1.0.0", - "foreign-types", - "libc", - "once_cell", - "openssl-sys", -] - [[package]] name = "openssl-probe" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" - -[[package]] -name = "openssl-src" -version = "111.17.0+1.1.1m" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05d6a336abd10814198f66e2a91ccd7336611f30334119ca8ce300536666fcf4" -dependencies = [ - "cc", -] - -[[package]] -name = "openssl-sys" -version = "0.9.72" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e46109c383602735fa0a2e48dd2b7c892b048e1bf69e5c3b1d804b7d9c203cb" -dependencies = [ - "autocfg 1.0.1", - "cc", - "libc", - "openssl-src", - "pkg-config", - "vcpkg", -] +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "os_str_bytes" -version = "4.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "addaa943333a514159c80c97ff4a93306530d965d27e139188283cd13e06a799" -dependencies = [ - "memchr", -] - -[[package]] -name = "page_size" -version = "0.4.2" +version = "6.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eebde548fbbf1ea81a99b128872779c437752fb99f217c45245e1a61dcd9edcd" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "paperclip" -version = "0.5.0" -source = "git+https://github.com/near/paperclip?branch=actix-web-4-beta.6#b9589c6e520eeee4a07ed94f7e32c476990d6536" -dependencies = [ - "anyhow", - "itertools", - "once_cell", - "paperclip-actix", - "paperclip-core", - "paperclip-macros", - "parking_lot 0.11.2", - "semver 0.9.0", - "serde", - "serde_derive", - "serde_json", - "serde_yaml", - "thiserror", - "url", -] - -[[package]] -name = "paperclip-actix" -version = "0.3.0" -source = "git+https://github.com/near/paperclip?branch=actix-web-4-beta.6#b9589c6e520eeee4a07ed94f7e32c476990d6536" -dependencies = [ - "actix-service", - "actix-web", - "futures", - "once_cell", - "paperclip-core", - "paperclip-macros", - "parking_lot 0.11.2", - "serde_json", -] - -[[package]] -name = "paperclip-core" -version = "0.3.0" -source = "git+https://github.com/near/paperclip?branch=actix-web-4-beta.6#b9589c6e520eeee4a07ed94f7e32c476990d6536" -dependencies = [ - "actix-web", - "mime", - "once_cell", - "paperclip-macros", - "parking_lot 0.11.2", - "pin-project", - "regex", - "serde", - "serde_json", - "serde_yaml", - "thiserror", -] - -[[package]] -name = "paperclip-macros" -version = "0.4.0" -source = "git+https://github.com/near/paperclip?branch=actix-web-4-beta.6#b9589c6e520eeee4a07ed94f7e32c476990d6536" -dependencies = [ - "heck", - "http", - "lazy_static", - "mime", - "proc-macro-error", - "proc-macro2", - "quote", - "strum", - "strum_macros", - "syn", -] +checksum = "029d8d0b2f198229de29dca79676f2738ff952edf3fde542eb8bf94d8c21b435" [[package]] name = "parity-scale-codec" @@ -4185,7 +2043,7 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "373b1a4c1338d9cd3d1fa53b3a11bdab5ab6bd80a20f7f7becd76953ae2be909" dependencies = [ - "arrayvec 0.7.1", + "arrayvec 0.7.2", "bitvec", "byte-slice-cast", "impl-trait-for-tuples", @@ -4199,7 +2057,7 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1557010476e0595c9b568d16dcfb81b93cdeb157612726f5170d31aa707bed27" dependencies = [ - "proc-macro-crate 1.1.0", + "proc-macro-crate 1.1.3", "proc-macro2", "quote", "syn", @@ -4217,79 +2075,29 @@ dependencies = [ "rand 0.7.3", ] -[[package]] -name = "parity-wasm" -version = "0.41.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddfc878dac00da22f8f61e7af3157988424567ab01d9920b962ef7dcbd7cd865" - -[[package]] -name = "parity-wasm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be5e13c266502aadf83426d87d81a0f5d1ef45b8027f5a471c360abfe4bfae92" - -[[package]] -name = "parking_lot" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e" -dependencies = [ - "lock_api 0.3.4", - "parking_lot_core 0.7.2", -] - [[package]] name = "parking_lot" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" -dependencies = [ - "instant", - "lock_api 0.4.5", - "parking_lot_core 0.8.5", -] - -[[package]] -name = "parking_lot_core" -version = "0.7.2" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3" -dependencies = [ - "cfg-if 0.1.10", - "cloudabi", - "libc", - "redox_syscall 0.1.57", - "smallvec", - "winapi", +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", ] [[package]] name = "parking_lot_core" -version = "0.8.5" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" +checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" dependencies = [ "cfg-if 1.0.0", - "instant", "libc", - "redox_syscall 0.2.10", + "redox_syscall", "smallvec", - "winapi", + "windows-sys", ] -[[package]] -name = "paste" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbf547ad0c65e31259204bd90935776d1c693cec2f4ff7abb7a1bbbd40dfe58" - -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - [[package]] name = "percent-encoding" version = "2.1.0" @@ -4298,18 +2106,18 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "pin-project" -version = "1.0.8" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "576bc800220cc65dac09e99e97b08b358cfab6e17078de8dc5fee223bd2d0c08" +checksum = "58ad3879ad3baf4e44784bc6a718a8698867bb991f8ce24d1bcbe2cfb4c3a75e" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.8" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e8fe8163d14ce7f0cdac2e040116f22eac817edabff0be91e8aff7e9accf389" +checksum = "744b6f092ba29c3650faf274db506afd39944f48420f6c86b17cfe0ee1cb36bb" dependencies = [ "proc-macro2", "quote", @@ -4318,9 +2126,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.7" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d31d11c69a6b52a174b42bdc0c30e5e11670f90788b2c471c31c1d17d449443" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" [[package]] name = "pin-utils" @@ -4328,17 +2136,11 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "pkg-config" -version = "0.3.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe" - [[package]] name = "ppv-lite86" -version = "0.2.14" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3ca011bd0129ff4ae15cd04c4eef202cadf6c51c21e47aba319b4e0501db741" +checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" [[package]] name = "pq-sys" @@ -4357,7 +2159,7 @@ checksum = "05e4722c697a58a99d5d06a08c30821d7c082a4632198de1eaa5a6c22ef42373" dependencies = [ "fixed-hash", "impl-codec", - "uint 0.9.1", + "uint 0.9.3", ] [[package]] @@ -4371,9 +2173,9 @@ dependencies = [ [[package]] name = "proc-macro-crate" -version = "1.1.0" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ebace6889caf889b4d3f76becee12e90353f2b8c7d875534a71e5742f8f6f83" +checksum = "e17d47ce914bf4de440332250b0edd23ce48c005f59fab39d3335866b114f11a" dependencies = [ "thiserror", "toml", @@ -4403,116 +2205,32 @@ dependencies = [ "version_check", ] -[[package]] -name = "proc-macro-hack" -version = "0.5.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" - -[[package]] -name = "proc-macro-nested" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" - [[package]] name = "proc-macro2" -version = "1.0.30" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edc3358ebc67bc8b7fa0c007f945b0b18226f78437d61bec735a9eb96b61ee70" +checksum = "9027b48e9d4c9175fa2218adf3557f91c1137021739951d4932f5f8268ac48aa" dependencies = [ "unicode-xid", ] -[[package]] -name = "prometheus" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8425533e7122f0c3cc7a37e6244b16ad3a2cc32ae7ac6276e2a75da0d9c200d" -dependencies = [ - "cfg-if 1.0.0", - "fnv", - "lazy_static", - "parking_lot 0.11.2", - "protobuf", - "regex", - "thiserror", -] - -[[package]] -name = "protobuf" -version = "2.27.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96" - -[[package]] -name = "psm" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eca0fa5dd7c4c96e184cec588f0b1db1ee3165e678db21c09793105acb17e6f" -dependencies = [ - "cc", -] - -[[package]] -name = "ptr_meta" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" -dependencies = [ - "ptr_meta_derive", -] - -[[package]] -name = "ptr_meta_derive" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "pwasm-utils" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f7a12f176deee919f4ba55326ee17491c8b707d0987aed822682c821b660192" -dependencies = [ - "byteorder", - "log", - "parity-wasm 0.41.0", -] - -[[package]] -name = "pwasm-utils" -version = "0.18.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "880b3384fb00b8f6ecccd5d358b93bd2201900ae3daad213791d1864f6441f5c" -dependencies = [ - "byteorder", - "log", - "parity-wasm 0.42.2", -] - [[package]] name = "quote" -version = "1.0.10" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38bc8cc6a5f2e3655e0899c1b848643b2562f853f114bfec7be120678e3ace05" +checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1" dependencies = [ "proc-macro2", ] [[package]] name = "r2d2" -version = "0.8.9" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f" +checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93" dependencies = [ "log", - "parking_lot 0.11.2", + "parking_lot", "scheduled-thread-pool", ] @@ -4522,24 +2240,6 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "643f8f41a8ebc4c5dc4515c82bb8abd397b527fc20fd681b7c011c2aee5d44fb" -[[package]] -name = "rand" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" -dependencies = [ - "autocfg 0.1.7", - "libc", - "rand_chacha 0.1.1", - "rand_core 0.4.2", - "rand_hc 0.1.0", - "rand_isaac", - "rand_jitter", - "rand_pcg", - "rand_xorshift", - "winapi", -] - [[package]] name = "rand" version = "0.7.3" @@ -4550,29 +2250,18 @@ dependencies = [ "libc", "rand_chacha 0.2.2", "rand_core 0.5.1", - "rand_hc 0.2.0", + "rand_hc", ] [[package]] name = "rand" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", "rand_chacha 0.3.1", "rand_core 0.6.3", - "rand_hc 0.3.1", -] - -[[package]] -name = "rand_chacha" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" -dependencies = [ - "autocfg 0.1.7", - "rand_core 0.3.1", ] [[package]] @@ -4595,21 +2284,6 @@ dependencies = [ "rand_core 0.6.3", ] -[[package]] -name = "rand_core" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" -dependencies = [ - "rand_core 0.4.2", -] - -[[package]] -name = "rand_core" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" - [[package]] name = "rand_core" version = "0.5.1" @@ -4625,16 +2299,7 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" dependencies = [ - "getrandom 0.2.3", -] - -[[package]] -name = "rand_hc" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" -dependencies = [ - "rand_core 0.3.1", + "getrandom 0.2.6", ] [[package]] @@ -4646,104 +2311,15 @@ dependencies = [ "rand_core 0.5.1", ] -[[package]] -name = "rand_hc" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7" -dependencies = [ - "rand_core 0.6.3", -] - -[[package]] -name = "rand_isaac" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" -dependencies = [ - "rand_core 0.3.1", -] - -[[package]] -name = "rand_jitter" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" -dependencies = [ - "libc", - "rand_core 0.4.2", - "winapi", -] - -[[package]] -name = "rand_pcg" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" -dependencies = [ - "autocfg 0.1.7", - "rand_core 0.4.2", -] - -[[package]] -name = "rand_xorshift" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" -dependencies = [ - "rand_core 0.3.1", -] - -[[package]] -name = "rayon" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" -dependencies = [ - "autocfg 1.0.1", - "crossbeam-deque", - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" -dependencies = [ - "crossbeam-channel", - "crossbeam-deque", - "crossbeam-utils", - "lazy_static", - "num_cpus", -] - -[[package]] -name = "redox_syscall" -version = "0.1.57" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" - [[package]] name = "redox_syscall" -version = "0.2.10" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" +checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" dependencies = [ "bitflags", ] -[[package]] -name = "redox_users" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" -dependencies = [ - "getrandom 0.2.3", - "redox_syscall 0.2.10", -] - [[package]] name = "reed-solomon-erasure" version = "4.0.2" @@ -4754,156 +2330,45 @@ dependencies = [ ] [[package]] -name = "regalloc" -version = "0.0.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d808cff91dfca7b239d40b972ba628add94892b1d9e19a842aedc5cfae8ab1a" -dependencies = [ - "log", - "rustc-hash", - "smallvec", -] - -[[package]] -name = "regex" -version = "1.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" -dependencies = [ - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.6.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" - -[[package]] -name = "region" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877e54ea2adcd70d80e9179344c97f93ef0dffd6b03e1f4529e6e83ab2fa9ae0" -dependencies = [ - "bitflags", - "libc", - "mach", - "winapi", -] - -[[package]] -name = "region" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76e189c2369884dce920945e2ddf79b3dff49e071a167dd1817fa9c4c00d512e" -dependencies = [ - "bitflags", - "libc", - "mach", - "winapi", -] - -[[package]] -name = "remove_dir_all" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" -dependencies = [ - "winapi", -] - -[[package]] -name = "rend" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79af64b4b6362ffba04eef3a4e10829718a4896dac19daa741851c86781edf95" -dependencies = [ - "bytecheck", -] - -[[package]] -name = "ring" -version = "0.16.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" -dependencies = [ - "cc", - "libc", - "once_cell", - "spin", - "untrusted", - "web-sys", - "winapi", -] - -[[package]] -name = "ripemd160" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eca4ecc81b7f313189bf73ce724400a07da2a6dac19588b03c8bd76a2dcc251" -dependencies = [ - "block-buffer", - "digest 0.9.0", - "opaque-debug", -] - -[[package]] -name = "rkyv" -version = "0.7.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf98e3e6c7ed44e474b454b1ebded3193ee5aba3428e29c55d59b1d65e49945e" -dependencies = [ - "bytecheck", - "hashbrown 0.12.0", - "ptr_meta", - "rend", - "rkyv_derive", - "seahash", -] - -[[package]] -name = "rkyv_derive" -version = "0.7.33" +name = "regex" +version = "1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc9940ec6a7c62b1d1f476f607c6caf0d7fbf74e43f77bc022143b878fcd3266" +checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286" dependencies = [ - "proc-macro2", - "quote", - "syn", + "aho-corasick", + "memchr", + "regex-syntax", ] [[package]] -name = "rocksdb" -version = "0.16.0" +name = "regex-automata" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c749134fda8bfc90d0de643d59bfc841dcb3ac8a1062e12b6754bd60235c48b3" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" dependencies = [ - "libc", - "librocksdb-sys", + "regex-syntax", ] [[package]] -name = "rustc-demangle" -version = "0.1.21" +name = "regex-syntax" +version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" [[package]] -name = "rustc-hash" -version = "1.1.0" +name = "ring" +version = "0.16.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", +] [[package]] name = "rustc-hex" @@ -4911,36 +2376,13 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" -[[package]] -name = "rustc_version" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -dependencies = [ - "semver 0.9.0", -] - [[package]] name = "rustc_version" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.5", -] - -[[package]] -name = "rustix" -version = "0.31.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2dcfc2778a90e38f56a708bfc90572422e11d6c7ee233d053d1f782cf9df6d2" -dependencies = [ - "bitflags", - "errno", - "io-lifetimes", - "libc", - "linux-raw-sys", - "winapi", + "semver", ] [[package]] @@ -4976,9 +2418,9 @@ checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f" [[package]] name = "ryu" -version = "1.0.5" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" [[package]] name = "schannel" @@ -4992,11 +2434,11 @@ dependencies = [ [[package]] name = "scheduled-thread-pool" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7" +checksum = "977a7519bff143a44f842fd07e80ad1329295bd71686457f18e496736f4bf9bf" dependencies = [ - "parking_lot 0.11.2", + "parking_lot", ] [[package]] @@ -5015,17 +2457,11 @@ dependencies = [ "untrusted", ] -[[package]] -name = "seahash" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" - [[package]] name = "security-framework" -version = "2.4.2" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "525bc1abfda2e1998d152c45cf13e696f76d0a4972310b22fac1658b05df7c87" +checksum = "2dc14f172faf8a0194a3aded622712b0de276821addc574fa54fc0a1167e10dc" dependencies = [ "bitflags", "core-foundation", @@ -5036,9 +2472,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.4.2" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9dd14d83160b528b7bfd66439110573efcfbe281b17fc2ca9f39f550d619c7e" +checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" dependencies = [ "core-foundation-sys", "libc", @@ -5046,58 +2482,24 @@ dependencies = [ [[package]] name = "semver" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -dependencies = [ - "semver-parser", -] - -[[package]] -name = "semver" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0486718e92ec9a68fbed73bb5ef687d71103b142595b406835649bebd33f72c7" - -[[package]] -name = "semver-parser" -version = "0.7.0" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd" [[package]] name = "serde" -version = "1.0.130" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f12d06de37cf59146fbdecab66aa99f9fe4f78722e3607577a5375d66bd0c913" +checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1" dependencies = [ "serde_derive", ] -[[package]] -name = "serde-bench" -version = "0.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d733da87e79faaac25616e33d26299a41143fd4cd42746cbb0e91d8feea243fd" -dependencies = [ - "byteorder", - "serde", -] - -[[package]] -name = "serde_bytes" -version = "0.11.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16ae07dd2f88a366f15bd0632ba725227018c69a1c8550a927324f8eb8368bb9" -dependencies = [ - "serde", -] - [[package]] name = "serde_derive" -version = "1.0.130" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7bc1a1ab1961464eae040d96713baa5a724a8152c1222492465b54322ec508b" +checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be" dependencies = [ "proc-macro2", "quote", @@ -5106,70 +2508,38 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.68" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f690853975602e1bfe1ccbf50504d67174e3bcf340f23b5ea9992e0587a52d8" -dependencies = [ - "indexmap", - "itoa 0.4.8", - "ryu", - "serde", -] - -[[package]] -name = "serde_urlencoded" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" -dependencies = [ - "form_urlencoded", - "itoa 0.4.8", - "ryu", - "serde", -] - -[[package]] -name = "serde_yaml" -version = "0.8.23" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a521f2940385c165a24ee286aa8599633d162077a54bdcae2a6fd5a7bfa7a0" +checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" dependencies = [ "indexmap", + "itoa", "ryu", "serde", - "yaml-rust", ] [[package]] -name = "sha-1" -version = "0.9.8" +name = "sha2" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99cd6713db3cf16b6c84e06321e049a9b9f699826e16096d23bbcc44d15d51a6" +checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ - "block-buffer", + "block-buffer 0.9.0", "cfg-if 1.0.0", "cpufeatures", "digest 0.9.0", "opaque-debug", ] -[[package]] -name = "sha1" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" - [[package]] name = "sha2" -version = "0.9.8" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b69f9a4c9740d74c5baa3fd2e547f9525fa8088a8a958e0ca2409a514e33f5fa" +checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676" dependencies = [ - "block-buffer", "cfg-if 1.0.0", "cpufeatures", - "digest 0.9.0", - "opaque-debug", + "digest 0.10.3", ] [[package]] @@ -5178,7 +2548,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f81199417d4e5de3f04b1e871023acea7389672c4135918f05aa9cbf2f2fa809" dependencies = [ - "block-buffer", + "block-buffer 0.9.0", "digest 0.9.0", "keccak", "opaque-debug", @@ -5193,12 +2563,6 @@ dependencies = [ "lazy_static", ] -[[package]] -name = "shlex" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" - [[package]] name = "signal-hook-registry" version = "1.4.0" @@ -5210,21 +2574,21 @@ dependencies = [ [[package]] name = "signature" -version = "1.3.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c19772be3c4dd2ceaacf03cb41d5885f2a02c4d8804884918e3a258480803335" +checksum = "f054c6c1a6e95179d6f23ed974060dcefb2d9388bb7256900badad682c499de4" [[package]] name = "slab" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" +checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32" [[package]] name = "smallvec" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ecab6c735a6bb4139c0caafd0cc3635748bbb3acf4550e8138122099251f309" +checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" [[package]] name = "smart-default" @@ -5239,9 +2603,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.4.2" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dc90fe6c7be1a323296982db1836d1ea9e47b6839496dde9a541bc496df3516" +checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0" dependencies = [ "libc", "winapi", @@ -5253,76 +2617,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" -[[package]] -name = "stable_deref_trait" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" - -[[package]] -name = "standback" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e113fb6f3de07a243d434a56ec6f186dfd51cb08448239fe7bcae73f87ff28ff" -dependencies = [ - "version_check", -] - [[package]] name = "static_assertions" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" -[[package]] -name = "stdweb" -version = "0.4.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" -dependencies = [ - "discard", - "rustc_version 0.2.3", - "stdweb-derive", - "stdweb-internal-macros", - "stdweb-internal-runtime", - "wasm-bindgen", -] - -[[package]] -name = "stdweb-derive" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" -dependencies = [ - "proc-macro2", - "quote", - "serde", - "serde_derive", - "syn", -] - -[[package]] -name = "stdweb-internal-macros" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" -dependencies = [ - "base-x", - "proc-macro2", - "quote", - "serde", - "serde_derive", - "serde_json", - "sha1", - "syn", -] - -[[package]] -name = "stdweb-internal-runtime" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" - [[package]] name = "strsim" version = "0.10.0" @@ -5331,22 +2631,23 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "strum" -version = "0.20.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7318c509b5ba57f18533982607f24070a55d353e90d4cae30c467cdb2ad5ac5c" +checksum = "e96acfc1b70604b8b2f1ffa4c57e59176c7dbb05d556c71ecd2f5498a1dee7f8" dependencies = [ "strum_macros", ] [[package]] name = "strum_macros" -version = "0.20.1" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee8bc6b87a5112aeeab1f4a9f7ab634fe6cbefc4850006df31267f4cfb9e3149" +checksum = "6878079b17446e4d3eba6192bb0a2950d5b14f0ed8424b852310e5a94345d0ef" dependencies = [ - "heck", + "heck 0.4.0", "proc-macro2", "quote", + "rustversion", "syn", ] @@ -5358,9 +2659,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.80" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d010a1623fbd906d51d650a9916aaefc05ffa0e4053ff7fe601167f3e715d194" +checksum = "a07e33e919ebcd69113d5be0e4d70c5707004ff45188910106854f38b960df4a" dependencies = [ "proc-macro2", "quote", @@ -5379,94 +2680,41 @@ dependencies = [ "unicode-xid", ] -[[package]] -name = "sysinfo" -version = "0.14.3" -source = "git+https://github.com/near/sysinfo?rev=3cb97ee79a02754407d2f0f63628f247d7c65e7b#3cb97ee79a02754407d2f0f63628f247d7c65e7b" -dependencies = [ - "cfg-if 0.1.10", - "doc-comment", - "libc", - "ntapi", - "once_cell", - "rayon", - "winapi", -] - [[package]] name = "tap" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" -[[package]] -name = "target-lexicon" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab0e7238dcc7b40a7be719a25365910f6807bd864f4cce6b2e6b873658e2b19d" - -[[package]] -name = "target-lexicon" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7fa7e55043acb85fca6b3c01485a2eeb6b69c5d21002e273c79e465f43b7ac1" - -[[package]] -name = "tempfile" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" -dependencies = [ - "cfg-if 1.0.0", - "fastrand", - "libc", - "redox_syscall 0.2.10", - "remove_dir_all", - "winapi", -] - [[package]] name = "termcolor" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" dependencies = [ "winapi-util", ] -[[package]] -name = "terminal_size" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "textwrap" -version = "0.14.2" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0066c8d12af8b5acd21e00547c3797fde4e8677254a7ee429176ccebbe93dd80" -dependencies = [ - "unicode-width", -] +checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" [[package]] name = "thiserror" -version = "1.0.30" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" +checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.30" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" +checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a" dependencies = [ "proc-macro2", "quote", @@ -5475,148 +2723,52 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd" +checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" dependencies = [ "once_cell", ] -[[package]] -name = "threadpool" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" -dependencies = [ - "num_cpus", -] - -[[package]] -name = "time" -version = "0.1.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "time" -version = "0.2.27" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4752a97f8eebd6854ff91f1c1824cd6160626ac4bd44287f7f4ea2035a02a242" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" dependencies = [ - "const_fn", "libc", - "standback", - "stdweb", - "time-macros", - "version_check", + "wasi 0.10.0+wasi-snapshot-preview1", "winapi", ] [[package]] name = "time" -version = "0.3.7" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "004cbc98f30fa233c61a38bc77e96a9106e65c88f2d3bef182ae952027e5753d" +checksum = "c2702e08a7a860f005826c6815dcac101b19b5eb330c27fe4a5928fec1d20ddd" dependencies = [ "libc", "num_threads", ] -[[package]] -name = "time-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1" -dependencies = [ - "proc-macro-hack", - "time-macros-impl", -] - -[[package]] -name = "time-macros-impl" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd3c141a1b43194f3f56a1411225df8646c55781d5f26db825b3d98507eb482f" -dependencies = [ - "proc-macro-hack", - "proc-macro2", - "quote", - "standback", - "syn", -] - -[[package]] -name = "tinyvec" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83b2a3d4d9091d0abd7eba4dc2710b1718583bd4d8992e2190720ea38f391f7" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" - [[package]] name = "tokio" -version = "1.12.0" +version = "1.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2c2416fdedca8443ae44b4527de1ea633af61d8f7169ffa6e72c5b53d24efcc" +checksum = "4903bf0427cf68dddd5aa6a93220756f8be0c34fcfa9f5e6191e103e15a31395" dependencies = [ - "autocfg 1.0.1", "bytes", "libc", "memchr", "mio", - "num_cpus", "once_cell", - "parking_lot 0.11.2", + "parking_lot", "pin-project-lite", "signal-hook-registry", - "tokio-macros", + "socket2", "winapi", ] -[[package]] -name = "tokio-macros" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2dd85aeaba7b68df939bd357c6afb36c87951be9e80bf9c859f2fc3e9fca0fd" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tokio-native-tls" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" -dependencies = [ - "native-tls", - "tokio", -] - -[[package]] -name = "tokio-openssl" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08f9ffb7809f1b20c1b398d92acf4cc719874b3b2b2d9ea2f09b4a80350878a" -dependencies = [ - "futures-util", - "openssl", - "openssl-sys", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.22.0" @@ -5641,32 +2793,32 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.6.9" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0" +checksum = "f988a1a1adc2fb21f9c12aa96441da33a1728193ae0b95d2be22dbd17fcb4e5c" dependencies = [ "bytes", "futures-core", "futures-sink", - "log", "pin-project-lite", "tokio", + "tracing", ] [[package]] name = "toml" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" dependencies = [ "serde", ] [[package]] name = "tower" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a89fd63ad6adf737582df5db40d286574513c69a11dac5214dc3b5603d6713e" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", @@ -5692,9 +2844,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.29" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "375a639232caf30edfc78e8d89b2d4c375515393e7af7e16f01cd96917fb2105" +checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09" dependencies = [ "cfg-if 1.0.0", "log", @@ -5705,9 +2857,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.18" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f480b8f81512e825f337ad51e94c1eb5d3bbdf2b363dcd01e2b19a9ffe3f8e" +checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c" dependencies = [ "proc-macro2", "quote", @@ -5716,18 +2868,19 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.21" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4ed65637b8390770814083d20756f87bfa2c21bf2f110babdc5438351746e4" +checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f" dependencies = [ "lazy_static", + "valuable", ] [[package]] name = "tracing-log" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6923477a48e41c1951f1999ef8bb5a3023eb723ceadafe78ffb65dc366761e3" +checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" dependencies = [ "lazy_static", "log", @@ -5736,9 +2889,9 @@ dependencies = [ [[package]] name = "tracing-serde" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb65ea441fbb84f9f6748fd496cf7f63ec9af5bca94dd86456978d055e8eb28b" +checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" dependencies = [ "serde", "tracing-core", @@ -5774,9 +2927,9 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] name = "typenum" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec" +checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" [[package]] name = "uint" @@ -5792,9 +2945,9 @@ dependencies = [ [[package]] name = "uint" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6470ab50f482bde894a037a57064480a246dbfdd5960bd65a44824693f08da5f" +checksum = "12f03af7ccf01dd611cc450a0d10dbc9b745770d096473e2faf0ca6e2d66d1e0" dependencies = [ "byteorder", "crunchy", @@ -5802,47 +2955,17 @@ dependencies = [ "static_assertions", ] -[[package]] -name = "unicase" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" -dependencies = [ - "version_check", -] - -[[package]] -name = "unicode-bidi" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f" - -[[package]] -name = "unicode-normalization" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" -dependencies = [ - "tinyvec", -] - [[package]] name = "unicode-segmentation" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b" - -[[package]] -name = "unicode-width" -version = "0.1.9" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" +checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99" [[package]] name = "unicode-xid" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" +checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04" [[package]] name = "untrusted" @@ -5850,54 +2973,17 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" -[[package]] -name = "url" -version = "2.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" -dependencies = [ - "form_urlencoded", - "idna", - "matches", - "percent-encoding", -] - [[package]] name = "urlencoding" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a1f0175e03a0973cf4afd476bef05c26e228520400eb1fd473ad417b1c00ffb" - -[[package]] -name = "uuid" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" -dependencies = [ - "getrandom 0.2.3", -] - -[[package]] -name = "validator" -version = "0.12.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d6937c33ec6039d8071bcf72933146b5bbe378d645d8fa59bdadabfc2a249" -dependencies = [ - "idna", - "lazy_static", - "regex", - "serde", - "serde_derive", - "serde_json", - "url", - "validator_types", -] +checksum = "68b90931029ab9b034b300b797048cf23723400aa757e8a2bfb9d748102f9821" [[package]] -name = "validator_types" -version = "0.12.0" +name = "valuable" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad9680608df133af2c1ddd5eaf1ddce91d60d61b6bc51494ef326458365a470a" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "vcpkg" @@ -5907,15 +2993,9 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "version_check" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" - -[[package]] -name = "void" -version = "1.0.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "want" @@ -5935,15 +3015,21 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" [[package]] name = "wasi" -version = "0.10.2+wasi-snapshot-preview1" +version = "0.10.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06" +checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad" dependencies = [ "cfg-if 1.0.0", "wasm-bindgen-macro", @@ -5951,9 +3037,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca" +checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4" dependencies = [ "bumpalo", "lazy_static", @@ -5966,9 +3052,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01" +checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5976,9 +3062,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc" +checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b" dependencies = [ "proc-macro2", "quote", @@ -5989,342 +3075,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.79" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2" - -[[package]] -name = "wasmer-compiler-near" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c11074b5b8f4170b5ebf0744e811728befb01a70757395c43b528b6441e9c924" -dependencies = [ - "enumset", - "loupe", - "rkyv", - "serde", - "serde_bytes", - "smallvec", - "target-lexicon 0.12.3", - "thiserror", - "wasmer-types-near", - "wasmer-vm-near", - "wasmparser 0.78.2", -] - -[[package]] -name = "wasmer-compiler-singlepass-near" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c95dc7a193f0b607ce19c3a71418ea0d325696087a53755b4610b5b5b02b335b" -dependencies = [ - "byteorder", - "dynasm", - "dynasmrt", - "lazy_static", - "loupe", - "memoffset", - "more-asserts", - "rayon", - "smallvec", - "wasmer-compiler-near", - "wasmer-types-near", - "wasmer-vm-near", -] - -[[package]] -name = "wasmer-engine-near" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55090b4c4cffc8460478fa0b0355d81044750655986a8be93e769f9df3caa6bf" -dependencies = [ - "backtrace", - "enumset", - "lazy_static", - "loupe", - "memmap2", - "more-asserts", - "rustc-demangle", - "serde", - "serde_bytes", - "target-lexicon 0.12.3", - "thiserror", - "wasmer-compiler-near", - "wasmer-types-near", - "wasmer-vm-near", -] - -[[package]] -name = "wasmer-engine-universal-near" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d2a5c1153cf6d9441e3d05101559071a3fb7f44e343f398d5ec89f2f5748f4" -dependencies = [ - "cfg-if 1.0.0", - "enumset", - "leb128", - "loupe", - "region 3.0.0", - "rkyv", - "wasmer-compiler-near", - "wasmer-engine-near", - "wasmer-types-near", - "wasmer-vm-near", - "winapi", -] - -[[package]] -name = "wasmer-runtime-core-near" -version = "0.18.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3fac37da3c625e98708c5dd92d3f642aaf700fd077168d3d0fff277ec6a165" -dependencies = [ - "bincode", - "blake3", - "borsh 0.9.1", - "cc", - "digest 0.8.1", - "errno", - "hex", - "indexmap", - "lazy_static", - "libc", - "nix", - "page_size", - "parking_lot 0.10.2", - "rustc_version 0.2.3", - "serde", - "serde-bench", - "serde_bytes", - "serde_derive", - "smallvec", - "target-lexicon 0.10.0", - "wasmparser 0.51.4", - "winapi", -] - -[[package]] -name = "wasmer-runtime-near" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "158e6fff11e5e1ef805af50637374d5bd43d92017beafa18992cdf7f3f7ae3e4" -dependencies = [ - "lazy_static", - "memmap", - "serde", - "serde_derive", - "wasmer-runtime-core-near", - "wasmer-singlepass-backend-near", -] - -[[package]] -name = "wasmer-singlepass-backend-near" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6edd0ba6c0bcf9b279186d4dbe81649dda3e5ef38f586865943de4dcd653f8" -dependencies = [ - "bincode", - "borsh 0.9.1", - "byteorder", - "dynasm", - "dynasmrt", - "lazy_static", - "libc", - "nix", - "serde", - "serde_derive", - "smallvec", - "wasmer-runtime-core-near", -] - -[[package]] -name = "wasmer-types-near" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4fae5b0041c76c1b114b3286503a54d42c38eb88146724919b5610c66ecd548" -dependencies = [ - "indexmap", - "loupe", - "rkyv", - "serde", - "thiserror", -] - -[[package]] -name = "wasmer-vm-near" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db06e0c8e20945000c075237f1b5afb682bf80e2bec875ed9b9a633ef41960c7" -dependencies = [ - "backtrace", - "cc", - "cfg-if 1.0.0", - "indexmap", - "libc", - "loupe", - "memoffset", - "more-asserts", - "region 3.0.0", - "rkyv", - "serde", - "thiserror", - "wasmer-types-near", - "winapi", -] - -[[package]] -name = "wasmparser" -version = "0.51.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aeb1956b19469d1c5e63e459d29e7b5aa0f558d9f16fcef09736f8a265e6c10a" - -[[package]] -name = "wasmparser" -version = "0.78.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52144d4c78e5cf8b055ceab8e5fa22814ce4315d6002ad32cfd914f37c12fd65" - -[[package]] -name = "wasmparser" -version = "0.81.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98930446519f63d00a836efdc22f67766ceae8dbcc1571379f2bcabc6b2b9abc" - -[[package]] -name = "wasmtime" -version = "0.33.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c9c724da92e39a85d2231d4c2a942c8be295211441dbca581c6c3f3f45a9f00" -dependencies = [ - "anyhow", - "backtrace", - "bincode", - "cfg-if 1.0.0", - "cpp_demangle", - "indexmap", - "lazy_static", - "libc", - "log", - "object", - "paste", - "psm", - "region 2.2.0", - "rustc-demangle", - "serde", - "target-lexicon 0.12.3", - "wasmparser 0.81.0", - "wasmtime-cranelift", - "wasmtime-environ", - "wasmtime-jit", - "wasmtime-runtime", - "winapi", -] - -[[package]] -name = "wasmtime-cranelift" -version = "0.33.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1762765dd69245f00e5d9783b695039e449a7be0f9c5383e4c78465dd6131aeb" -dependencies = [ - "anyhow", - "cranelift-codegen", - "cranelift-entity", - "cranelift-frontend", - "cranelift-native", - "cranelift-wasm", - "gimli", - "log", - "more-asserts", - "object", - "target-lexicon 0.12.3", - "thiserror", - "wasmparser 0.81.0", - "wasmtime-environ", -] - -[[package]] -name = "wasmtime-environ" -version = "0.33.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4468301d95ec71710bb6261382efe27d1296447711645e3dbabaea6e4de3504" -dependencies = [ - "anyhow", - "cranelift-entity", - "gimli", - "indexmap", - "log", - "more-asserts", - "object", - "serde", - "target-lexicon 0.12.3", - "thiserror", - "wasmparser 0.81.0", - "wasmtime-types", -] - -[[package]] -name = "wasmtime-jit" -version = "0.33.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab0ae6e581ff014b470ec35847ea3c0b4c3ace89a55df5a04c802a11f4574e7d" -dependencies = [ - "addr2line", - "anyhow", - "bincode", - "cfg-if 1.0.0", - "gimli", - "object", - "region 2.2.0", - "serde", - "target-lexicon 0.12.3", - "thiserror", - "wasmtime-environ", - "wasmtime-runtime", - "winapi", -] - -[[package]] -name = "wasmtime-runtime" -version = "0.33.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9c28877ae37a367cda7b52b8887589816152e95dde9b7c80cc686f52761961" -dependencies = [ - "anyhow", - "backtrace", - "cc", - "cfg-if 1.0.0", - "indexmap", - "lazy_static", - "libc", - "log", - "mach", - "memoffset", - "more-asserts", - "rand 0.8.4", - "region 2.2.0", - "rustix", - "thiserror", - "wasmtime-environ", - "winapi", -] - -[[package]] -name = "wasmtime-types" -version = "0.33.1" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "395726e8f5dd8c57cb0db445627b842343f7e29ed7489467fdf7953ed9d3cd4f" -dependencies = [ - "cranelift-entity", - "serde", - "thiserror", - "wasmparser 0.81.0", -] +checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744" [[package]] name = "web-sys" -version = "0.3.56" +version = "0.3.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c060b319f29dd25724f09a2ba1418f142f539b2be99fbf4d2d5a8f7330afb8eb" +checksum = "7b17e741662c70c8bd24ac5c5b18de314a2c26c32bf8346ee1e6f53de919c283" dependencies = [ "js-sys", "wasm-bindgen", @@ -6384,49 +3143,74 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "wyz" -version = "0.2.0" +name = "windows-sys" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" +dependencies = [ + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_msvc", +] [[package]] -name = "xmlparser" -version = "0.13.3" +name = "windows_aarch64_msvc" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "114ba2b24d2167ef6d67d7d04c8cc86522b87f490025f39f0303b7db5bf5e3d8" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" [[package]] -name = "xz2" -version = "0.1.6" +name = "windows_i686_gnu" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c179869f34fc7c01830d3ce7ea2086bc3a07e0d35289b667d0a8bf910258926c" -dependencies = [ - "lzma-sys", -] +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" [[package]] -name = "yaml-rust" -version = "0.4.5" +name = "windows_i686_msvc" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" -dependencies = [ - "linked-hash-map", -] +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" + +[[package]] +name = "wyz" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214" + +[[package]] +name = "xmlparser" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "114ba2b24d2167ef6d67d7d04c8cc86522b87f490025f39f0303b7db5bf5e3d8" [[package]] name = "zeroize" -version = "1.4.2" +version = "1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf68b08513768deaa790264a7fac27a58cbf2705cfcdc9448362229217d7e970" +checksum = "94693807d016b2f2d2e14420eb3bfcca689311ff775dcf113d74ea624b7cdf07" dependencies = [ "zeroize_derive", ] [[package]] name = "zeroize_derive" -version = "1.2.0" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdff2024a851a322b08f179173ae2ba620445aef1e838f0c196820eade4ae0c7" +checksum = "3f8f187641dad4f680d25c4bfc4225b418165984179f26ca76ec4fb6441d3a17" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 32a1ea91..e830e378 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,23 +1,19 @@ [package] name = "indexer-explorer" -version = "0.10.12-lake.1" +version = "0.10.19" authors = ["Near Inc "] edition = "2021" [dependencies] -actix = "=0.11.0-beta.2" -actix-rt = "=2.2.0" # remove it once actix is upgraded to 0.11+ -actix-web = "=4.0.0-beta.6" -actix-http = "=3.0.0-beta.6" -actix-tls = "=3.0.0-beta.5" -actix_derive = "=0.6.0-beta.1" +actix = "0.13.0" +actix-rt = "2.2.0" anyhow = "1.0.51" base64 = "0.11" bigdecimal = "=0.1.0" borsh = "0.7.1" cached = "0.23.0" chrono = "0.4.19" -clap = { version = "3.0.0-beta.5", features = ["color", "derive", "env"] } +clap = { version = "3.1.6", features = ["color", "derive", "env"] } diesel = { version = "1.4.7", features = ["postgres", "numeric", "serde_json"] } # Using hacky diesel-derive-enum https://github.com/adwhit/diesel-derive-enum/issues/52 diesel-derive-enum = { git = "https://github.com/khorolets/diesel-derive-enum.git", branch = "lookup-hack", features = ["postgres"] } @@ -32,16 +28,13 @@ openssl-probe = { version = "0.1.2" } r2d2 = "0.8.8" serde = { version = "1", features = ["derive"] } serde_json = "1.0.55" -tokio = { version = "1.1", features = ["sync", "time", "macros", "rt-multi-thread"] } -tokio-stream = { version = "0.1.8" } +tokio = { version = "1.1", features = ["sync", "time"] } +tokio-stream = { version = "0.1" } tracing = "0.1.13" tracing-subscriber = "0.2.4" uint = { version = "0.8.3", default-features = false } -actix-diesel = { git = "https://github.com/frol/actix-diesel", branch = "actix-0.11-beta.2" } -near-indexer = { git = "https://github.com/near/nearcore", rev = "5f09a3bf042b32d1ff26554433ad6449199ea02a" } -near-crypto = { git = "https://github.com/near/nearcore", rev = "5f09a3bf042b32d1ff26554433ad6449199ea02a" } -near-primitives = { git = "https://github.com/near/nearcore", rev = "5f09a3bf042b32d1ff26554433ad6449199ea02a" } -near-indexer-primitives = { git = "https://github.com/near/nearcore", rev = "5f09a3bf042b32d1ff26554433ad6449199ea02a" } -near-client = { git = "https://github.com/near/nearcore", rev = "539f254f793a3324a29e6e97e3b804b9fa4f27a2" } -near-lake-framework = { git = "https://github.com/near/near-lake-framework", rev = "34f77424051bf032bc5f1d21565ef6e021937b71" } +actix-diesel = { git = "https://github.com/frol/actix-diesel", rev = "3a001986c89dfabfc3c448d8bae28525101b4992" } +near-lake-framework = "0.5" +near-primitives = "0.14" +near-crypto = "0.14" \ No newline at end of file diff --git a/rust-toolchain b/rust-toolchain index 69478d18..91951fd8 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -1.58.1 +1.61.0 diff --git a/src/aggregated/account_details.rs b/src/aggregated/account_details.rs index ed59ed95..74474023 100644 --- a/src/aggregated/account_details.rs +++ b/src/aggregated/account_details.rs @@ -19,7 +19,7 @@ pub(crate) async fn get_contract_code_hash( view_client: &Addr, account_id: &near_primitives::types::AccountId, block_height: &near_primitives::types::BlockHeight, -) -> anyhow::Result { +) -> anyhow::Result { get_account_view_for_block_height(view_client, account_id, block_height) .await .map(|account| account.code_hash) @@ -30,11 +30,11 @@ async fn get_account_view_for_block_height( view_client: &Addr, account_id: &near_primitives::types::AccountId, block_height: &near_primitives::types::BlockHeight, -) -> anyhow::Result { +) -> anyhow::Result { let block_reference = near_primitives::types::BlockReference::BlockId( near_primitives::types::BlockId::Height(*block_height), ); - let request = near_primitives::views::QueryRequest::ViewAccount { + let request = near_lake_framework::near_indexer_primitives::views::QueryRequest::ViewAccount { account_id: account_id.clone(), }; let query = Query::new(block_reference, request); @@ -56,7 +56,7 @@ async fn get_account_view_for_block_height( })?; match account_response.kind { - near_primitives::views::QueryResponseKind::ViewAccount(account) => Ok(account), + near_lake_framework::near_indexer_primitives::views::QueryResponseKind::ViewAccount(account) => Ok(account), _ => anyhow::bail!( "Failed to extract ViewAccount response for account {}, block {}", account_id, diff --git a/src/aggregated/circulating_supply/lockup.rs b/src/aggregated/circulating_supply/lockup.rs index 9c5bddc6..5f58703a 100644 --- a/src/aggregated/circulating_supply/lockup.rs +++ b/src/aggregated/circulating_supply/lockup.rs @@ -24,7 +24,7 @@ pub(super) async fn get_lockup_contract_state( let block_reference = near_primitives::types::BlockReference::BlockId( near_primitives::types::BlockId::Height(*block_height), ); - let request = near_primitives::views::QueryRequest::ViewState { + let request = near_lake_framework::near_indexer_primitives::views::QueryRequest::ViewState { account_id: account_id.clone(), prefix: vec![].into(), }; @@ -47,7 +47,7 @@ pub(super) async fn get_lockup_contract_state( })?; let view_state_result = match state_response.kind { - near_primitives::views::QueryResponseKind::ViewState(x) => x, + near_lake_framework::near_indexer_primitives::views::QueryResponseKind::ViewState(x) => x, _ => { anyhow::bail!( "Failed to extract ViewState response for lockup contract {}, block_height {}", @@ -89,7 +89,7 @@ pub(super) async fn get_lockup_contract_state( // https://github.com/near/core-contracts/pull/136 // For each contract, we should choose the logic based on the binary version of the contract pub(super) fn is_bug_inside_contract( - code_hash: &near_primitives::hash::CryptoHash, + code_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, account_id: &near_primitives::types::AccountId, ) -> anyhow::Result { match &*code_hash.to_string() { diff --git a/src/db_adapters/access_keys.rs b/src/db_adapters/access_keys.rs index 88a60585..84836e9e 100644 --- a/src/db_adapters/access_keys.rs +++ b/src/db_adapters/access_keys.rs @@ -8,15 +8,13 @@ use diesel::{ExpressionMethods, PgConnection, QueryDsl}; use futures::try_join; use tracing::info; -use near_indexer::near_primitives; - use crate::models; use crate::schema; pub(crate) async fn handle_access_keys( pool: &actix_diesel::Database, - outcomes: &[near_indexer::IndexerExecutionOutcomeWithReceipt], - block_height: near_primitives::types::BlockHeight, + outcomes: &[near_lake_framework::near_indexer_primitives::IndexerExecutionOutcomeWithReceipt], + block_height: near_lake_framework::near_indexer_primitives::types::BlockHeight, ) -> anyhow::Result<()> { if outcomes.is_empty() { return Ok(()); @@ -26,8 +24,8 @@ pub(crate) async fn handle_access_keys( .filter(|outcome_with_receipt| { matches!( outcome_with_receipt.execution_outcome.outcome.status, - near_primitives::views::ExecutionStatusView::SuccessValue(_) - | near_primitives::views::ExecutionStatusView::SuccessReceiptId(_) + near_lake_framework::near_indexer_primitives::views::ExecutionStatusView::SuccessValue(_) + | near_lake_framework::near_indexer_primitives::views::ExecutionStatusView::SuccessReceiptId(_) ) }) .map(|outcome_with_receipt| &outcome_with_receipt.receipt); @@ -36,10 +34,10 @@ pub(crate) async fn handle_access_keys( let mut deleted_accounts = HashMap::::new(); for receipt in successful_receipts { - if let near_primitives::views::ReceiptEnumView::Action { actions, .. } = &receipt.receipt { + if let near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { actions, .. } = &receipt.receipt { for action in actions { match action { - near_primitives::views::ActionView::DeleteAccount { .. } => { + near_lake_framework::near_indexer_primitives::views::ActionView::DeleteAccount { .. } => { deleted_accounts.insert( receipt.receiver_id.to_string(), receipt.receipt_id.to_string(), @@ -54,7 +52,7 @@ pub(crate) async fn handle_access_keys( Some(receipt.receipt_id.to_string()); }); } - near_primitives::views::ActionView::AddKey { + near_lake_framework::near_indexer_primitives::views::ActionView::AddKey { public_key, access_key, } => { @@ -69,7 +67,7 @@ pub(crate) async fn handle_access_keys( ), ); } - near_primitives::views::ActionView::DeleteKey { public_key } => { + near_lake_framework::near_indexer_primitives::views::ActionView::DeleteKey { public_key } => { access_keys .entry((public_key.to_string(), receipt.receiver_id.to_string())) .and_modify(|existing_access_key| { @@ -87,7 +85,7 @@ pub(crate) async fn handle_access_keys( last_update_block_height: block_height.into(), }); } - near_indexer::near_primitives::views::ActionView::Transfer { .. } => { + near_lake_framework::near_indexer_primitives::views::ActionView::Transfer { .. } => { if receipt.receiver_id.len() != 64usize { continue; } @@ -100,9 +98,9 @@ pub(crate) async fn handle_access_keys( models::access_keys::AccessKey::from_action_view( &near_crypto::PublicKey::from(public_key.clone()), &receipt.receiver_id, - &near_primitives::views::AccessKeyView { + &near_lake_framework::near_indexer_primitives::views::AccessKeyView { nonce: 0, - permission: near_primitives::views::AccessKeyPermissionView::FullAccess + permission: near_lake_framework::near_indexer_primitives::views::AccessKeyPermissionView::FullAccess }, &receipt.receipt_id, block_height, diff --git a/src/db_adapters/account_changes.rs b/src/db_adapters/account_changes.rs index a971195e..375ee09a 100644 --- a/src/db_adapters/account_changes.rs +++ b/src/db_adapters/account_changes.rs @@ -8,8 +8,8 @@ use crate::schema; /// Saves state change related to account to database pub(crate) async fn store_account_changes( pool: &actix_diesel::Database, - shards: &[near_indexer::IndexerShard], - block_hash: &near_indexer::near_primitives::hash::CryptoHash, + shards: &[near_lake_framework::near_indexer_primitives::IndexerShard], + block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, block_timestamp: u64, ) -> anyhow::Result<()> { let futures = shards.iter().map(|shard| { @@ -21,8 +21,8 @@ pub(crate) async fn store_account_changes( async fn store_account_changes_for_chunk( pool: &actix_diesel::Database, - state_changes: &[near_indexer::near_primitives::views::StateChangeWithCauseView], - block_hash: &near_indexer::near_primitives::hash::CryptoHash, + state_changes: &[near_lake_framework::near_indexer_primitives::views::StateChangeWithCauseView], + block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, block_timestamp: u64, ) -> anyhow::Result<()> { if state_changes.is_empty() { diff --git a/src/db_adapters/accounts.rs b/src/db_adapters/accounts.rs index d00d8778..d93dd18d 100644 --- a/src/db_adapters/accounts.rs +++ b/src/db_adapters/accounts.rs @@ -10,16 +10,14 @@ use futures::try_join; use tracing::info; -use near_indexer::near_primitives; - use crate::models; use crate::schema; /// Saves new Accounts to database or deletes the ones should be deleted pub(crate) async fn handle_accounts( pool: &actix_diesel::Database, - outcomes: &[near_indexer::IndexerExecutionOutcomeWithReceipt], - block_height: near_primitives::types::BlockHeight, + outcomes: &[near_lake_framework::near_indexer_primitives::IndexerExecutionOutcomeWithReceipt], + block_height: near_lake_framework::near_indexer_primitives::types::BlockHeight, ) -> anyhow::Result<()> { if outcomes.is_empty() { return Ok(()); @@ -29,20 +27,20 @@ pub(crate) async fn handle_accounts( .filter(|outcome_with_receipt| { matches!( outcome_with_receipt.execution_outcome.outcome.status, - near_primitives::views::ExecutionStatusView::SuccessValue(_) - | near_primitives::views::ExecutionStatusView::SuccessReceiptId(_) + near_lake_framework::near_indexer_primitives::views::ExecutionStatusView::SuccessValue(_) + | near_lake_framework::near_indexer_primitives::views::ExecutionStatusView::SuccessReceiptId(_) ) }) .map(|outcome_with_receipt| &outcome_with_receipt.receipt); let mut accounts = - HashMap::::new(); + HashMap::::new(); for receipt in successful_receipts { - if let near_primitives::views::ReceiptEnumView::Action { actions, .. } = &receipt.receipt { + if let near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { actions, .. } = &receipt.receipt { for action in actions { match action { - near_primitives::views::ActionView::CreateAccount => { + near_lake_framework::near_indexer_primitives::views::ActionView::CreateAccount => { accounts.insert( receipt.receiver_id.clone(), models::accounts::Account::new_from_receipt( @@ -52,7 +50,7 @@ pub(crate) async fn handle_accounts( ), ); } - near_primitives::views::ActionView::Transfer { .. } => { + near_lake_framework::near_indexer_primitives::views::ActionView::Transfer { .. } => { if receipt.receiver_id.len() == 64usize { accounts.insert( receipt.receiver_id.clone(), @@ -64,7 +62,7 @@ pub(crate) async fn handle_accounts( ); } } - near_primitives::views::ActionView::DeleteAccount { .. } => { + near_lake_framework::near_indexer_primitives::views::ActionView::DeleteAccount { .. } => { accounts .entry(receipt.receiver_id.clone()) .and_modify(|existing_account| { @@ -225,8 +223,8 @@ pub(crate) async fn store_accounts_from_genesis( pub(crate) async fn get_lockup_account_ids_at_block_height( pool: &actix_diesel::Database, - block_height: &near_primitives::types::BlockHeight, -) -> anyhow::Result> { + block_height: &near_lake_framework::near_indexer_primitives::types::BlockHeight, +) -> anyhow::Result> { // Diesel does not support named joins // https://github.com/diesel-rs/diesel/pull/2254 // Raw SQL (diesel-1.4.7/src/query_builder/functions.rs:464) does not support async methods @@ -269,7 +267,7 @@ pub(crate) async fn get_lockup_account_ids_at_block_height( results .into_iter() .map(|account_id_string| - near_primitives::types::AccountId::try_from(account_id_string) + near_lake_framework::near_indexer_primitives::types::AccountId::try_from(account_id_string) .expect("Selecting lockup account ids bumped into the account_id which is not valid; that should never happen")) .collect() }) diff --git a/src/db_adapters/assets/events.rs b/src/db_adapters/assets/events.rs index becb4f77..51b84b33 100644 --- a/src/db_adapters/assets/events.rs +++ b/src/db_adapters/assets/events.rs @@ -8,7 +8,7 @@ use super::event_types; pub(crate) async fn store_events( pool: &Database, - streamer_message: &near_indexer::StreamerMessage, + streamer_message: &near_lake_framework::near_indexer_primitives:: StreamerMessage, ) -> anyhow::Result<()> { let futures = streamer_message.shards.iter().map(|shard| { collect_and_store_events(pool, shard, streamer_message.block.header.timestamp) @@ -43,7 +43,7 @@ pub(crate) async fn detect_db_error( async fn collect_and_store_events( pool: &Database, - shard: &near_indexer::IndexerShard, + shard: &near_lake_framework::near_indexer_primitives::IndexerShard, block_timestamp: u64, ) -> anyhow::Result<()> { let mut ft_events_with_outcomes = Vec::new(); @@ -80,7 +80,7 @@ async fn collect_and_store_events( } fn extract_events( - outcome: &near_indexer::IndexerExecutionOutcomeWithReceipt, + outcome: &near_lake_framework::near_indexer_primitives::IndexerExecutionOutcomeWithReceipt, ) -> Vec { let prefix = "EVENT_JSON:"; outcome.execution_outcome.outcome.logs.iter().filter_map(|untrimmed_log| { diff --git a/src/db_adapters/assets/fungible_token_events.rs b/src/db_adapters/assets/fungible_token_events.rs index c78b8a87..9a55b608 100644 --- a/src/db_adapters/assets/fungible_token_events.rs +++ b/src/db_adapters/assets/fungible_token_events.rs @@ -11,11 +11,11 @@ use super::event_types; pub(crate) async fn store_ft_events( pool: &Database, - shard: &near_indexer::IndexerShard, + shard: &near_lake_framework::near_indexer_primitives::IndexerShard, block_timestamp: u64, events_with_outcomes: &[( assets::event_types::Nep141Event, - &near_indexer::IndexerExecutionOutcomeWithReceipt, + &near_lake_framework::near_indexer_primitives::IndexerExecutionOutcomeWithReceipt, )], ) -> anyhow::Result<()> { let ft_events = compose_ft_db_events(events_with_outcomes, block_timestamp, &shard.shard_id); @@ -48,10 +48,10 @@ async fn detect_ft_db_error(async_error: &AsyncError) -> fn compose_ft_db_events( events_with_outcomes: &[( assets::event_types::Nep141Event, - &near_indexer::IndexerExecutionOutcomeWithReceipt, + &near_lake_framework::near_indexer_primitives::IndexerExecutionOutcomeWithReceipt, )], block_timestamp: u64, - shard_id: &near_indexer::near_primitives::types::ShardId, + shard_id: &near_lake_framework::near_indexer_primitives::types::ShardId, ) -> Vec { let mut ft_events = Vec::new(); for (event, outcome) in events_with_outcomes { diff --git a/src/db_adapters/assets/non_fungible_token_events.rs b/src/db_adapters/assets/non_fungible_token_events.rs index b5c41f27..7e6bee0e 100644 --- a/src/db_adapters/assets/non_fungible_token_events.rs +++ b/src/db_adapters/assets/non_fungible_token_events.rs @@ -11,11 +11,11 @@ use super::event_types; pub(crate) async fn store_nft_events( pool: &Database, - shard: &near_indexer::IndexerShard, + shard: &near_lake_framework::near_indexer_primitives::IndexerShard, block_timestamp: u64, events_with_outcomes: &[( assets::event_types::Nep171Event, - &near_indexer::IndexerExecutionOutcomeWithReceipt, + &near_lake_framework::near_indexer_primitives::IndexerExecutionOutcomeWithReceipt, )], ) -> anyhow::Result<()> { let nft_events = compose_nft_db_events(events_with_outcomes, block_timestamp, &shard.shard_id); @@ -48,10 +48,10 @@ async fn detect_nft_db_error(async_error: &AsyncError) -> fn compose_nft_db_events( events_with_outcomes: &[( assets::event_types::Nep171Event, - &near_indexer::IndexerExecutionOutcomeWithReceipt, + &near_lake_framework::near_indexer_primitives::IndexerExecutionOutcomeWithReceipt, )], block_timestamp: u64, - shard_id: &near_indexer::near_primitives::types::ShardId, + shard_id: &near_lake_framework::near_indexer_primitives::types::ShardId, ) -> Vec { let mut nft_events = Vec::new(); for (event, outcome) in events_with_outcomes { diff --git a/src/db_adapters/blocks.rs b/src/db_adapters/blocks.rs index ed50eb28..f2a22653 100644 --- a/src/db_adapters/blocks.rs +++ b/src/db_adapters/blocks.rs @@ -3,15 +3,13 @@ use anyhow::Context; use bigdecimal::{BigDecimal, ToPrimitive}; use diesel::{ExpressionMethods, PgConnection, QueryDsl}; -use near_indexer::near_primitives; - use crate::models; use crate::schema; /// Saves block to database pub(crate) async fn store_block( pool: &actix_diesel::Database, - block: &near_primitives::views::BlockView, + block: &near_lake_framework::near_indexer_primitives::views::BlockView, ) -> anyhow::Result<()> { let block_model = models::blocks::Block::from(block); diff --git a/src/db_adapters/chunks.rs b/src/db_adapters/chunks.rs index b083c0cd..c177e77a 100644 --- a/src/db_adapters/chunks.rs +++ b/src/db_adapters/chunks.rs @@ -7,8 +7,8 @@ use crate::schema; /// Saves chunks to database pub(crate) async fn store_chunks( pool: &actix_diesel::Database, - shards: &[near_indexer::IndexerShard], - block_hash: &near_indexer::near_primitives::hash::CryptoHash, + shards: &[near_lake_framework::near_indexer_primitives::IndexerShard], + block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, ) -> anyhow::Result<()> { if shards.is_empty() { return Ok(()); diff --git a/src/db_adapters/execution_outcomes.rs b/src/db_adapters/execution_outcomes.rs index c9f2b5de..8ade3a53 100644 --- a/src/db_adapters/execution_outcomes.rs +++ b/src/db_adapters/execution_outcomes.rs @@ -8,7 +8,7 @@ use crate::schema; pub(crate) async fn store_execution_outcomes( pool: &actix_diesel::Database, - shards: &[near_indexer::IndexerShard], + shards: &[near_lake_framework::near_indexer_primitives::IndexerShard], block_timestamp: u64, receipts_cache: crate::ReceiptsCache, ) -> anyhow::Result<()> { @@ -28,8 +28,8 @@ pub(crate) async fn store_execution_outcomes( /// Saves ExecutionOutcome to database and then saves ExecutionOutcomesReceipts pub async fn store_execution_outcomes_for_chunk( pool: &actix_diesel::Database, - execution_outcomes: &[near_indexer::IndexerExecutionOutcomeWithReceipt], - shard_id: near_indexer::near_primitives::types::ShardId, + execution_outcomes: &[near_lake_framework::near_indexer_primitives::IndexerExecutionOutcomeWithReceipt], + shard_id: near_lake_framework::near_indexer_primitives::types::ShardId, block_timestamp: u64, receipts_cache: crate::ReceiptsCache, ) -> anyhow::Result<()> { diff --git a/src/db_adapters/genesis.rs b/src/db_adapters/genesis.rs index 96518fa2..8dfd941d 100644 --- a/src/db_adapters/genesis.rs +++ b/src/db_adapters/genesis.rs @@ -85,7 +85,7 @@ pub(crate) async fn store_genesis_records( } match record { - near_indexer::near_primitives::state_record::StateRecord::Account { + near_primitives::state_record::StateRecord::Account { account_id, .. } => { @@ -94,7 +94,7 @@ pub(crate) async fn store_genesis_records( genesis_height, )); } - near_indexer::near_primitives::state_record::StateRecord::AccessKey { + near_primitives::state_record::StateRecord::AccessKey { account_id, public_key, access_key, diff --git a/src/db_adapters/mod.rs b/src/db_adapters/mod.rs index 6f9b6ef5..13177b1e 100644 --- a/src/db_adapters/mod.rs +++ b/src/db_adapters/mod.rs @@ -6,7 +6,7 @@ pub(crate) mod assets; pub(crate) mod blocks; pub(crate) mod chunks; pub(crate) mod execution_outcomes; -pub(crate) mod genesis; +//pub(crate) mod genesis; pub(crate) mod receipts; pub(crate) mod transactions; diff --git a/src/db_adapters/receipts.rs b/src/db_adapters/receipts.rs index 8bd80ced..48393455 100644 --- a/src/db_adapters/receipts.rs +++ b/src/db_adapters/receipts.rs @@ -2,8 +2,6 @@ use std::collections::HashMap; use std::convert::TryFrom; use std::str::FromStr; -use near_indexer::near_primitives; - use actix_diesel::dsl::AsyncRunQueryDsl; use cached::Cached; use diesel::pg::expression::array_comparison::any; @@ -19,8 +17,8 @@ use crate::schema; /// Saves receipts to database pub(crate) async fn store_receipts( pool: &actix_diesel::Database, - shards: &[near_indexer::IndexerShard], - block_hash: &near_indexer::near_primitives::hash::CryptoHash, + shards: &[near_lake_framework::near_indexer_primitives::IndexerShard], + block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, block_timestamp: u64, strict_mode: bool, receipts_cache: crate::ReceiptsCache, @@ -46,15 +44,15 @@ pub(crate) async fn store_receipts( async fn store_chunk_receipts( pool: &actix_diesel::Database, - receipts: &[near_indexer::near_primitives::views::ReceiptView], - block_hash: &near_indexer::near_primitives::hash::CryptoHash, - chunk_hash: &near_indexer::near_primitives::hash::CryptoHash, + receipts: &[near_lake_framework::near_indexer_primitives::views::ReceiptView], + block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, + chunk_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, block_timestamp: u64, strict_mode: bool, receipts_cache: crate::ReceiptsCache, ) -> anyhow::Result<()> { let mut skipping_receipt_ids = - std::collections::HashSet::::new(); + std::collections::HashSet::::new(); let tx_hashes_for_receipts = find_tx_hashes_for_receipts( pool, @@ -75,10 +73,10 @@ async fn store_chunk_receipts( // In case of Action Receipt we are looking for ReceiptId // In case of Data Receipt we are looking for DataId let receipt_or_data_id = match r.receipt { - near_primitives::views::ReceiptEnumView::Action { .. } => { + near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { .. } => { crate::ReceiptOrDataId::ReceiptId(r.receipt_id) } - near_primitives::views::ReceiptEnumView::Data { data_id, .. } => { + near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Data { data_id, .. } => { crate::ReceiptOrDataId::DataId(data_id) } }; @@ -110,7 +108,7 @@ async fn store_chunk_receipts( // to ReceiptsCache let mut receipts_cache_lock = receipts_cache.lock().await; for receipt in receipts { - if let near_primitives::views::ReceiptEnumView::Action { + if let near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { output_data_receivers, .. } = &receipt.receipt @@ -135,15 +133,15 @@ async fn store_chunk_receipts( save_receipts(pool, receipt_models).await?; let (action_receipts, data_receipts): ( - Vec<&near_indexer::near_primitives::views::ReceiptView>, - Vec<&near_indexer::near_primitives::views::ReceiptView>, + Vec<&near_lake_framework::near_indexer_primitives::views::ReceiptView>, + Vec<&near_lake_framework::near_indexer_primitives::views::ReceiptView>, ) = receipts .iter() .filter(|r| !skipping_receipt_ids.contains(&r.receipt_id)) .partition(|receipt| { matches!( receipt.receipt, - near_indexer::near_primitives::views::ReceiptEnumView::Action { .. } + near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { .. } ) }); @@ -159,10 +157,10 @@ async fn store_chunk_receipts( /// Looks for already created parent transaction hash for given receipts async fn find_tx_hashes_for_receipts( pool: &actix_diesel::Database, - mut receipts: Vec, + mut receipts: Vec, strict_mode: bool, - block_hash: &near_indexer::near_primitives::hash::CryptoHash, - chunk_hash: &near_indexer::near_primitives::hash::CryptoHash, + block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, + chunk_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, receipts_cache: crate::ReceiptsCache, ) -> anyhow::Result> { let mut tx_hashes_for_receipts: HashMap< @@ -174,7 +172,7 @@ async fn find_tx_hashes_for_receipts( // add receipt-transaction pairs from the cache to the response tx_hashes_for_receipts.extend(receipts.iter().filter_map(|receipt| { match receipt.receipt { - near_primitives::views::ReceiptEnumView::Action { .. } => receipts_cache_lock + near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { .. } => receipts_cache_lock .cache_get(&crate::ReceiptOrDataId::ReceiptId(receipt.receipt_id)) .map(|parent_transaction_hash| { ( @@ -182,7 +180,7 @@ async fn find_tx_hashes_for_receipts( parent_transaction_hash.clone(), ) }), - near_primitives::views::ReceiptEnumView::Data { data_id, .. } => { + near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Data { data_id, .. } => { // Pair DataId:ParentTransactionHash won't be used after this moment // We want to clean it up to prevent our cache from growing receipts_cache_lock @@ -201,10 +199,10 @@ async fn find_tx_hashes_for_receipts( // discard the Receipts already in cache from the attempts to search receipts.retain(|r| match r.receipt { - near_primitives::views::ReceiptEnumView::Data { data_id, .. } => { + near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Data { data_id, .. } => { !tx_hashes_for_receipts.contains_key(&crate::ReceiptOrDataId::DataId(data_id)) } - near_primitives::views::ReceiptEnumView::Action { .. } => { + near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { .. } => { !tx_hashes_for_receipts.contains_key(&crate::ReceiptOrDataId::ReceiptId(r.receipt_id)) } }); @@ -225,7 +223,7 @@ async fn find_tx_hashes_for_receipts( let data_ids: Vec = receipts .iter() .filter_map(|r| match r.receipt { - near_indexer::near_primitives::views::ReceiptEnumView::Data { data_id, .. } => { + near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Data { data_id, .. } => { Some(data_id.to_string()) } _ => None, @@ -262,7 +260,7 @@ async fn find_tx_hashes_for_receipts( |(receipt_id_string, transaction_hash_string): (String, String)| { ( crate::ReceiptOrDataId::DataId( - near_primitives::hash::CryptoHash::from_str( + near_lake_framework::near_indexer_primitives::CryptoHash::from_str( &receipt_id_string, ) .expect("Failed to convert String to CryptoHash"), @@ -298,7 +296,7 @@ async fn find_tx_hashes_for_receipts( )> = receipts .iter() .filter_map(|r| match r.receipt { - near_indexer::near_primitives::views::ReceiptEnumView::Data { + near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Data { data_id, .. } => tx_hashes_for_data_id_via_data_output_hashmap .get(&crate::ReceiptOrDataId::DataId(data_id)) @@ -341,7 +339,7 @@ async fn find_tx_hashes_for_receipts( .filter(|r| { matches!( r.receipt, - near_indexer::near_primitives::views::ReceiptEnumView::Action { .. } + near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { .. } ) }) .map(|r| r.receipt_id.to_string()) @@ -364,7 +362,7 @@ async fn find_tx_hashes_for_receipts( |(receipt_id_string, transaction_hash_string)| { ( crate::ReceiptOrDataId::ReceiptId( - near_primitives::hash::CryptoHash::from_str(&receipt_id_string) + near_lake_framework::near_indexer_primitives::CryptoHash::from_str(&receipt_id_string) .expect("Failed to convert String to CryptoHash"), ), transaction_hash_string, @@ -392,7 +390,7 @@ async fn find_tx_hashes_for_receipts( .filter(|r| { matches!( r.receipt, - near_indexer::near_primitives::views::ReceiptEnumView::Action { .. } + near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { .. } ) }) .map(|r| r.receipt_id.to_string()) @@ -414,7 +412,7 @@ async fn find_tx_hashes_for_receipts( |(receipt_id_string, transaction_hash_string)| { ( crate::ReceiptOrDataId::ReceiptId( - near_primitives::hash::CryptoHash::from_str(&receipt_id_string) + near_lake_framework::near_indexer_primitives::CryptoHash::from_str(&receipt_id_string) .expect("Failed to convert String to CryptoHash"), ), transaction_hash_string, @@ -472,7 +470,7 @@ async fn save_receipts( async fn store_receipt_actions( pool: &actix_diesel::Database, - receipts: Vec<&near_indexer::near_primitives::views::ReceiptView>, + receipts: Vec<&near_lake_framework::near_indexer_primitives::views::ReceiptView>, block_timestamp: u64, ) -> anyhow::Result<()> { let receipt_actions: Vec = receipts @@ -483,7 +481,7 @@ async fn store_receipt_actions( let receipt_action_actions: Vec = receipts .iter() .filter_map(|receipt| { - if let near_indexer::near_primitives::views::ReceiptEnumView::Action { + if let near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { actions, .. } = &receipt.receipt { @@ -507,7 +505,7 @@ async fn store_receipt_actions( let receipt_action_input_data: Vec = receipts .iter() .filter_map(|receipt| { - if let near_indexer::near_primitives::views::ReceiptEnumView::Action { + if let near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { input_data_ids, .. } = &receipt.receipt @@ -528,7 +526,7 @@ async fn store_receipt_actions( let receipt_action_output_data: Vec = receipts .iter() .filter_map(|receipt| { - if let near_indexer::near_primitives::views::ReceiptEnumView::Action { + if let near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { output_data_receivers, .. } = &receipt.receipt @@ -591,7 +589,7 @@ async fn store_receipt_actions( async fn store_receipt_data( pool: &actix_diesel::Database, - receipts: Vec<&near_indexer::near_primitives::views::ReceiptView>, + receipts: Vec<&near_lake_framework::near_indexer_primitives::views::ReceiptView>, ) -> anyhow::Result<()> { let receipt_data_models: Vec = receipts .iter() diff --git a/src/db_adapters/transactions.rs b/src/db_adapters/transactions.rs index 586818a6..1523e9bd 100644 --- a/src/db_adapters/transactions.rs +++ b/src/db_adapters/transactions.rs @@ -4,18 +4,16 @@ use cached::Cached; use diesel::{ExpressionMethods, PgConnection, QueryDsl}; use futures::future::try_join_all; -use near_indexer::near_primitives; - use crate::models; use crate::schema; /// Saves Transactions to database pub(crate) async fn store_transactions( pool: &actix_diesel::Database, - shards: &[near_indexer::IndexerShard], - block_hash: &near_indexer::near_primitives::hash::CryptoHash, + shards: &[near_lake_framework::near_indexer_primitives::IndexerShard], + block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, block_timestamp: u64, - block_height: near_primitives::types::BlockHeight, + block_height: near_lake_framework::near_indexer_primitives::types::BlockHeight, receipts_cache: crate::ReceiptsCache, ) -> anyhow::Result<()> { let mut tried_to_insert_transactions_count = 0; @@ -30,7 +28,7 @@ pub(crate) async fn store_transactions( .transactions .iter() .enumerate() - .collect::>(), + .collect::>(), &chunk.header.chunk_hash, block_hash, block_timestamp, @@ -74,7 +72,7 @@ pub(crate) async fn store_transactions( .to_string(); !inserted_receipt_ids.contains(converted_into_receipt_id) }) - .collect::>(), + .collect::>(), &chunk.header.chunk_hash, block_hash, block_timestamp, @@ -88,7 +86,7 @@ pub(crate) async fn store_transactions( async fn collect_converted_to_receipt_ids( pool: &actix_diesel::Database, - block_hash: &near_indexer::near_primitives::hash::CryptoHash, + block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, ) -> anyhow::Result> { Ok(schema::transactions::table .select(schema::transactions::dsl::converted_into_receipt_id) @@ -100,9 +98,9 @@ async fn collect_converted_to_receipt_ids( async fn store_chunk_transactions( pool: &actix_diesel::Database, - transactions: Vec<(usize, &near_indexer::IndexerTransactionWithOutcome)>, - chunk_hash: &near_indexer::near_primitives::hash::CryptoHash, - block_hash: &near_indexer::near_primitives::hash::CryptoHash, + transactions: Vec<(usize, &near_lake_framework::near_indexer_primitives::IndexerTransactionWithOutcome)>, + chunk_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, + block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, block_timestamp: u64, // hack for supporting duplicated transaction hashes. Empty for most of transactions transaction_hash_suffix: &str, diff --git a/src/main.rs b/src/main.rs index 9b5fd36e..fc93fd77 100644 --- a/src/main.rs +++ b/src/main.rs @@ -45,7 +45,7 @@ pub type ReceiptsCache = async fn handle_message( pool: &actix_diesel::Database, - streamer_message: near_indexer::StreamerMessage, + streamer_message: near_lake_framework::near_indexer_primitives::StreamerMessage, strict_mode: bool, receipts_cache: ReceiptsCache, ) -> anyhow::Result<()> { @@ -154,7 +154,7 @@ async fn handle_message( } async fn listen_blocks( - stream: mpsc::Receiver, + stream: mpsc::Receiver, pool: Database, concurrency: std::num::NonZeroU16, strict_mode: bool, @@ -214,7 +214,7 @@ async fn listen_blocks( tokio::time::sleep(std::time::Duration::from_secs(7)).await; } -fn main() { +fn main() -> anyhow::Result<()> { // We use it to automatically search the for root certificates to perform HTTPS calls // (sending telemetry and downloading genesis) openssl_probe::init_ssl_cert_env_vars(); @@ -262,14 +262,14 @@ fn main() { .with_writer(std::io::stderr) .init(); - let config = near_lake_framework::LakeConfig { - s3_bucket_name: opts.s3_bucket_name.clone(), - s3_region_name: opts.s3_region_name.clone(), - start_block_height: opts.start_block_height, // want to start from the freshest - }; + let config = near_lake_framework::LakeConfigBuilder::default() + .s3_bucket_name(opts.s3_bucket_name.clone()) + .s3_region_name(opts.s3_region_name.clone()) + .start_block_height(opts.start_block_height) // want to start from the freshest + .build()?; let system = actix::System::new(); system.block_on(async move { - let stream = near_lake_framework::streamer(config); + let (lake_handle, stream) = near_lake_framework::streamer(config); listen_blocks( stream, @@ -281,7 +281,15 @@ fn main() { .await; actix::System::current().stop(); - }); - system.run().unwrap(); + // propagate errors from the sender + match lake_handle.await { + Ok(Ok(())) => Ok(()), + Ok(Err(e)) => Err(e), + Err(e) => Err(anyhow::Error::from(e)), // JoinError + } + })?; + + system.run()?; + Ok(()) } diff --git a/src/models/access_keys.rs b/src/models/access_keys.rs index 89dca675..a6bf8ab6 100644 --- a/src/models/access_keys.rs +++ b/src/models/access_keys.rs @@ -17,10 +17,10 @@ pub struct AccessKey { impl AccessKey { pub fn from_action_view( public_key: &near_crypto::PublicKey, - account_id: &near_indexer::near_primitives::types::AccountId, - access_key: &near_indexer::near_primitives::views::AccessKeyView, - create_by_receipt_id: &near_indexer::near_primitives::hash::CryptoHash, - last_update_block_height: near_indexer::near_primitives::types::BlockHeight, + account_id: &near_lake_framework::near_indexer_primitives::types::AccountId, + access_key: &near_lake_framework::near_indexer_primitives::views::AccessKeyView, + create_by_receipt_id: &near_lake_framework::near_indexer_primitives::CryptoHash, + last_update_block_height: near_lake_framework::near_indexer_primitives::types::BlockHeight, ) -> Self { Self { public_key: public_key.to_string(), @@ -34,9 +34,9 @@ impl AccessKey { pub fn from_genesis( public_key: &near_crypto::PublicKey, - account_id: &near_indexer::near_primitives::types::AccountId, - access_key: &near_indexer::near_primitives::account::AccessKey, - last_update_block_height: near_indexer::near_primitives::types::BlockHeight, + account_id: &near_lake_framework::near_indexer_primitives::types::AccountId, + access_key: &near_primitives::account::AccessKey, + last_update_block_height: near_lake_framework::near_indexer_primitives::types::BlockHeight, ) -> Self { Self { public_key: public_key.to_string(), diff --git a/src/models/account_changes.rs b/src/models/account_changes.rs index 16d94a72..c69c8ab1 100644 --- a/src/models/account_changes.rs +++ b/src/models/account_changes.rs @@ -22,23 +22,23 @@ pub struct AccountChange { impl AccountChange { pub fn from_state_change_with_cause( - state_change_with_cause: &near_indexer::near_primitives::views::StateChangeWithCauseView, - changed_in_block_hash: &near_indexer::near_primitives::hash::CryptoHash, + state_change_with_cause: &near_lake_framework::near_indexer_primitives::views::StateChangeWithCauseView, + changed_in_block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, changed_in_block_timestamp: u64, index_in_block: i32, ) -> Option { - let near_indexer::near_primitives::views::StateChangeWithCauseView { cause, value } = + let near_lake_framework::near_indexer_primitives::views::StateChangeWithCauseView { cause, value } = state_change_with_cause; let (account_id, account): ( String, - Option<&near_indexer::near_primitives::views::AccountView>, + Option<&near_lake_framework::near_indexer_primitives::views::AccountView>, ) = match value { - near_indexer::near_primitives::views::StateChangeValueView::AccountUpdate { + near_lake_framework::near_indexer_primitives::views::StateChangeValueView::AccountUpdate { account_id, account, } => (account_id.to_string(), Some(account)), - near_indexer::near_primitives::views::StateChangeValueView::AccountDeletion { + near_lake_framework::near_indexer_primitives::views::StateChangeValueView::AccountDeletion { account_id, } => (account_id.to_string(), None), _ => return None, @@ -48,16 +48,16 @@ impl AccountChange { affected_account_id: account_id, changed_in_block_timestamp: changed_in_block_timestamp.into(), changed_in_block_hash: changed_in_block_hash.to_string(), - caused_by_transaction_hash: if let near_indexer::near_primitives::views::StateChangeCauseView::TransactionProcessing {tx_hash } = cause { + caused_by_transaction_hash: if let near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::TransactionProcessing {tx_hash } = cause { Some(tx_hash.to_string()) } else { None }, caused_by_receipt_id: match cause { - near_indexer::near_primitives::views::StateChangeCauseView::ActionReceiptProcessingStarted { receipt_hash} => Some(receipt_hash.to_string()), - near_indexer::near_primitives::views::StateChangeCauseView::ActionReceiptGasReward { receipt_hash } => Some(receipt_hash.to_string()), - near_indexer::near_primitives::views::StateChangeCauseView::ReceiptProcessing { receipt_hash } => Some(receipt_hash.to_string()), - near_indexer::near_primitives::views::StateChangeCauseView::PostponedReceipt { receipt_hash } => Some(receipt_hash.to_string()), + near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::ActionReceiptProcessingStarted { receipt_hash} => Some(receipt_hash.to_string()), + near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::ActionReceiptGasReward { receipt_hash } => Some(receipt_hash.to_string()), + near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::ReceiptProcessing { receipt_hash } => Some(receipt_hash.to_string()), + near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::PostponedReceipt { receipt_hash } => Some(receipt_hash.to_string()), _ => None, }, update_reason: cause.into(), diff --git a/src/models/accounts.rs b/src/models/accounts.rs index dbda5a7c..6a75aff0 100644 --- a/src/models/accounts.rs +++ b/src/models/accounts.rs @@ -14,9 +14,9 @@ pub struct Account { impl Account { pub fn new_from_receipt( - account_id: &near_indexer::near_primitives::types::AccountId, - created_by_receipt_id: &near_indexer::near_primitives::hash::CryptoHash, - last_update_block_height: near_indexer::near_primitives::types::BlockHeight, + account_id: &near_lake_framework::near_indexer_primitives::types::AccountId, + created_by_receipt_id: &near_lake_framework::near_indexer_primitives::CryptoHash, + last_update_block_height: near_lake_framework::near_indexer_primitives::types::BlockHeight, ) -> Self { Self { account_id: account_id.to_string(), @@ -27,8 +27,8 @@ impl Account { } pub fn new_from_genesis( - account_id: &near_indexer::near_primitives::types::AccountId, - last_update_block_height: near_indexer::near_primitives::types::BlockHeight, + account_id: &near_lake_framework::near_indexer_primitives::types::AccountId, + last_update_block_height: near_lake_framework::near_indexer_primitives::types::BlockHeight, ) -> Self { Self { account_id: account_id.to_string(), diff --git a/src/models/blocks.rs b/src/models/blocks.rs index 257c306b..68ff55dc 100644 --- a/src/models/blocks.rs +++ b/src/models/blocks.rs @@ -2,8 +2,6 @@ use std::str::FromStr; use bigdecimal::BigDecimal; -use near_indexer::near_primitives; - use crate::schema; use schema::blocks; @@ -18,8 +16,8 @@ pub struct Block { pub author_account_id: String, } -impl From<&near_primitives::views::BlockView> for Block { - fn from(block_view: &near_primitives::views::BlockView) -> Self { +impl From<&near_lake_framework::near_indexer_primitives::views::BlockView> for Block { + fn from(block_view: &near_lake_framework::near_indexer_primitives::views::BlockView) -> Self { Self { block_height: block_view.header.height.into(), block_hash: block_view.header.hash.to_string(), diff --git a/src/models/chunks.rs b/src/models/chunks.rs index ba90f03b..c65ba835 100644 --- a/src/models/chunks.rs +++ b/src/models/chunks.rs @@ -16,8 +16,8 @@ pub struct Chunk { impl Chunk { pub fn from_chunk_view( - chunk_view: &near_indexer::IndexerChunkView, - block_hash: &near_indexer::near_primitives::hash::CryptoHash, + chunk_view: &near_lake_framework::near_indexer_primitives::IndexerChunkView, + block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, ) -> Self { Self { included_in_block_hash: block_hash.to_string(), diff --git a/src/models/enums.rs b/src/models/enums.rs index 71623f35..ccc32b76 100644 --- a/src/models/enums.rs +++ b/src/models/enums.rs @@ -9,11 +9,11 @@ pub enum ReceiptKind { Data, } -impl From<&near_indexer::near_primitives::views::ReceiptEnumView> for ReceiptKind { - fn from(receipt_enum_view: &near_indexer::near_primitives::views::ReceiptEnumView) -> Self { +impl From<&near_lake_framework::near_indexer_primitives::views::ReceiptEnumView> for ReceiptKind { + fn from(receipt_enum_view: &near_lake_framework::near_indexer_primitives::views::ReceiptEnumView) -> Self { match receipt_enum_view { - near_indexer::near_primitives::views::ReceiptEnumView::Action { .. } => Self::Action, - near_indexer::near_primitives::views::ReceiptEnumView::Data { .. } => Self::Data, + near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { .. } => Self::Action, + near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Data { .. } => Self::Data, } } } @@ -44,15 +44,15 @@ pub enum ExecutionOutcomeStatus { SuccessReceiptId, } -impl From for ExecutionOutcomeStatus { - fn from(status: near_indexer::near_primitives::views::ExecutionStatusView) -> Self { +impl From for ExecutionOutcomeStatus { + fn from(status: near_lake_framework::near_indexer_primitives::views::ExecutionStatusView) -> Self { match status { - near_indexer::near_primitives::views::ExecutionStatusView::Unknown => Self::Unknown, - near_indexer::near_primitives::views::ExecutionStatusView::Failure(_) => Self::Failure, - near_indexer::near_primitives::views::ExecutionStatusView::SuccessValue(_) => { + near_lake_framework::near_indexer_primitives::views::ExecutionStatusView::Unknown => Self::Unknown, + near_lake_framework::near_indexer_primitives::views::ExecutionStatusView::Failure(_) => Self::Failure, + near_lake_framework::near_indexer_primitives::views::ExecutionStatusView::SuccessValue(_) => { Self::SuccessValue } - near_indexer::near_primitives::views::ExecutionStatusView::SuccessReceiptId(_) => { + near_lake_framework::near_indexer_primitives::views::ExecutionStatusView::SuccessReceiptId(_) => { Self::SuccessReceiptId } } @@ -70,26 +70,26 @@ pub enum AccessKeyPermission { FunctionCall, } -impl From<&near_indexer::near_primitives::views::AccessKeyPermissionView> for AccessKeyPermission { - fn from(item: &near_indexer::near_primitives::views::AccessKeyPermissionView) -> Self { +impl From<&near_lake_framework::near_indexer_primitives::views::AccessKeyPermissionView> for AccessKeyPermission { + fn from(item: &near_lake_framework::near_indexer_primitives::views::AccessKeyPermissionView) -> Self { match item { - near_indexer::near_primitives::views::AccessKeyPermissionView::FunctionCall { + near_lake_framework::near_indexer_primitives::views::AccessKeyPermissionView::FunctionCall { .. } => Self::FunctionCall, - near_indexer::near_primitives::views::AccessKeyPermissionView::FullAccess => { + near_lake_framework::near_indexer_primitives::views::AccessKeyPermissionView::FullAccess => { Self::FullAccess } } } } -impl From<&near_indexer::near_primitives::account::AccessKeyPermission> for AccessKeyPermission { - fn from(item: &near_indexer::near_primitives::account::AccessKeyPermission) -> Self { +impl From<&near_primitives::account::AccessKeyPermission> for AccessKeyPermission { + fn from(item: &near_primitives::account::AccessKeyPermission) -> Self { match item { - near_indexer::near_primitives::account::AccessKeyPermission::FunctionCall { + near_primitives::account::AccessKeyPermission::FunctionCall { .. } => Self::FunctionCall, - near_indexer::near_primitives::account::AccessKeyPermission::FullAccess => { + near_primitives::account::AccessKeyPermission::FullAccess => { Self::FullAccess } } @@ -112,21 +112,21 @@ pub enum StateChangeReasonKind { Resharding, } -impl From<&near_indexer::near_primitives::views::StateChangeCauseView> for StateChangeReasonKind { +impl From<&near_lake_framework::near_indexer_primitives::views::StateChangeCauseView> for StateChangeReasonKind { fn from( - state_change_cause_view: &near_indexer::near_primitives::views::StateChangeCauseView, + state_change_cause_view: &near_lake_framework::near_indexer_primitives::views::StateChangeCauseView, ) -> Self { match state_change_cause_view { - near_indexer::near_primitives::views::StateChangeCauseView::TransactionProcessing { .. } => Self::TransactionProcessing, - near_indexer::near_primitives::views::StateChangeCauseView::ActionReceiptProcessingStarted { .. } => Self::ActionReceiptProcessingStarted, - near_indexer::near_primitives::views::StateChangeCauseView::ActionReceiptGasReward { .. } => Self::ActionReceiptGasReward, - near_indexer::near_primitives::views::StateChangeCauseView::ReceiptProcessing { .. } => Self::ReceiptProcessing, - near_indexer::near_primitives::views::StateChangeCauseView::PostponedReceipt { .. } => Self::PostponedReceipt, - near_indexer::near_primitives::views::StateChangeCauseView::UpdatedDelayedReceipts { .. } => Self::UpdatedDelayedReceipts, - near_indexer::near_primitives::views::StateChangeCauseView::ValidatorAccountsUpdate { .. } => Self::ValidatorAccountsUpdate, - near_indexer::near_primitives::views::StateChangeCauseView::Migration { .. } => Self::Migration, - near_indexer::near_primitives::views::StateChangeCauseView::Resharding { .. } => Self::Resharding, - near_indexer::near_primitives::views::StateChangeCauseView::NotWritableToDisk | near_indexer::near_primitives::views::StateChangeCauseView::InitialState => panic!("Unexpected variant {:?} received", state_change_cause_view), + near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::TransactionProcessing { .. } => Self::TransactionProcessing, + near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::ActionReceiptProcessingStarted { .. } => Self::ActionReceiptProcessingStarted, + near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::ActionReceiptGasReward { .. } => Self::ActionReceiptGasReward, + near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::ReceiptProcessing { .. } => Self::ReceiptProcessing, + near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::PostponedReceipt { .. } => Self::PostponedReceipt, + near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::UpdatedDelayedReceipts { .. } => Self::UpdatedDelayedReceipts, + near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::ValidatorAccountsUpdate { .. } => Self::ValidatorAccountsUpdate, + near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::Migration { .. } => Self::Migration, + near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::Resharding { .. } => Self::Resharding, + near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::NotWritableToDisk | near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::InitialState => panic!("Unexpected variant {:?} received", state_change_cause_view), } } } diff --git a/src/models/execution_outcomes.rs b/src/models/execution_outcomes.rs index bac7f79b..5e1599a3 100644 --- a/src/models/execution_outcomes.rs +++ b/src/models/execution_outcomes.rs @@ -22,7 +22,7 @@ pub struct ExecutionOutcome { impl ExecutionOutcome { pub fn from_execution_outcome( - execution_outcome: &near_indexer::near_primitives::views::ExecutionOutcomeWithIdView, + execution_outcome: &near_lake_framework::near_indexer_primitives::views::ExecutionOutcomeWithIdView, index_in_chunk: i32, executed_in_block_timestamp: u64, shard_id: u64, diff --git a/src/models/receipts.rs b/src/models/receipts.rs index 545505fd..0f02c5c5 100644 --- a/src/models/receipts.rs +++ b/src/models/receipts.rs @@ -3,7 +3,7 @@ use std::str::FromStr; use bigdecimal::BigDecimal; -use near_indexer::near_primitives::views::DataReceiverView; +use near_lake_framework::near_indexer_primitives::views::DataReceiverView; use crate::models::enums::{ActionKind, ReceiptKind}; use crate::schema; @@ -27,10 +27,10 @@ pub struct Receipt { impl Receipt { pub fn from_receipt_view( - receipt: &near_indexer::near_primitives::views::ReceiptView, - block_hash: &near_indexer::near_primitives::hash::CryptoHash, + receipt: &near_lake_framework::near_indexer_primitives::views::ReceiptView, + block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, transaction_hash: &str, - chunk_hash: &near_indexer::near_primitives::hash::CryptoHash, + chunk_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, index_in_chunk: i32, block_timestamp: u64, ) -> Self { @@ -56,13 +56,13 @@ pub struct DataReceipt { pub data: Option>, } -impl TryFrom<&near_indexer::near_primitives::views::ReceiptView> for DataReceipt { +impl TryFrom<&near_lake_framework::near_indexer_primitives::views::ReceiptView> for DataReceipt { type Error = &'static str; fn try_from( - receipt_view: &near_indexer::near_primitives::views::ReceiptView, + receipt_view: &near_lake_framework::near_indexer_primitives::views::ReceiptView, ) -> Result { - if let near_indexer::near_primitives::views::ReceiptEnumView::Data { data_id, data } = + if let near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Data { data_id, data } = &receipt_view.receipt { Ok(Self { @@ -85,13 +85,13 @@ pub struct ActionReceipt { pub gas_price: BigDecimal, } -impl TryFrom<&near_indexer::near_primitives::views::ReceiptView> for ActionReceipt { +impl TryFrom<&near_lake_framework::near_indexer_primitives::views::ReceiptView> for ActionReceipt { type Error = &'static str; fn try_from( - receipt_view: &near_indexer::near_primitives::views::ReceiptView, + receipt_view: &near_lake_framework::near_indexer_primitives::views::ReceiptView, ) -> Result { - if let near_indexer::near_primitives::views::ReceiptEnumView::Action { + if let near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { signer_id, signer_public_key, gas_price, @@ -127,7 +127,7 @@ impl ActionReceiptAction { pub fn from_action_view( receipt_id: String, index: i32, - action_view: &near_indexer::near_primitives::views::ActionView, + action_view: &near_lake_framework::near_indexer_primitives::views::ActionView, predecessor_account_id: String, receiver_account_id: String, block_timestamp: u64, diff --git a/src/models/serializers.rs b/src/models/serializers.rs index b425cb11..a448ee3e 100644 --- a/src/models/serializers.rs +++ b/src/models/serializers.rs @@ -1,8 +1,8 @@ use serde::{Deserialize, Serialize}; use serde_json::json; -use near_indexer::near_primitives::serialize::option_u128_dec_format; -use near_indexer::near_primitives::views::ActionView; +use near_primitives::serialize::option_u128_dec_format; +use near_lake_framework::near_indexer_primitives::views::ActionView; use crate::models::enums::ActionKind; @@ -10,12 +10,12 @@ use crate::models::enums::ActionKind; /// to change serde parameters of serialization. #[derive(Serialize, Deserialize, Debug, Clone)] pub(crate) struct AccessKeyView { - pub nonce: near_indexer::near_primitives::types::Nonce, + pub nonce: near_lake_framework::near_indexer_primitives::types::Nonce, pub permission: AccessKeyPermissionView, } -impl From<&near_indexer::near_primitives::views::AccessKeyView> for AccessKeyView { - fn from(access_key_view: &near_indexer::near_primitives::views::AccessKeyView) -> Self { +impl From<&near_lake_framework::near_indexer_primitives::views::AccessKeyView> for AccessKeyView { + fn from(access_key_view: &near_lake_framework::near_indexer_primitives::views::AccessKeyView) -> Self { Self { nonce: access_key_view.nonce, permission: access_key_view.permission.clone().into(), @@ -34,22 +34,22 @@ impl From<&near_indexer::near_primitives::views::AccessKeyView> for AccessKeyVie pub(crate) enum AccessKeyPermissionView { FunctionCall { #[serde(with = "option_u128_dec_format")] - allowance: Option, + allowance: Option, receiver_id: String, method_names: Vec, }, FullAccess, } -impl From +impl From for AccessKeyPermissionView { - fn from(permission: near_indexer::near_primitives::views::AccessKeyPermissionView) -> Self { + fn from(permission: near_lake_framework::near_indexer_primitives::views::AccessKeyPermissionView) -> Self { match permission { - near_indexer::near_primitives::views::AccessKeyPermissionView::FullAccess => { + near_lake_framework::near_indexer_primitives::views::AccessKeyPermissionView::FullAccess => { Self::FullAccess } - near_indexer::near_primitives::views::AccessKeyPermissionView::FunctionCall { + near_lake_framework::near_indexer_primitives::views::AccessKeyPermissionView::FunctionCall { allowance, receiver_id, method_names, @@ -66,7 +66,7 @@ impl From } pub(crate) fn extract_action_type_and_value_from_action_view( - action_view: &near_indexer::near_primitives::views::ActionView, + action_view: &near_lake_framework::near_indexer_primitives::views::ActionView, ) -> (crate::models::enums::ActionKind, serde_json::Value) { match action_view { ActionView::CreateAccount => (ActionKind::CreateAccount, json!({})), diff --git a/src/models/transactions.rs b/src/models/transactions.rs index edcd9423..5ed99a8d 100644 --- a/src/models/transactions.rs +++ b/src/models/transactions.rs @@ -26,12 +26,12 @@ pub struct Transaction { impl Transaction { pub fn from_indexer_transaction( - tx: &near_indexer::IndexerTransactionWithOutcome, + tx: &near_lake_framework::near_indexer_primitives::IndexerTransactionWithOutcome, // hack for supporting duplicated transaction hashes transaction_hash: &str, converted_into_receipt_id: &str, - block_hash: &near_indexer::near_primitives::hash::CryptoHash, - chunk_hash: &near_indexer::near_primitives::hash::CryptoHash, + block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, + chunk_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, block_timestamp: u64, index_in_chunk: i32, ) -> Self { @@ -74,7 +74,7 @@ impl TransactionAction { pub fn from_action_view( transaction_hash: String, index: i32, - action_view: &near_indexer::near_primitives::views::ActionView, + action_view: &near_lake_framework::near_indexer_primitives::views::ActionView, ) -> Self { let (action_kind, args) = crate::models::extract_action_type_and_value_from_action_view(action_view); From 0e898711989821760fa6b3435097bf2347b5c444 Mon Sep 17 00:00:00 2001 From: Bohdan Khorolets Date: Fri, 19 Aug 2022 13:41:04 +0300 Subject: [PATCH 05/42] refactor: Upgrade to NEAR Lake Framework 0.5.1 (#2) * refactor: Upgrade to NEAR Lake Framework 0.5.1 * remove .buildkite, add GH Actions workflows, drop redis requirement, fix clippy warnings * change the version to be relative to the original indexer-for-explorer --- .buildkite/pipeline.yml | 13 - .dockerignore | 6 - .github/workflows/release.yml | 117 +++++ .github/workflows/test.yml | 85 ++++ CHANGELOG.md | 184 +------ Cargo.lock | 464 +++++++++++++++++- Cargo.toml | 11 +- Cross.toml | 6 + Dockerfile | 77 --- README.md | 163 ++---- cross/x86_64-unknown-linux-gnu/Dockerfile | 14 + src/aggregated/account_details.rs | 66 --- src/aggregated/circulating_supply/lockup.rs | 207 -------- .../circulating_supply/lockup_types.rs | 211 -------- src/aggregated/circulating_supply/mod.rs | 247 ---------- src/aggregated/mod.rs | 16 - src/configs.rs | 188 ++++++- src/db_adapters/access_keys.rs | 30 +- src/db_adapters/accounts.rs | 95 +--- .../aggregated/circulating_supply.rs | 61 --- src/db_adapters/aggregated/mod.rs | 1 - src/db_adapters/assets/events.rs | 2 +- .../assets/fungible_token_events.rs | 2 +- .../assets/non_fungible_token_events.rs | 2 +- src/db_adapters/blocks.rs | 20 +- src/db_adapters/genesis.rs | 129 ----- src/db_adapters/mod.rs | 2 - src/db_adapters/receipts.rs | 47 +- src/db_adapters/transactions.rs | 18 +- src/main.rs | 163 ++---- src/models/account_changes.rs | 6 +- src/models/enums.rs | 40 +- src/models/mod.rs | 16 +- src/models/receipts.rs | 6 +- src/models/serializers.rs | 10 +- 35 files changed, 1021 insertions(+), 1704 deletions(-) delete mode 100644 .buildkite/pipeline.yml delete mode 100644 .dockerignore create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/test.yml create mode 100644 Cross.toml delete mode 100644 Dockerfile create mode 100644 cross/x86_64-unknown-linux-gnu/Dockerfile delete mode 100644 src/aggregated/account_details.rs delete mode 100644 src/aggregated/circulating_supply/lockup.rs delete mode 100644 src/aggregated/circulating_supply/lockup_types.rs delete mode 100644 src/aggregated/circulating_supply/mod.rs delete mode 100644 src/aggregated/mod.rs delete mode 100644 src/db_adapters/aggregated/circulating_supply.rs delete mode 100644 src/db_adapters/aggregated/mod.rs delete mode 100644 src/db_adapters/genesis.rs diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml deleted file mode 100644 index 889b1df5..00000000 --- a/.buildkite/pipeline.yml +++ /dev/null @@ -1,13 +0,0 @@ -steps: - - command: "source ~/.cargo/env && cargo fmt --all -- --check" - label: "Formatter" - - - command: "source ~/.cargo/env && cargo clippy" - label: "Linter" - - - command: "source ~/.cargo/env && cargo test" - label: "Tests" - - - command: "fossa init && fossa analyze --server-scan && fossa test" - label: "Fossa" - branches: "master" diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 9489c90d..00000000 --- a/.dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -.git -.gitignore -.dockerignore -Dockerfile -target -*.swp diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..831ee039 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,117 @@ +name: release +on: + push: + # Enable when testing release infrastructure on a branch. + # branches: + # - ci/release-check + tags: + - "[0-9]+.[0-9]+.[0-9]+" + - "[0-9]+.[0-9]+.[0-9]+-*" +jobs: + create-release: + name: create-release + runs-on: ubuntu-latest + # env: + # Set to force version number, e.g., when no tag exists. + # NEAR_LAKE_VERSION: TEST-0.0.0 + outputs: + upload_url: ${{ steps.release.outputs.upload_url }} + indexer_version: ${{ env.INDEXER_VERSION }} + steps: + - name: Get the release version from the tag + shell: bash + if: env.INDEXER_VERSION == '' + run: | + # Apparently, this is the right way to get a tag name. Really? + # + # See: https://github.community/t5/GitHub-Actions/How-to-get-just-the-tag-name/m-p/32167/highlight/true#M1027 + echo "INDEXER_VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV + echo "version is: ${{ env.INDEXER_VERSION }}" + - name: Create GitHub release + id: release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ env.INDEXER_VERSION }} + release_name: ${{ env.INDEXER_VERSION }} + + build-release: + name: build-release + needs: ['create-release'] + runs-on: ${{ matrix.os }} + env: + # For some builds, we use cross to test on 32-bit and big-endian + # systems. + CARGO: cargo + # When CARGO is set to CROSS, this is set to `--target matrix.target`. + TARGET_FLAGS: "" + # When CARGO is set to CROSS, TARGET_DIR includes matrix.target. + TARGET_DIR: ./target + # For some builds, we disable ledger support + FEATURES_FLAGS: + # Emit backtraces on panics. + RUST_BACKTRACE: 1 + # Build static releases with PCRE2. + PCRE2_SYS_STATIC: 1 + strategy: + fail-fast: false + matrix: + include: + - build: linux-x86_64 + os: ubuntu-18.04 + rust: stable + target: x86_64-unknown-linux-gnu + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + fetch-depth: 1 + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ matrix.rust }} + profile: minimal + override: true + target: ${{ matrix.target }} + + - name: Use Cross + run: | + cargo install cross + echo "CARGO=cross" >> $GITHUB_ENV + echo "TARGET_FLAGS=--target ${{ matrix.target }}" >> $GITHUB_ENV + echo "TARGET_DIR=./target/${{ matrix.target }}" >> $GITHUB_ENV + + - name: Build cross environment + shell: bash + run: | + if [ -d "./cross/${{ matrix.target }}" ]; then + docker build --tag "cross:${{ matrix.target }}" "./cross/${{ matrix.target }}" + fi + + - name: Show command used for Cargo + run: | + echo "cargo command is: ${{ env.CARGO }}" + echo "target flag is: ${{ env.TARGET_FLAGS }}" + echo "target dir is: ${{ env.TARGET_DIR }}" + + - name: Build release binary + run: ${{ env.CARGO }} build --verbose --release ${{ env.TARGET_FLAGS }} ${{ env.FEATURES_FLAGS }} + + - name: Strip release binary (linux and macos) + run: | + strip "${{ env.TARGET_DIR }}/release/indexer-explorer-lake" + cp "${{ env.TARGET_DIR }}/release/indexer-explorer-lake" indexer-explorer-lake + echo "ASSET=indexer-explorer-lake" >> $GITHUB_ENV + + - name: Upload release archive + uses: actions/upload-release-asset@v1.0.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create-release.outputs.upload_url }} + asset_path: ${{ env.ASSET }} + asset_name: ${{ env.ASSET }} + asset_content_type: application/octet-stream diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..3a44e32e --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,85 @@ +name: ci +on: + pull_request: + branches: [main] + push: + branches: [main] + # schedule: + # - cron: '00 01 * * *' +jobs: + test: + name: test + env: + # For some builds, we use cross to test on 32-bit and big-endian + # systems. + CARGO: cargo + # When CARGO is set to CROSS, this is set to `--target matrix.target`. + TARGET_FLAGS: "" + # When CARGO is set to CROSS, TARGET_DIR includes matrix.target. + TARGET_DIR: ./target + # For some builds, we disable ledger support + FEATURES_FLAGS: + # Emit backtraces on panics. + RUST_BACKTRACE: 1 + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - build: linux-x86_64 + os: ubuntu-18.04 + rust: stable + target: x86_64-unknown-linux-gnu + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ matrix.rust }} + profile: minimal + override: true + + - name: Use Cross + run: | + cargo install cross + echo "CARGO=cross" >> $GITHUB_ENV + echo "TARGET_FLAGS=--target ${{ matrix.target }}" >> $GITHUB_ENV + echo "TARGET_DIR=./target/${{ matrix.target }}" >> $GITHUB_ENV + + - name: Build cross environment + shell: bash + run: | + if [ -d "./cross/${{ matrix.target }}" ]; then + docker build --tag "cross:${{ matrix.target }}" "./cross/${{ matrix.target }}" + fi + + - name: Show command used for Cargo + run: | + echo "cargo command is: ${{ env.CARGO }}" + echo "target flag is: ${{ env.TARGET_FLAGS }}" + + - name: Build indexer-explorer-lake and all crates + run: ${{ env.CARGO }} build --verbose --all ${{ env.TARGET_FLAGS }} ${{ env.FEATURES_FLAGS }} + + - name: Run tests + run: ${{ env.CARGO }} test --verbose --all ${{ env.TARGET_FLAGS }} ${{ env.FEATURES_FLAGS }} + + rustfmt: + name: rustfmt + runs-on: ubuntu-18.04 + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + profile: minimal + components: rustfmt + - name: Check formatting + run: | + cargo fmt --all -- --check diff --git a/CHANGELOG.md b/CHANGELOG.md index 65f8e62b..55753126 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,187 +1,5 @@ # Changelog -## 0.10.12 - -* Upgrade `nearcore` to `1.25.0-rc.1` - -## 0.10.11 - -* Upgrade `nearcore` to `1.24.0-rc.4` - -## 0.10.10 - -* Upgrade `nearcore` to `1.24.0-rc.3` - -## 0.10.9 - -* Add [FT events](https://nomicon.io/Standards/FungibleToken/Event.html) support: `assets__fungible_token_events` table stores the information about FT `mint`, `transfer`, `burn` events -* Add index on `action_receipt_actions` table. Applying migration could take some time -* Introduce `ReceiptsCache` to optimize the way we match `Receipts` with parent `Transactions` -* Upgrade nearcore to `1.24.0-rc.2` - -## 0.10.8 - -* Upgrade nearcore to `1.24.0-rc.1` - -## 0.10.7 - -* Upgrade `nearcore` to 1.23.1 -* Optimized the latency of putting the data into database - -## 0.10.6 - -* Upgrade `nearcore` to 1.23.1-rc.2 - - -## 0.10.5 - -* Upgrade `nearcore` to 1.23.1-rc.1 - -## 0.10.4 - -* Upgrade `nearcore` to 1.23.0 -* Handle transaction hash collisions (issue #84) - -## 0.10.3 - -* Upgrade `nearcore` to 1.22.0 -* Add [NFT events](https://nomicon.io/Standards/NonFungibleToken/Event.html) support: `assets__non_fungible_token_events` table stores the information about NFT `mint`, `transfer`, `burn` events - -## 0.10.2 - -- Change the retry logic. Make indexer fail with error if is has retried for 5 min -- Upgrade `nearcore` to 1.22.0 - -## 0.10.1 - -* Upgrade `nearcore` to 1.21.1 - ## 0.10.0 -* Drop `--allow-missing-relations-in-first-blocks` flag -* Introduce `--non-strict-mode` which does the same as `--allow-missing-relations-in-first-blocks` flag did but endlessly -* Add `--stop-after-number-of-blocks ` flag to plan Indexer for Explorer to stop once it indexed the provided `` of blocks. May be useful for debug or maintenance purposes. - -## Breaking changes - -* The flag `--allow-missing-relations-in-first-blocks` is not available anymore in favor of `--non-strict-mode` flag - -## 0.9.3 - -* Escape `args_json` on the fly to avoid null-byte issues -* Upgrade to NEAR Indexer Framework `0.10.0` -* Refactor the storing Accounts and AccessKeys from genesis to optimize memory usage -* Improve logging to better understand what Indexer for Explorer is doing on the start - -## 0.9.2 (hotfix) - -* Change `receiver_id` field type to `String` to be compatible with `nearcore` `AccessKeyPermissionView` struct (it caused problems during AccessKey serialization) - -## 0.9.1 - -* Upgrade `nearcore` to 1.21.0 (rc1) - -## 0.9.0 (nearcore dependency contains bug) - -* Upgrade `nearcore` to 1.21.0 - -## Breaking changes - -* `init` command has changed according to changes in `nearcore`: - - `download` argument has been replaced with `download_config` and `download_genesis` - - `boot_nodes` argument was added - - `download_config_url` was added -* `AccountId` from `near-primitives` was replaced with separate crate `near-account-id` and it is no longer an alias for `String` - - All the fields related to an account id have type `near_account_id::AccountId` - -## 0.8.0 - -* Background calculation of circulating supply and storing it to DB -* Improvements on some tables (add indexes, simplify sorting etc.) - -## 0.7.1 - -* Update nearcore version to 1.20.0-rc.2 - -## 0.7.0 - -* Handle null-bytes in `AddKey` actions -* Update nearcore version to 1.20.0 - -## Breaking change - -`init_configs` function from nearcore has been extended with additional optional parameter `max_gas_burnt_view`. We've extended NEAR Indexer for Explorer `InitConfigArgs` - -## 0.6.9 - -* Add `--concurrency` parameter to adjust the number of simultaneously running asynchronous adapters - -## 0.6.8 - -* Update NEAR Indexer Framework version to 0.9.2 (with optimized delayed receipts tracking system) - -## 0.6.7 - -* Remove duplicates from `account_changes` table by fixing unique index ([see issue #74](https://github.com/near/near-indexer-for-explorer/issues/74)) - -## 0.6.6 (hotfix) - -* Upgrade `nearcore` to 1.19.1 (hotfix) - -## 0.6.5 - -* Update NEAR Indexer Framework version to 0.9.1 (previous contained a bug with processing delayed local receipts) - -## 0.6.4 (contains bug) - -* Fix the overwriting of `created_by_receipt_id` for implicit accounts that may confuse users ([see issue #68 for ref](https://github.com/near/near-indexer-for-explorer/issues/68)) - -## 0.6.3 - -* Denormalize table `action_receipt_actions` in order to speed up some queries by avoid - additional joins -* Extend `extract_action_type_and_value_from_action_view` function to try to parse base64 encoded args - as a JSON object to put them decoded in the function call args `action_receipt_actions.args` additionally - -## 0.6.2 - -* Upgrade `nearcore` dependency to exclude recent updates to runtime which caused a bug ([see for ref](https://github.com/near/nearcore/releases/tag/1.19.0-rc.2)) - -## 0.6.1 - -* Upgrade `nearcore` to support newer protocol version (45) - -## 0.6.0 - -* Upgrade `nearcore` to get NEAR Indexer Framework 0.9.0 -* Corresponding changes into adapters according to changes in `StreamerMessage` structure -* NEAR Indexer for Explorer now uses stable Rust (`rust-toolchain` has been updated accordingly) - -## 0.5.0 - -* Tweak `sync-from-interruption` mode to start syncing from N blocks earlier that actual interruption - -## 0.4.0 - -* Update `nearcore` dependency -* Update underlying dependencies to correspond `nearcore` - -**The way of starting `actix` runtime has changes** - -## 0.3.0 - -* Migrate from `tokio-diesel` to `actix-diesel` (patched by @frol) - -## 0.2.3 - -* Upgrade `nearcore` dependency -* Upgrade some external dependencies (`actix`, `tokio`) - -## 0.2.2 - -* Fill `deleted_by_receipt_id` if `access_key` on owner account deletion - -## 0.2.1 - -* Add `access_key` on transfer to implicit account action -* Upgrade `nearcore` dependency +* Upgrade `near-lake-framework` to `0.5.1` diff --git a/Cargo.lock b/Cargo.lock index 765e7f78..f851fc44 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -146,6 +146,27 @@ dependencies = [ "event-listener", ] +[[package]] +name = "async-stream" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-trait" version = "0.1.53" @@ -1173,6 +1194,15 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +[[package]] +name = "encoding_rs" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "event-listener" version = "2.5.2" @@ -1206,6 +1236,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.0.1" @@ -1224,9 +1269,9 @@ checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" [[package]] name = "futures" -version = "0.3.21" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f73fe65f54d1e12b726f517d3e2135ca3125a437b6d998caf1962961f7172d9e" +checksum = "ab30e97ab6aacfe635fad58f22c2bb06c8b685f7421eb1e064a729e2a5f481fa" dependencies = [ "futures-channel", "futures-core", @@ -1239,9 +1284,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.21" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3083ce4b914124575708913bca19bfe887522d6e2e6d0952943f5eac4a74010" +checksum = "2bfc52cbddcfd745bf1740338492bb0bd83d76c67b445f91c5fb29fae29ecaa1" dependencies = [ "futures-core", "futures-sink", @@ -1249,15 +1294,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.21" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3" +checksum = "d2acedae88d38235936c3922476b10fced7b2b68136f5e3c03c2d5be348a1115" [[package]] name = "futures-executor" -version = "0.3.21" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9420b90cfa29e327d0429f19be13e7ddb68fa1cccb09d65e5706b8c7a749b8a6" +checksum = "1d11aa21b5b587a64682c0094c2bdd4df0076c5324961a40cc3abd7f37930528" dependencies = [ "futures-core", "futures-task", @@ -1266,15 +1311,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.21" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b" +checksum = "93a66fc6d035a26a3ae255a6d2bca35eda63ae4c5512bef54449113f7a1228e5" [[package]] name = "futures-macro" -version = "0.3.21" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512" +checksum = "0db9cce532b0eae2ccf2766ab246f114b56b9cf6d445e00c2549fbc100ca045d" dependencies = [ "proc-macro2", "quote", @@ -1283,21 +1328,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.21" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21163e139fa306126e6eedaf49ecdb4588f939600f0b1e770f4205ee4b7fa868" +checksum = "ca0bae1fe9752cf7fd9b0064c674ae63f97b37bc714d745cbde0afb7ec4e6765" [[package]] name = "futures-task" -version = "0.3.21" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a" +checksum = "842fc63b931f4056a24d59de13fb1272134ce261816e063e634ad0c15cdc5306" [[package]] name = "futures-util" -version = "0.3.21" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a" +checksum = "f0828a5471e340229c11c77ca80017937ce3c58cb788a17e5f1c2d5c485a9577" dependencies = [ "futures-channel", "futures-core", @@ -1485,12 +1530,36 @@ dependencies = [ "webpki", ] +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "ident_case" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "impl-codec" version = "0.5.1" @@ -1512,13 +1581,15 @@ dependencies = [ ] [[package]] -name = "indexer-explorer" -version = "0.10.19" +name = "indexer-explorer-lake" +version = "0.10.0" dependencies = [ "actix", "actix-diesel", "actix-rt", "anyhow", + "aws-sdk-s3", + "aws-types", "base64 0.11.0", "bigdecimal", "borsh 0.7.2", @@ -1532,6 +1603,7 @@ dependencies = [ "hex", "itertools", "near-crypto", + "near-jsonrpc-client", "near-lake-framework", "near-primitives", "near-sdk", @@ -1566,6 +1638,12 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "ipnet" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" + [[package]] name = "itertools" version = "0.10.3" @@ -1663,6 +1741,12 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8452105ba047068f40ff7093dd1d9da90898e63dd61736462e9cdda6a90ad3c3" +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + [[package]] name = "mio" version = "0.8.3" @@ -1675,6 +1759,24 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "native-tls" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "near-account-id" version = "0.14.0" @@ -1685,6 +1787,68 @@ dependencies = [ "serde", ] +[[package]] +name = "near-chain-configs" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3632a1c691603cb32dd9944c95d8eadbf2c09f45abd95350ea6848c649036a0b" +dependencies = [ + "anyhow", + "chrono", + "derive_more", + "near-crypto", + "near-primitives", + "num-rational", + "serde", + "serde_json", + "sha2 0.10.2", + "smart-default", + "tracing", +] + +[[package]] +name = "near-chain-primitives" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a734353027698b21633a49d478e564c61ae0171c32f6912bb8844add15d72ebe" +dependencies = [ + "chrono", + "near-crypto", + "near-primitives", + "thiserror", + "tracing", +] + +[[package]] +name = "near-chunks-primitives" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17f6f22f1ab710731dfba4101f12a99cac120d6af80b99899bd335bb8971477" +dependencies = [ + "near-chain-primitives", + "near-primitives", +] + +[[package]] +name = "near-client-primitives" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1065d86012eeea838661434549f33bb6267c9950fd2aadd2af617fe773def38" +dependencies = [ + "actix", + "chrono", + "near-chain-configs", + "near-chain-primitives", + "near-chunks-primitives", + "near-crypto", + "near-network-primitives", + "near-primitives", + "serde", + "serde_json", + "strum", + "thiserror", +] + [[package]] name = "near-crypto" version = "0.14.0" @@ -1722,13 +1886,51 @@ dependencies = [ "serde_json", ] +[[package]] +name = "near-jsonrpc-client" +version = "0.4.0-beta.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bba462f54bc35289a1013ed3a2ebfa67cc8b12699f81c12dd67687f200c7b871" +dependencies = [ + "borsh 0.9.3", + "lazy_static", + "log", + "near-chain-configs", + "near-crypto", + "near-jsonrpc-primitives", + "near-primitives", + "reqwest", + "serde", + "serde_json", + "thiserror", + "uuid", +] + +[[package]] +name = "near-jsonrpc-primitives" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34a14ee8ca393c0140cb232789259ebc61b13b4cceb177267d0131f50d0eda6c" +dependencies = [ + "near-chain-configs", + "near-client-primitives", + "near-crypto", + "near-primitives", + "near-rpc-error-macro 0.14.0", + "serde", + "serde_json", + "thiserror", + "uuid", +] + [[package]] name = "near-lake-framework" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0316ee594f99e37604a0ec839e0130519e8e42110cfe4049d64d8f4b021c49e" +checksum = "6e64c96682214091519e78f991cb17c1991f0c1b32d2f9bd9977fe71ede4e6e5" dependencies = [ "anyhow", + "async-stream", "aws-config", "aws-sdk-s3", "aws-types", @@ -1742,6 +1944,24 @@ dependencies = [ "tracing", ] +[[package]] +name = "near-network-primitives" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa998a1e70ebf8cf3efa76c4562ef36038cc88b4aee60efb708d14273910357" +dependencies = [ + "actix", + "anyhow", + "borsh 0.9.3", + "chrono", + "near-crypto", + "near-primitives", + "serde", + "strum", + "tokio", + "tracing", +] + [[package]] name = "near-primitives" version = "0.14.0" @@ -2025,12 +2245,51 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +[[package]] +name = "openssl" +version = "0.10.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "618febf65336490dfcf20b73f885f5651a0c89c64c2d4a8c3662585a70bf5bd0" +dependencies = [ + "bitflags", + "cfg-if 1.0.0", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-sys" +version = "0.9.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5f9bd0c2710541a3cda73d6f9ac4f1b240de4ae261065d309dbe73d9dceb42f" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "os_str_bytes" version = "6.0.1" @@ -2136,6 +2395,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" + [[package]] name = "ppv-lite86" version = "0.2.16" @@ -2355,6 +2620,52 @@ version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "reqwest" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75aa69a3f06bbcc66ede33af2af253c6f7a86b1ca0033f60c580a27074fbf92" +dependencies = [ + "base64 0.13.0", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "ring" version = "0.16.20" @@ -2518,6 +2829,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "sha2" version = "0.9.9" @@ -2686,6 +3009,20 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tempfile" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +dependencies = [ + "cfg-if 1.0.0", + "fastrand", + "libc", + "redox_syscall", + "remove_dir_all", + "winapi", +] + [[package]] name = "termcolor" version = "1.1.3" @@ -2751,6 +3088,21 @@ dependencies = [ "num_threads", ] +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + [[package]] name = "tokio" version = "1.18.2" @@ -2761,6 +3113,7 @@ dependencies = [ "libc", "memchr", "mio", + "num_cpus", "once_cell", "parking_lot", "pin-project-lite", @@ -2769,6 +3122,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.22.0" @@ -2955,6 +3318,21 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "unicode-bidi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" + +[[package]] +name = "unicode-normalization" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "854cbdc4f7bc6ae19c820d44abdc3277ac3e1b2b93db20a636825d9322fb60e6" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-segmentation" version = "1.9.0" @@ -2973,12 +3351,33 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "url" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", +] + [[package]] name = "urlencoding" version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68b90931029ab9b034b300b797048cf23723400aa757e8a2bfb9d748102f9821" +[[package]] +name = "uuid" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +dependencies = [ + "getrandom 0.2.6", +] + [[package]] name = "valuable" version = "0.1.0" @@ -3050,6 +3449,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f741de44b75e14c35df886aff5f1eb73aa114fa5d4d00dcd37b5e01259bf3b2" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.80" @@ -3185,6 +3596,15 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] + [[package]] name = "wyz" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index e830e378..160ef076 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] -name = "indexer-explorer" -version = "0.10.19" +name = "indexer-explorer-lake" +version = "0.10.0" authors = ["Near Inc "] edition = "2021" @@ -8,6 +8,8 @@ edition = "2021" actix = "0.13.0" actix-rt = "2.2.0" anyhow = "1.0.51" +aws-types = "0.13.0" +aws-sdk-s3 = "0.13.0" base64 = "0.11" bigdecimal = "=0.1.0" borsh = "0.7.1" @@ -35,6 +37,7 @@ tracing-subscriber = "0.2.4" uint = { version = "0.8.3", default-features = false } actix-diesel = { git = "https://github.com/frol/actix-diesel", rev = "3a001986c89dfabfc3c448d8bae28525101b4992" } -near-lake-framework = "0.5" +near-jsonrpc-client = "0.4.0-beta.0" +near-lake-framework = "0.5.1" near-primitives = "0.14" -near-crypto = "0.14" \ No newline at end of file +near-crypto = "0.14" diff --git a/Cross.toml b/Cross.toml new file mode 100644 index 00000000..36c74329 --- /dev/null +++ b/Cross.toml @@ -0,0 +1,6 @@ +[target.x86_64-unknown-linux-gnu] +# Run the following command to get the build environment: +# ``` +# docker build --tag cross:x86_64-unknown-linux-gnu ./cross/x86_64-unknown-linux-gnu/ +# ``` +image = "cross:x86_64-unknown-linux-gnu" diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 7a0afaac..00000000 --- a/Dockerfile +++ /dev/null @@ -1,77 +0,0 @@ -# syntax=docker/dockerfile-upstream:experimental - -# ============================== BUILD ====================================== -FROM ubuntu:18.04 as build - -RUN apt-get update -qq && apt-get install -y \ - git \ - cmake \ - g++ \ - pkg-config \ - libssl-dev \ - curl \ - llvm \ - clang \ - libpq-dev \ - ca-certificates \ - && rm -rf /var/lib/apt/lists/* - -COPY ./rust-toolchain /tmp/rust-toolchain - -ENV RUSTUP_HOME=/usr/local/rustup \ - CARGO_HOME=/usr/local/cargo \ - PATH=/usr/local/cargo/bin:$PATH - -RUN curl https://sh.rustup.rs -sSf | \ - sh -s -- -y --no-modify-path --default-toolchain "$(cat /tmp/rust-toolchain)" - -RUN cargo install diesel_cli --no-default-features --features "postgres" --bin diesel - -WORKDIR /near - -# This is some clever stuff we do to build JUST the Cargo.toml dependencies first, so that Docker can cache them so long as Cargo.toml doesn't change -# We do this because building dependencies takes ~45 minutes -RUN cargo +"$(cat /tmp/rust-toolchain)" new --bin indexer-explorer -WORKDIR /near/indexer-explorer - -COPY ./Cargo.toml ./Cargo.toml -COPY ./Cargo.lock ./Cargo.lock - -ENV CARGO_TARGET_DIR=/tmp/target -ENV RUSTC_FLAGS='-C target-cpu=x86-64' -ENV PORTABLE=ON -RUN cargo +"$(cat /tmp/rust-toolchain)" build --release -RUN rm src/*.rs -RUN rm /tmp/target/release/indexer-explorer* - -# Now that the dependencies are built, copy the actual code in and build that too -COPY . . - -# This touch is necessary so that Rust doesn't skip the build (even though the source has completely changed... Rust cache is weird :P) -RUN touch src/main.rs - -RUN cargo +"$(cat /tmp/rust-toolchain)" build --release -p indexer-explorer - -# ============================== EXECUTION ====================================== -FROM ubuntu:18.04 as execution - -RUN apt-get update -qq && apt-get install -y \ - libssl-dev \ - libpq-dev \ - ca-certificates \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /near/indexer-explorer - -COPY --from=build /usr/local/cargo/bin/diesel . -COPY --from=build /tmp/target/release/indexer-explorer . -# Diesel needs a migrations directory to run -COPY --from=build /near/indexer-explorer/migrations ./migrations - -# If the --store-genesis flag isn't set, the accounts in genesis won't get created in the DB which will lead to foreign key constraint violations -# See https://github.com/near/near-indexer-for-explorer/issues/167 -CMD ./diesel migration run && \ - ./indexer-explorer --home-dir /root/.near/localnet init ${BOOT_NODES:+--boot-nodes=${BOOT_NODES}} --chain-id localnet && \ - sed -i 's/"tracked_shards": \[\]/"tracked_shards": \[0\]/' /root/.near/localnet/config.json && \ - sed -i 's/"archive": false/"archive": true/' /root/.near/localnet/config.json && \ - ./indexer-explorer --home-dir /root/.near/localnet run --store-genesis sync-from-latest diff --git a/README.md b/README.md index 9514d6c2..028c37b7 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# NEAR Indexer for Explorer +# NEAR Indexer for Explorer Lake -NEAR Indexer for Explorer is built on top of [NEAR Indexer microframework](https://github.com/nearprotocol/nearcore/tree/master/chain/indexer) to watch the network and store all the events in the PostgreSQL database. +NEAR Indexer for Explorer Lake is built on top of [NEAR Lake Framework](https://github.com/near/near-lake-framework-rs) to watch the network and store all the events in the PostgreSQL database. ## Shared Public Access @@ -16,8 +16,8 @@ NOTE: Please, keep in mind that the access to the database is shared across ever ## Self-hosting The final setup consists of the following components: -* PostgreSQL database (you can run it locally or in the cloud), which can hold the whole history of the blockchain (as of January 2022, mainnet takes 1.1TB of data in PostgreSQL storage, and testnet takes 420GB) -* NEAR Indexer for Explorer binary that operates as a regular NEAR Protocol peer-to-peer node, so you will operate it as any other [Archival Node in NEAR](https://docs.near.org/docs/develop/node/archival/hardware-archival) +* PostgreSQL database (you can run it locally or in the cloud), which can hold the whole history of the blockchain (as of August 2022, mainnet takes 3TB of data in PostgreSQL storage, and testnet takes 1TB) +* NEAR Indexer for Explorer Lake binary that operates as a NEAR Lake Framework based indexer, it requires [AWS S3 credentials](https://near-indexers.io/tutorials/lake/credentials) ### Prepare Development Environment @@ -39,15 +39,22 @@ Setup PostgreSQL database, create a database with the regular tools, and note th Clone this repository and open the project folder ```bash -$ git clone git@github.com:near/near-indexer-for-explorer.git +$ git clone https://gitbub.com/khorolets/near-indexer-for-explorer.git $ cd near-indexer-for-explorer ``` -You need to provide database credentials in `.env` file like below (replace `user`, `password`, `host` and `db_name` with yours): +You need to provide credentials via `.env` file for: +- database -```bash -$ echo "DATABASE_URL=postgres://user:password@host/db_name" > .env -``` + (replace `user`, `password`, `host` and `db_name` with yours) + ```bash + $ echo "DATABASE_URL=postgres://user:password@host/db_name" > .env + ``` +- AWS S3 (permission to read from buckets): + ```bash + $ echo "LAKE_AWS_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" >> .env + $ echo "LAKE_AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + ``` Then you need to apply migrations to create necessary database structure. For this you'll need `diesel-cli`, you can install it like so: @@ -68,48 +75,23 @@ $ diesel migration run $ cargo build --release ``` -### Configure NEAR Indexer for Explorer - -To connect NEAR Indexer for Explorer to the specific chain you need to have necessary configs, you can generate it as follows: - -```bash -$ ./target/release/near-indexer --home-dir ~/.near/testnet init --chain-id testnet --download-config --download-genesis -``` +### Run NEAR Indexer for Explorer -The above code will download the official genesis config and generate necessary configs. You can replace `testnet` in the command above to different network ID (`betanet`, `mainnet`). +Command to run NEAR Indexer for Explorer Lake have to include the chain-id and start options: -**NB!** According to changes in `nearcore` config generation we don't fill all the necessary fields in the config file. -While this issue is open https://github.com/nearprotocol/nearcore/issues/3156 you need to download config you want and replace the generated one manually. - - [testnet config.json](https://s3-us-west-1.amazonaws.com/build.nearprotocol.com/nearcore-deploy/testnet/config.json) - - [betanet config.json](https://s3-us-west-1.amazonaws.com/build.nearprotocol.com/nearcore-deploy/betanet/config.json) - - [mainnet config.json](https://s3-us-west-1.amazonaws.com/build.nearprotocol.com/nearcore-deploy/mainnet/config.json) +You can choose NEAR Indexer for Explorer Lake start options: + - `from-latest` - start indexing blocks from the latest finalized block + - `from-interruption` - start indexing blocks from the block NEAR Indexer was interrupted last time but earlier for `` if provided + - `from-block --height ` - start indexing blocks from the specific block height -Configs for the specified network are in the `--home-dir` provided folder. We need to ensure that NEAR Indexer for Explorer follows -all the necessary shards, so `"tracked_shards"` parameters in `~/.near/testnet/config.json` needs to be configured properly. -For example, with a single shared network, you just add the shard #0 to the list: +Unlike the original [NEAR Indexer for Explorer](https://github.com/near/near-indexer-for-explorer) you **can't** tell Indexer to store data from genesis (Accounts and Access Keys) by adding key `--store-genesis` to the `run` command. So please, ensure you took care about the genesis data in your database in order this indexer to work properly. ``` -... -"tracked_shards": [0], -... +TODO: Implement a tool to fill the database with the genesis data ``` -### Run NEAR Indexer for Explorer - -Command to run NEAR Indexer for Explorer have to contain sync mode. - -You can choose NEAR Indexer for Explorer sync mode by setting what to stream: - - `sync-from-latest` - start indexing blocks from the latest finalized block - - `sync-from-interruption --delta ` - start indexing blocks from the block NEAR Indexer was interrupted last time but earlier for `` if provided - - `sync-from-block --height ` - start indexing blocks from the specific block height - -Optionally you can tell Indexer to store data from genesis (Accounts and Access Keys) by adding key `--store-genesis` to the `run` command. - -NEAR Indexer for Explorer works in strict mode by default, but you can disable it for specific amount of blocks. The strict mode means that every piece of data -will be retried to store to database in case of error. Errors may occur when the parent piece of data is still processed but the child piece is already -trying to be stored. So Indexer keeps retrying to store the data until success. However if you're running Indexer not from the genesis it is possible that you -really miss some of parent data and it'll be impossible to store child one, so you can disable strict mode for 1000 blocks to ensure you've passed the strong -relation data area and you're running Indexer where it is impossible to loose any piece of data. +NEAR Indexer for Explorer Lake works in strict mode by default, but you can disable it. The strict mode means that every piece of data +will be retried to store to database in case of error. Errors may occur when the parent piece of data is still processed but the child piece is already trying to be stored. So Indexer keeps retrying to store the data until success. However, if you're running Indexer not from the genesis it is possible that you really miss some of parent data and it'll be impossible to store a child one, so you can disable strict mode to ensure you've passed the strong relation data area and you're running Indexer where it is impossible to loose any piece of data. To disable strict mode you need to provide: @@ -117,14 +99,17 @@ To disable strict mode you need to provide: --non-strict-mode ``` -Sometimes you may want to index block while sync process is happening, by default an indexer node is waiting for full sync to complete but you can enable indexing while the node is syncing by passing `--stream-while-syncing` +Unlike the original [NEAR Indexer for Explorer](https://github.com/near/near-indexer-for-explorer) the Lake-based version is not syncing with the network in a traditional way, so we've dropped the `--stream-while-syncing` feature. By default NEAR Indexer for Explorer processes only a single block at a time. You can adjust this with the `--concurrency` argument (when the blocks are mostly empty, it is fine to go with as many as 100 blocks of concurrency). So final command to run NEAR Indexer for Explorer can look like: ```bash -$ cargo run --release -- --home-dir ~/.near/testnet run --store-genesis --stream-while-syncing --non-strict-mode --concurrency 1 sync-from-latest +$ ./target/release/indexer-explorer-lake \ + --non-strict-mode \ + --concurrency 1 \ + from-latest ``` After the network is synced, you should see logs of every block height currently received by NEAR Indexer for Explorer. @@ -165,89 +150,3 @@ GRANT readonly TO explorer; ```bash $ PGPASSWORD="password" psql -h 127.0.0.1 -U explorer databasename ``` - -## Syncing - -Whenever you run NEAR Indexer for Explorer for any network except localnet you'll need to sync with the network. This is required because it's a natural behavior of `nearcore` node and NEAR Indexer for Explorer is a wrapper for the regular `nearcore` node. In order to work and index the data your node must be synced with the network. This process can take a while, so we suggest to download a fresh backup of the `data` folder and put it in you `--home-dir` of your choice (by default it is `~/.near`) - -Running your NEAR Indexer for Explorer node on top of a backup data will reduce the time of syncing process because your node will download only missing data and it will take reasonable time. - -All the backups can be downloaded from the public S3 bucket which contains latest daily snapshots: - -* [Recent 5-epoch Mainnet data folder](https://near-protocol-public.s3-accelerate.amazonaws.com/backups/mainnet/rpc/data.tar) -* [Recent 5-epoch Testnet data folder](https://near-protocol-public.s3-accelerate.amazonaws.com/backups/testnet/rpc/data.tar) - - -## Running NEAR Indexer for Explorer as archival node - -It's not necessary but in order to index everything in the network it is better to do it from the genesis. `nearcore` node is running in non-archival mode by default. That means that the node keeps data only for [5 last epochs](https://docs.near.org/docs/concepts/epoch). In order to index data from the genesis we need to turn the node in archival mode. - -To do it we need to update `config.json` located in `--home-dir` or your choice (by default it is `~/.near`). - -Find next keys in the config and update them as following: - -```json -{ - ... - "archive": true, - "tracked_shards": [0], - ... -} -``` - -The syncing process in archival mode can take a lot of time, so it's better to download a backup provided by NEAR and put it in your `data` folder. After that your node will need to sync only missing data and it should take reasonable time. - -All the backups can be downloaded from the public S3 bucket which contains latest daily snapshots: - -* [Archival Mainnet data folder](https://near-protocol-public.s3-accelerate.amazonaws.com/backups/mainnet/archive/data.tar) -* [Archival Testnet data folder](https://near-protocol-public.s3-accelerate.amazonaws.com/backups/testnet/archive/data.tar) - -See https://docs.near.org/docs/roles/integrator/exchange-integration#running-an-archival-node for reference - -## Local debugging - -If you want to play with the code locally, it's better not to copy existing mainnet/testnet (it requires LOTS of memory), but to have your own small example. -You need to have empty DB (we suggest to use [Docker](https://hub.docker.com/_/postgres) for that). -Go through steps [above](https://github.com/near/near-indexer-for-explorer#self-hosting) until (including) diesel migration. -Then, - -```bash -$ cargo run --release -- --home-dir ~/.near/localnet init --chain-id localnet -``` - -Edit `~/.near/localnet/config.json` by adding tracking shards and archiving option (see [example above](https://github.com/near/near-indexer-for-explorer#running-near-indexer-for-explorer-as-archival-node)). - -```bash -$ cargo run -- --home-dir ~/.near/localnet run --store-genesis sync-from-latest -``` - -Congrats, the blocks are being produced right now! -There should be some lines in the DB. -Now, we need to generate some activity to add new examples. - -```bash -$ npm i -g near-cli -$ NEAR_ENV=local near create-account awesome.test.near --initialBalance 30 --masterAccount test.near --keyPath=~/.near/localnet/validator_key.json -$ NEAR_ENV=local near send test.near awesome.test.near 5 -``` - -All available commands are [here](https://github.com/near/near-cli#near-cli-command-line-interface). - -You can stop and re-run the example at any time. -Blocks will continue producing from the last state. - -## Troubleshooting - -When operating normally, you should see "INFO indexer_for_explorer: Block height ..." messages in the logs. - -### The node is fully synced and running, but no indexer messages and no transactions in the database (not indexing) - -Make sure the blocks you want to save exist on the node. Check them via [JSON RPC](https://docs.near.org/docs/api/rpc#block-details): - -``` -curl http://127.0.0.1:3030/ -X POST --header 'Content-type: application/json' --data '{"jsonrpc": "2.0", "id": "dontcare", "method": "block", "params": {"block_id": 9820214}}' -``` - -NOTE: Block #9820214 is the first block after genesis block (#9820210) on Mainnet. - -If it returns an error that the block does not exist or missing, it means that your node does not have the necessary data. Your options here are to start from the blocks that are recorded on the node or start an archival node (see above) and make sure you have the full network history (either use a backup or let the node sync from scratch (it is quite slow, so backup is recommended)) diff --git a/cross/x86_64-unknown-linux-gnu/Dockerfile b/cross/x86_64-unknown-linux-gnu/Dockerfile new file mode 100644 index 00000000..87bfb42a --- /dev/null +++ b/cross/x86_64-unknown-linux-gnu/Dockerfile @@ -0,0 +1,14 @@ +FROM ubuntu:18.04 + +# Copy from nearcore: +# https://github.com/near/nearcore/blob/master/Dockerfile +RUN apt-get update -qq && \ + apt-get install -y \ + git \ + cmake \ + g++ \ + pkg-config \ + libssl-dev \ + curl \ + llvm \ + clang diff --git a/src/aggregated/account_details.rs b/src/aggregated/account_details.rs deleted file mode 100644 index 74474023..00000000 --- a/src/aggregated/account_details.rs +++ /dev/null @@ -1,66 +0,0 @@ -use actix::Addr; -use anyhow::Context; - -use near_client::{Query, ViewClientActor}; -use near_indexer::near_primitives; - -pub(crate) async fn get_account_balance( - view_client: &Addr, - account_id: &near_primitives::types::AccountId, - block_height: &near_primitives::types::BlockHeight, -) -> anyhow::Result { - get_account_view_for_block_height(view_client, account_id, block_height) - .await - .map(|account| account.amount) - .with_context(|| format!("Unable to get account balance for {}", account_id)) -} - -pub(crate) async fn get_contract_code_hash( - view_client: &Addr, - account_id: &near_primitives::types::AccountId, - block_height: &near_primitives::types::BlockHeight, -) -> anyhow::Result { - get_account_view_for_block_height(view_client, account_id, block_height) - .await - .map(|account| account.code_hash) - .with_context(|| format!("Unable to get contract code hash for {}", account_id)) -} - -async fn get_account_view_for_block_height( - view_client: &Addr, - account_id: &near_primitives::types::AccountId, - block_height: &near_primitives::types::BlockHeight, -) -> anyhow::Result { - let block_reference = near_primitives::types::BlockReference::BlockId( - near_primitives::types::BlockId::Height(*block_height), - ); - let request = near_lake_framework::near_indexer_primitives::views::QueryRequest::ViewAccount { - account_id: account_id.clone(), - }; - let query = Query::new(block_reference, request); - - let account_response = view_client - .send(query) - .await - .with_context(|| { - format!( - "Failed to deliver ViewAccount for account {}, block {}", - account_id, block_height - ) - })? - .with_context(|| { - format!( - "Invalid ViewAccount query for account {}, block {}", - account_id, block_height - ) - })?; - - match account_response.kind { - near_lake_framework::near_indexer_primitives::views::QueryResponseKind::ViewAccount(account) => Ok(account), - _ => anyhow::bail!( - "Failed to extract ViewAccount response for account {}, block {}", - account_id, - block_height - ), - } -} diff --git a/src/aggregated/circulating_supply/lockup.rs b/src/aggregated/circulating_supply/lockup.rs deleted file mode 100644 index 5f58703a..00000000 --- a/src/aggregated/circulating_supply/lockup.rs +++ /dev/null @@ -1,207 +0,0 @@ -use std::time::Duration; - -use actix::Addr; -use anyhow::Context; - -use near_client::{Query, ViewClientActor}; -use near_indexer::near_primitives; -use near_sdk::borsh::BorshDeserialize; -use near_sdk::json_types::{U128, U64}; - -use super::lockup_types::{ - LockupContract, TransfersInformation, VestingInformation, VestingSchedule, WrappedBalance, U256, -}; - -// The timestamp (nanos) when transfers were enabled in the Mainnet after community voting -// Tuesday, 13 October 2020 18:38:58.293 -pub(super) const TRANSFERS_ENABLED: Duration = Duration::from_nanos(1602614338293769340); - -pub(super) async fn get_lockup_contract_state( - view_client: &Addr, - account_id: &near_primitives::types::AccountId, - block_height: &near_primitives::types::BlockHeight, -) -> anyhow::Result { - let block_reference = near_primitives::types::BlockReference::BlockId( - near_primitives::types::BlockId::Height(*block_height), - ); - let request = near_lake_framework::near_indexer_primitives::views::QueryRequest::ViewState { - account_id: account_id.clone(), - prefix: vec![].into(), - }; - let query = Query::new(block_reference, request); - - let state_response = view_client - .send(query) - .await - .with_context(|| { - format!( - "Failed to deliver ViewState for lockup contract {}, block_height {}", - account_id, block_height - ) - })? - .with_context(|| { - format!( - "Invalid ViewState query for lockup contract {}, block_height {}", - account_id, block_height - ) - })?; - - let view_state_result = match state_response.kind { - near_lake_framework::near_indexer_primitives::views::QueryResponseKind::ViewState(x) => x, - _ => { - anyhow::bail!( - "Failed to extract ViewState response for lockup contract {}, block_height {}", - account_id, - block_height - ) - } - }; - let view_state = view_state_result.values.get(0).with_context(|| { - format!( - "Failed to find encoded lockup contract for {}, block_height {}", - account_id, block_height - ) - })?; - - let mut state = LockupContract::try_from_slice( - base64::decode(&view_state.value) - .with_context(|| { - format!( - "Failed to decode `view_state` for lockup contract {}", - account_id - ) - })? - .as_slice(), - ) - .with_context(|| format!("Failed to construct LockupContract for {}", account_id))?; - - // If owner of the lockup account didn't call the - // `check_transfers_vote` contract method we won't be able to - // get proper information based on timestamp, that's why we inject - // the `transfer_timestamp` which is phase2 timestamp - state.lockup_information.transfers_information = TransfersInformation::TransfersEnabled { - transfers_timestamp: U64(TRANSFERS_ENABLED.as_nanos() as u64), - }; - Ok(state) -} - -// The lockup contract implementation had a bug that affected lockup start date. -// https://github.com/near/core-contracts/pull/136 -// For each contract, we should choose the logic based on the binary version of the contract -pub(super) fn is_bug_inside_contract( - code_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, - account_id: &near_primitives::types::AccountId, -) -> anyhow::Result { - match &*code_hash.to_string() { - // The first implementation, with the bug - "3kVY9qcVRoW3B5498SMX6R3rtSLiCdmBzKs7zcnzDJ7Q" => Ok(true), - // We have 6 lockups created at 6th of April 2021, assume it's buggy - "DiC9bKCqUHqoYqUXovAnqugiuntHWnM3cAc7KrgaHTu" => Ok(true), - // Another 5 lockups created in May/June 2021, assume they are OK - "Cw7bnyp4B6ypwvgZuMmJtY6rHsxP2D4PC8deqeJ3HP7D" => Ok(false), - // The most fresh one - "4Pfw2RU6e35dUsHQQoFYfwX8KFFvSRNwMSNLXuSFHXrC" => Ok(false), - other => anyhow::bail!( - "Unable to recognise the version of contract {}, code hash {}", - account_id, - other - ), - } -} - -// This is almost a copy of https://github.com/near/core-contracts/blob/master/lockup/src/getters.rs#L64 -impl LockupContract { - /// Returns the amount of tokens that are locked in the account due to lockup or vesting. - pub fn get_locked_amount(&self, timestamp: u64, has_bug: bool) -> WrappedBalance { - let lockup_amount = self.lockup_information.lockup_amount; - if let TransfersInformation::TransfersEnabled { - transfers_timestamp, - } = &self.lockup_information.transfers_information - { - let lockup_timestamp = std::cmp::max( - transfers_timestamp - .0 - .saturating_add(self.lockup_information.lockup_duration), - self.lockup_information.lockup_timestamp.unwrap_or(0), - ); - let block_timestamp = timestamp; - if lockup_timestamp <= block_timestamp { - let unreleased_amount = - if let Some(release_duration) = self.lockup_information.release_duration { - let start_lockup = if has_bug { - transfers_timestamp.0 - } else { - lockup_timestamp - }; - let end_timestamp = start_lockup.saturating_add(release_duration); - if block_timestamp >= end_timestamp { - // Everything is released - 0 - } else { - let time_left = U256::from(end_timestamp - block_timestamp); - let unreleased_amount = U256::from(lockup_amount) * time_left - / U256::from(release_duration); - // The unreleased amount can't be larger than lockup_amount because the - // time_left is smaller than total_time. - unreleased_amount.as_u128() - } - } else { - 0 - }; - - let unvested_amount = match &self.vesting_information { - VestingInformation::VestingSchedule(vs) => { - self.get_unvested_amount(vs.clone(), block_timestamp) - } - VestingInformation::Terminating(terminating) => terminating.unvested_amount, - // Vesting is private, so we can assume the vesting started before lockup date. - _ => U128(0), - }; - return std::cmp::max( - unreleased_amount - .saturating_sub(self.lockup_information.termination_withdrawn_tokens), - unvested_amount.0, - ) - .into(); - } - } - // The entire balance is still locked before the lockup timestamp. - (lockup_amount - self.lockup_information.termination_withdrawn_tokens).into() - } - - /// Returns the amount of tokens that are locked in this account due to vesting schedule. - /// Takes raw vesting schedule, in case the internal vesting schedule is private. - pub fn get_unvested_amount( - &self, - vesting_schedule: VestingSchedule, - block_timestamp: u64, - ) -> WrappedBalance { - let lockup_amount = self.lockup_information.lockup_amount; - match &self.vesting_information { - VestingInformation::Terminating(termination_information) => { - termination_information.unvested_amount - } - VestingInformation::None => U128::from(0), - _ => { - if block_timestamp < vesting_schedule.cliff_timestamp.0 { - // Before the cliff, nothing is vested - lockup_amount.into() - } else if block_timestamp >= vesting_schedule.end_timestamp.0 { - // After the end, everything is vested - 0.into() - } else { - // cannot overflow since block_timestamp < vesting_schedule.end_timestamp - let time_left = U256::from(vesting_schedule.end_timestamp.0 - block_timestamp); - // The total time is positive. Checked at the contract initialization. - let total_time = U256::from( - vesting_schedule.end_timestamp.0 - vesting_schedule.start_timestamp.0, - ); - let unvested_amount = U256::from(lockup_amount) * time_left / total_time; - // The unvested amount can't be larger than lockup_amount because the - // time_left is smaller than total_time. - unvested_amount.as_u128().into() - } - } - } - } -} diff --git a/src/aggregated/circulating_supply/lockup_types.rs b/src/aggregated/circulating_supply/lockup_types.rs deleted file mode 100644 index 905910b8..00000000 --- a/src/aggregated/circulating_supply/lockup_types.rs +++ /dev/null @@ -1,211 +0,0 @@ -#![allow( - clippy::assign_op_pattern, - clippy::manual_range_contains, - clippy::ptr_offset_with_cast -)] -// Copied from lockup contract code -// https://github.com/near/core-contracts/blob/master/lockup/src/types.rs -// https://github.com/near/core-contracts/blob/master/lockup/src/lib.rs - -use uint::construct_uint; - -use near_sdk::borsh::{self, BorshDeserialize, BorshSerialize}; -use near_sdk::json_types::{Base64VecU8, U128, U64}; -use near_sdk::serde::{Deserialize, Serialize}; -use near_sdk::{AccountId, Balance}; - -construct_uint! { - /// 256-bit unsigned integer. - pub struct U256(4); -} - -/// Raw type for duration in nanoseconds -pub type Duration = u64; -/// Raw type for timestamp in nanoseconds -pub type Timestamp = u64; - -/// Timestamp in nanosecond wrapped into a struct for JSON serialization as a string. -pub type WrappedTimestamp = U64; -/// Balance wrapped into a struct for JSON serialization as a string. -pub type WrappedBalance = U128; - -#[derive(BorshDeserialize, BorshSerialize)] -pub struct LockupContract { - /// The account ID of the owner. - pub owner_account_id: AccountId, - - /// Information about lockup schedule and the amount. - pub lockup_information: LockupInformation, - - /// Information about vesting including schedule or termination status. - pub vesting_information: VestingInformation, - - /// Account ID of the staking pool whitelist contract. - pub staking_pool_whitelist_account_id: AccountId, - - /// Information about staking and delegation. - /// `Some` means the staking information is available and the staking pool contract is selected. - /// `None` means there is no staking pool selected. - pub staking_information: Option, - - /// The account ID that the NEAR Foundation, that has the ability to terminate vesting. - pub foundation_account_id: Option, -} - -/// Contains information about token lockups. -#[derive(BorshDeserialize, BorshSerialize)] -pub struct LockupInformation { - /// The amount in yocto-NEAR tokens locked for this account. - pub lockup_amount: Balance, - /// The amount of tokens that were withdrawn by NEAR foundation due to early termination - /// of vesting. - /// This amount has to be accounted separately from the lockup_amount to make sure - /// linear release is not being affected. - pub termination_withdrawn_tokens: Balance, - /// [deprecated] - the duration in nanoseconds of the lockup period from - /// the moment the transfers are enabled. During this period tokens are locked and - /// the release doesn't start. Instead of this, use `lockup_timestamp` and `release_duration` - pub lockup_duration: Duration, - /// If present, it is the duration when the full lockup amount will be available. The tokens - /// are linearly released from the moment tokens are unlocked, defined by: - /// `max(transfers_timestamp + lockup_duration, lockup_timestamp)`. - /// If not present, the tokens are not locked (though, vesting logic could be used). - pub release_duration: Option, - /// The optional absolute lockup timestamp in nanoseconds which locks the tokens until this - /// timestamp passes. Until this moment the tokens are locked and the release doesn't start. - /// If not present, `transfers_timestamp` will be used. - pub lockup_timestamp: Option, - /// The information about the transfers. Either transfers are already enabled, then it contains - /// the timestamp when they were enabled. Or the transfers are currently disabled and - /// it contains the account ID of the transfer poll contract. - pub transfers_information: TransfersInformation, -} - -/// Contains information about the transfers. Whether transfers are enabled or disabled. -#[derive(BorshDeserialize, BorshSerialize, Deserialize, Serialize, Debug)] -#[serde(crate = "near_sdk::serde")] -pub enum TransfersInformation { - /// The timestamp when the transfers were enabled. - TransfersEnabled { - transfers_timestamp: WrappedTimestamp, - }, - /// The account ID of the transfers poll contract, to check if the transfers are enabled. - /// The lockup period can start only after the transfer voted to be enabled. - /// At the launch of the network transfers are disabled for all lockup contracts, once transfers - /// are enabled, they can't be disabled and don't need to be checked again. - TransfersDisabled { transfer_poll_account_id: AccountId }, -} - -/// Describes the status of transactions with the staking pool contract or terminated unvesting -/// amount withdrawal. -#[derive(BorshDeserialize, BorshSerialize, Deserialize, Serialize, PartialEq)] -#[serde(crate = "near_sdk::serde")] -pub enum TransactionStatus { - /// There are no transactions in progress. - Idle, - /// There is a transaction in progress. - Busy, -} - -/// Contains information about current stake and delegation. -#[derive(BorshDeserialize, BorshSerialize)] -pub struct StakingInformation { - /// The Account ID of the staking pool contract. - pub staking_pool_account_id: AccountId, - - /// Contains status whether there is a transaction in progress. - pub status: TransactionStatus, - - /// The amount of tokens that were deposited from this account to the staking pool. - /// NOTE: The unstaked amount on the staking pool might be higher due to staking rewards. - pub deposit_amount: WrappedBalance, -} - -/// Contains information about vesting schedule. -#[derive(BorshDeserialize, BorshSerialize, Deserialize, Serialize, Clone, PartialEq, Debug)] -#[serde(crate = "near_sdk::serde")] -pub struct VestingSchedule { - /// The timestamp in nanosecond when the vesting starts. E.g. the start date of employment. - pub start_timestamp: WrappedTimestamp, - /// The timestamp in nanosecond when the first part of lockup tokens becomes vested. - /// The remaining tokens will vest continuously until they are fully vested. - /// Example: a 1 year of employment at which moment the 1/4 of tokens become vested. - pub cliff_timestamp: WrappedTimestamp, - /// The timestamp in nanosecond when the vesting ends. - pub end_timestamp: WrappedTimestamp, -} - -/// Initialization argument type to define the vesting schedule -#[derive(Serialize, Deserialize, Debug)] -#[serde(crate = "near_sdk::serde")] -pub enum VestingScheduleOrHash { - /// [deprecated] After transfers are enabled, only public schedule is used. - /// The vesting schedule is private and this is a hash of (vesting_schedule, salt). - /// In JSON, the hash has to be encoded with base64 to a string. - VestingHash(Base64VecU8), - /// The vesting schedule (public) - VestingSchedule(VestingSchedule), -} - -/// Contains information about vesting that contains vesting schedule and termination information. -#[derive(Serialize, BorshDeserialize, BorshSerialize, PartialEq, Clone, Debug)] -#[serde(crate = "near_sdk::serde")] -pub enum VestingInformation { - None, - /// [deprecated] After transfers are enabled, only public schedule is used. - /// Vesting schedule is hashed for privacy and only will be revealed if the NEAR foundation - /// has to terminate vesting. - /// The contract assume the vesting schedule doesn't affect lockup release and duration, because - /// the vesting started before transfers were enabled and the duration is shorter or the same. - VestingHash(Base64VecU8), - /// Explicit vesting schedule. - VestingSchedule(VestingSchedule), - /// The information about the early termination of the vesting schedule. - /// It means the termination of the vesting is currently in progress. - /// Once the unvested amount is transferred out, `VestingInformation` is removed. - Terminating(TerminationInformation), -} - -/// Describes the status of transactions with the staking pool contract or terminated unvesting -/// amount withdrawal. -#[derive( - BorshDeserialize, BorshSerialize, Deserialize, Serialize, PartialEq, Copy, Clone, Debug, -)] -#[serde(crate = "near_sdk::serde")] -pub enum TerminationStatus { - /// Initial stage of the termination in case there are deficit on the account. - VestingTerminatedWithDeficit, - /// A transaction to unstake everything is in progress. - UnstakingInProgress, - /// The transaction to unstake everything from the staking pool has completed. - EverythingUnstaked, - /// A transaction to withdraw everything from the staking pool is in progress. - WithdrawingFromStakingPoolInProgress, - /// Everything is withdrawn from the staking pool. Ready to withdraw out of the account. - ReadyToWithdraw, - /// A transaction to withdraw tokens from the account is in progress. - WithdrawingFromAccountInProgress, -} - -/// Contains information about early termination of the vesting schedule. -#[derive(BorshDeserialize, BorshSerialize, Deserialize, Serialize, PartialEq, Clone, Debug)] -#[serde(crate = "near_sdk::serde")] -pub struct TerminationInformation { - /// The amount of tokens that are unvested and has to be transferred back to NEAR Foundation. - /// These tokens are effectively locked and can't be transferred out and can't be restaked. - pub unvested_amount: WrappedBalance, - - /// The status of the withdrawal. When the unvested amount is in progress of withdrawal the - /// status will be marked as busy, to avoid withdrawing the funds twice. - pub status: TerminationStatus, -} - -/// Contains a vesting schedule with a salt. -#[derive(BorshSerialize, Deserialize, Serialize, Clone, Debug)] -#[serde(crate = "near_sdk::serde")] -pub struct VestingScheduleWithSalt { - /// The vesting schedule - pub vesting_schedule: VestingSchedule, - /// Salt to make the hash unique - pub salt: Base64VecU8, -} diff --git a/src/aggregated/circulating_supply/mod.rs b/src/aggregated/circulating_supply/mod.rs deleted file mode 100644 index 84629d60..00000000 --- a/src/aggregated/circulating_supply/mod.rs +++ /dev/null @@ -1,247 +0,0 @@ -use std::ops::{Add, Sub}; -use std::str::FromStr; -use std::time::{Duration, SystemTime}; - -use actix::Addr; -use actix_diesel::Database; -use anyhow::Context; -use bigdecimal::{BigDecimal, ToPrimitive}; -use chrono::NaiveDateTime; -use diesel::PgConnection; -use tracing::{error, info, warn}; - -use near_indexer::near_primitives; - -use crate::aggregated::{account_details, circulating_supply}; -use crate::db_adapters::accounts; -use crate::db_adapters::aggregated::circulating_supply::{ - add_circulating_supply, get_precomputed_circulating_supply_for_timestamp, -}; -use crate::db_adapters::blocks; -use crate::models; -use crate::models::aggregated::circulating_supply::CirculatingSupply; - -mod lockup; -mod lockup_types; - -const DAY: Duration = Duration::from_secs(60 * 60 * 24); -const RETRY_DURATION: Duration = Duration::from_secs(60 * 60 * 2); - -// Compute circulating supply on a daily basis, starting from 13 Oct 2020 -// (Transfers enabled moment on the Mainnet), and put it to the Indexer DB. -// Circulating supply is calculated by the formula: -// total_supply - sum(locked_tokens_on_each_lockup) - sum(locked_foundation_account) -// The value is always computed for the last block in a day (UTC). -pub(super) async fn run_circulating_supply_computation( - view_client: Addr, - pool: Database, -) { - // We perform actual computations 00:10 UTC each day to be sure that the data is finalized - let mut day_to_compute = lockup::TRANSFERS_ENABLED - .sub(Duration::from_secs( - lockup::TRANSFERS_ENABLED.as_secs() % DAY.as_secs(), - )) - .add(DAY) - .add(Duration::from_secs(10 * 60)); - - loop { - let now = SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .expect("Time went backwards"); - - if now < day_to_compute { - tokio::time::sleep_until(tokio::time::Instant::now().add(day_to_compute.sub(now))) - .await; - } - wait_for_loading_needed_blocks(&view_client, &day_to_compute).await; - - match check_and_collect_daily_circulating_supply(&view_client, &pool, &day_to_compute).await - { - Ok(_) => { - day_to_compute = day_to_compute.add(DAY); - } - Err(err) => { - error!( - target: crate::AGGREGATED, - "Failed to compute circulating supply for {}: {}. Retry in {} hours", - NaiveDateTime::from_timestamp(day_to_compute.as_secs() as i64, 0).date(), - err, - RETRY_DURATION.as_secs() / 60 / 60, - ); - tokio::time::sleep(RETRY_DURATION).await; - } - }; - } -} - -async fn check_and_collect_daily_circulating_supply( - view_client: &Addr, - pool: &Database, - request_datetime: &Duration, -) -> anyhow::Result> { - let start_of_day = request_datetime.as_nanos() - - request_datetime.as_nanos() % circulating_supply::DAY.as_nanos(); - let printable_date = NaiveDateTime::from_timestamp(request_datetime.as_secs() as i64, 0).date(); - let block = blocks::get_latest_block_before_timestamp(pool, start_of_day as u64).await?; - let block_timestamp = block - .block_timestamp - .to_u64() - .context("`block_timestamp` expected to be u64")?; - - match get_precomputed_circulating_supply_for_timestamp(pool, block_timestamp).await { - Ok(None) => { - info!( - target: crate::AGGREGATED, - "Computing circulating supply for {} (timestamp {})", - printable_date, - block_timestamp - ); - let supply = compute_circulating_supply_for_block(pool, view_client, &block).await?; - add_circulating_supply(pool, &supply).await; - info!( - target: crate::AGGREGATED, - "Circulating supply for {} (timestamp {}) is {}", - printable_date, - block_timestamp, - supply.circulating_tokens_supply - ); - Ok(Some(supply)) - } - Ok(Some(supply)) => { - info!( - target: crate::AGGREGATED, - "Circulating supply for {} (timestamp {}) was already computed: {}", - printable_date, - block_timestamp, - supply - ); - Ok(None) - } - Err(err) => Err(err), - } -} - -async fn compute_circulating_supply_for_block( - pool: &Database, - view_client: &Addr, - block: &models::Block, -) -> anyhow::Result { - let block_timestamp = block - .block_timestamp - .to_u64() - .context("`block_timestamp` expected to be u64")?; - let block_height = block - .block_height - .to_u64() - .context("`block_height` expected to be u64")?; - let total_supply = block - .total_supply - .to_string() - .parse::() - .context("`total_supply` expected to be u128")?; - - let lockup_account_ids = - accounts::get_lockup_account_ids_at_block_height(pool, &block_height).await?; - - let mut lockups_locked_tokens: u128 = 0; - let mut unfinished_lockup_contracts_count: i32 = 0; - - for lockup_account_id in &lockup_account_ids { - let state = - lockup::get_lockup_contract_state(view_client, lockup_account_id, &block_height) - .await - .with_context(|| { - format!( - "Failed to get lockup contract details for {}", - lockup_account_id - ) - })?; - let code_hash = - account_details::get_contract_code_hash(view_client, lockup_account_id, &block_height) - .await?; - let is_lockup_with_bug = lockup::is_bug_inside_contract(&code_hash, lockup_account_id)?; - let locked_amount = state - .get_locked_amount(block_timestamp, is_lockup_with_bug) - .0; - lockups_locked_tokens += locked_amount; - if locked_amount > 0 { - unfinished_lockup_contracts_count += 1; - } - } - - // The list is taken from the conversation with Yessin - let foundation_locked_account_ids: [near_primitives::types::AccountId; 2] = [ - near_primitives::types::AccountId::from_str("lockup.near") - .expect("lockup.near expected to be a valid AccountId"), - near_primitives::types::AccountId::from_str("contributors.near") - .expect("contributors.near expected to be a valid AccountId"), - ]; - let mut foundation_locked_tokens: u128 = 0; - for account_id in &foundation_locked_account_ids { - foundation_locked_tokens += - account_details::get_account_balance(view_client, account_id, &block_height).await?; - } - - let circulating_supply: u128 = total_supply - foundation_locked_tokens - lockups_locked_tokens; - - Ok(CirculatingSupply { - computed_at_block_timestamp: BigDecimal::from(block_timestamp), - computed_at_block_hash: (&block.block_hash).to_string(), - circulating_tokens_supply: BigDecimal::from_str(&circulating_supply.to_string()) - .context("`circulating_tokens_supply` expected to be u128")?, - total_tokens_supply: BigDecimal::from_str(&total_supply.to_string()) - .context("`total_supply` expected to be u128")?, - total_lockup_contracts_count: lockup_account_ids.len() as i32, - unfinished_lockup_contracts_count, - foundation_locked_tokens: BigDecimal::from_str(&foundation_locked_tokens.to_string()) - .context("`foundation_locked_supply` expected to be u128")?, - lockups_locked_tokens: BigDecimal::from_str(&lockups_locked_tokens.to_string()) - .context("`lockups_locked_supply` expected to be u128")?, - }) -} - -async fn wait_for_loading_needed_blocks( - view_client: &Addr, - day_to_compute: &Duration, -) { - loop { - match get_final_block_timestamp(view_client).await { - Ok(timestamp) => { - if timestamp > *day_to_compute { - return; - } - warn!( - target: crate::AGGREGATED, - "Blocks are not loaded to calculate circulating supply for {}. Wait for {} hours", - NaiveDateTime::from_timestamp(day_to_compute.as_secs() as i64, 0).date(), - circulating_supply::RETRY_DURATION.as_secs() / 60 / 60, - ); - } - Err(err) => { - error!( - target: crate::AGGREGATED, - "Failed to get latest block timestamp: {}. Retry in {} hours", - err, - circulating_supply::RETRY_DURATION.as_secs() / 60 / 60, - ); - } - } - tokio::time::sleep(circulating_supply::RETRY_DURATION).await; - } -} - -async fn get_final_block_timestamp( - view_client: &Addr, -) -> anyhow::Result { - let block_reference = - near_primitives::types::BlockReference::Finality(near_primitives::types::Finality::Final); - let query = near_client::GetBlock(block_reference); - - let block_response = view_client - .send(query) - .await - .context("Failed to deliver response")? - .context("Invalid request")?; - - Ok(Duration::from_nanos(block_response.header.timestamp)) -} diff --git a/src/aggregated/mod.rs b/src/aggregated/mod.rs deleted file mode 100644 index 5c24db0c..00000000 --- a/src/aggregated/mod.rs +++ /dev/null @@ -1,16 +0,0 @@ -use actix_diesel::Database; -use diesel::PgConnection; -use near_indexer::Indexer; - -mod account_details; -mod circulating_supply; - -pub(crate) fn spawn_aggregated_computations(pool: Database, indexer: &Indexer) { - let view_client = indexer.client_actors().0; - if indexer.near_config().genesis.config.chain_id == "mainnet" { - actix::spawn(circulating_supply::run_circulating_supply_computation( - view_client, - pool, - )); - } -} diff --git a/src/configs.rs b/src/configs.rs index 70ef675c..4394c1ea 100644 --- a/src/configs.rs +++ b/src/configs.rs @@ -1,17 +1,31 @@ -use clap::Parser; +use clap::{Parser, Subcommand}; +use tracing_subscriber::EnvFilter; -/// NEAR Indexer for Explorer +use near_jsonrpc_client::{methods, JsonRpcClient}; +use near_lake_framework::near_indexer_primitives::types::{BlockReference, Finality}; + +/// NEAR Indexer for Explorer Lake /// Watches for stream of blocks from the chain +/// built on top of NEAR Lake Framework #[derive(Parser, Debug)] #[clap( version, author, about, - setting(clap::AppSettings::DisableHelpSubcommand), - setting(clap::AppSettings::PropagateVersion), - setting(clap::AppSettings::NextLineHelp) + disable_help_subcommand(true), + propagate_version(true), + next_line_help(true) )] pub(crate) struct Opts { + /// Connection string to connect to the PostgreSQL Database to fetch AlertRules from + #[clap(long, env)] + pub database_url: String, + /// AWS Access Key with the rights to read from AWS S3 + #[clap(long, env)] + pub lake_aws_access_key: String, + /// AWS Secret Access Key with the rights to read from AWS S3 + #[clap(long, env)] + pub lake_aws_secret_access_key: String, /// Enabled Indexer for Explorer debug level of logs #[clap(long)] pub debug: bool, @@ -21,16 +35,156 @@ pub(crate) struct Opts { /// Sets the concurrency for indexing. Note: concurrency (set to 2+) may lead to warnings due to tight constraints between transactions and receipts (those will get resolved eventually, but unless it is the second pass of indexing, concurrency won't help at the moment). #[clap(long, default_value = "1")] pub concurrency: std::num::NonZeroU16, - /// Store initial data from genesis like Accounts, AccessKeys - #[clap(long)] - pub store_genesis: bool, - /// AWS S3 bucket name to get the stream from - #[clap(long)] - pub s3_bucket_name: String, - /// AWS S3 bucket region - #[clap(long)] - pub s3_region_name: String, - /// Block height to start the stream from - #[clap(long, short)] - pub start_block_height: u64, + /// Chain ID: testnet or mainnet + #[clap(subcommand)] + pub chain_id: ChainId, +} + +#[derive(Subcommand, Debug, Clone)] +pub enum ChainId { + #[clap(subcommand)] + Mainnet(StartOptions), + #[clap(subcommand)] + Testnet(StartOptions), +} + +#[allow(clippy::enum_variant_names)] +#[derive(Subcommand, Debug, Clone)] +pub enum StartOptions { + /// Start from specific block height + FromBlock { height: u64 }, + /// Start from interruption (last_indexed_block value from Redis) + FromInterruption, + /// Start from the final block on the network (queries JSON RPC for finality: final) + FromLatest, +} + +impl Opts { + /// Returns [StartOptions] for current [Opts] + pub fn start_options(&self) -> &StartOptions { + match &self.chain_id { + ChainId::Mainnet(start_options) | ChainId::Testnet(start_options) => start_options, + } + } + + // Creates AWS Credentials for NEAR Lake + fn lake_credentials(&self) -> aws_types::credentials::SharedCredentialsProvider { + let provider = aws_types::Credentials::new( + self.lake_aws_access_key.clone(), + self.lake_aws_secret_access_key.clone(), + None, + None, + "alertexer_lake", + ); + aws_types::credentials::SharedCredentialsProvider::new(provider) + } + + /// Creates AWS Shared Config for NEAR Lake + pub fn lake_aws_sdk_config(&self) -> aws_types::sdk_config::SdkConfig { + aws_types::sdk_config::SdkConfig::builder() + .credentials_provider(self.lake_credentials()) + .region(aws_types::region::Region::new("eu-central-1")) + .build() + } + + pub fn rpc_url(&self) -> &str { + match self.chain_id { + ChainId::Mainnet(_) => "https://rpc.mainnet.near.org", + ChainId::Testnet(_) => "https://rpc.testnet.near.org", + } + } +} + +impl Opts { + pub async fn to_lake_config(&self) -> near_lake_framework::LakeConfig { + let s3_config = aws_sdk_s3::config::Builder::from(&self.lake_aws_sdk_config()).build(); + + let config_builder = near_lake_framework::LakeConfigBuilder::default().s3_config(s3_config); + + match &self.chain_id { + ChainId::Mainnet(_) => config_builder + .mainnet() + .start_block_height(get_start_block_height(self).await), + ChainId::Testnet(_) => config_builder + .testnet() + .start_block_height(get_start_block_height(self).await), + } + .build() + .expect("Failed to build LakeConfig") + } +} + +async fn get_start_block_height(opts: &Opts) -> u64 { + match opts.start_options() { + StartOptions::FromBlock { height } => *height, + StartOptions::FromInterruption => { + let pool = crate::models::establish_connection(&opts.database_url); + let last_indexed_block: u64 = match crate::db_adapters::blocks::latest_block_height( + &pool, + ) + .await + { + Ok(last_indexed_block) => { + if let Some(last_indexed_block) = last_indexed_block { + last_indexed_block + } else { + final_block_height(opts).await + } + } + Err(err) => { + tracing::warn!( + target: "alertexer", + "Failed to get last indexer block from Database. Failing to the latest one...\n{:#?}", + err + ); + final_block_height(opts).await + } + }; + last_indexed_block + } + StartOptions::FromLatest => final_block_height(opts).await, + } +} + +pub(crate) fn init_tracing(debug: bool) -> anyhow::Result<()> { + let mut env_filter = + EnvFilter::new("near_lake_framework=info,indexer_for_explorer=info,stats=info"); + + if debug { + env_filter = env_filter + .add_directive("indexer_for_explorer=debug".parse()?) + .add_directive("near_lake_framework=debug".parse()?); + } + + if let Ok(rust_log) = std::env::var("RUST_LOG") { + if !rust_log.is_empty() { + for directive in rust_log.split(',').filter_map(|s| match s.parse() { + Ok(directive) => Some(directive), + Err(err) => { + eprintln!("Ignoring directive `{}`: {}", s, err); + None + } + }) { + env_filter = env_filter.add_directive(directive); + } + } + } + + tracing_subscriber::fmt::Subscriber::builder() + .with_env_filter(env_filter) + .with_writer(std::io::stderr) + .init(); + + Ok(()) +} + +async fn final_block_height(opts: &Opts) -> u64 { + let client = JsonRpcClient::connect(opts.rpc_url()); + let request = methods::block::RpcBlockRequest { + block_reference: BlockReference::Finality(Finality::Final), + }; + + let latest_block = client.call(request).await.unwrap(); + + latest_block.header.height } diff --git a/src/db_adapters/access_keys.rs b/src/db_adapters/access_keys.rs index 84836e9e..7adfa013 100644 --- a/src/db_adapters/access_keys.rs +++ b/src/db_adapters/access_keys.rs @@ -2,11 +2,10 @@ use std::collections::HashMap; use std::convert::TryFrom; use actix_diesel::dsl::AsyncRunQueryDsl; -use actix_diesel::Database; + use bigdecimal::BigDecimal; use diesel::{ExpressionMethods, PgConnection, QueryDsl}; use futures::try_join; -use tracing::info; use crate::models; use crate::schema; @@ -34,7 +33,11 @@ pub(crate) async fn handle_access_keys( let mut deleted_accounts = HashMap::::new(); for receipt in successful_receipts { - if let near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { actions, .. } = &receipt.receipt { + if let near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { + actions, + .. + } = &receipt.receipt + { for action in actions { match action { near_lake_framework::near_indexer_primitives::views::ActionView::DeleteAccount { .. } => { @@ -225,24 +228,3 @@ pub(crate) async fn handle_access_keys( Ok(()) } - -pub(crate) async fn store_access_keys_from_genesis( - pool: Database, - access_keys_models: Vec, -) -> anyhow::Result<()> { - info!( - target: crate::INDEXER_FOR_EXPLORER, - "Adding/updating access keys from genesis..." - ); - - crate::await_retry_or_panic!( - diesel::insert_into(schema::access_keys::table) - .values(access_keys_models.clone()) - .on_conflict_do_nothing() - .execute_async(&pool), - 10, - "AccessKeys were stored from genesis".to_string(), - &access_keys_models - ); - Ok(()) -} diff --git a/src/db_adapters/accounts.rs b/src/db_adapters/accounts.rs index d93dd18d..01a72daa 100644 --- a/src/db_adapters/accounts.rs +++ b/src/db_adapters/accounts.rs @@ -1,14 +1,9 @@ use std::collections::HashMap; -use std::convert::TryFrom; use actix_diesel::dsl::AsyncRunQueryDsl; -use actix_diesel::Database; -use anyhow::Context; -use bigdecimal::BigDecimal; -use diesel::{BoolExpressionMethods, ExpressionMethods, PgConnection, QueryDsl}; -use futures::try_join; -use tracing::info; +use diesel::{ExpressionMethods, PgConnection, QueryDsl}; +use futures::try_join; use crate::models; use crate::schema; @@ -33,11 +28,17 @@ pub(crate) async fn handle_accounts( }) .map(|outcome_with_receipt| &outcome_with_receipt.receipt); - let mut accounts = - HashMap::::new(); + let mut accounts = HashMap::< + near_lake_framework::near_indexer_primitives::types::AccountId, + models::accounts::Account, + >::new(); for receipt in successful_receipts { - if let near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { actions, .. } = &receipt.receipt { + if let near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { + actions, + .. + } = &receipt.receipt + { for action in actions { match action { near_lake_framework::near_indexer_primitives::views::ActionView::CreateAccount => { @@ -198,77 +199,3 @@ pub(crate) async fn handle_accounts( try_join!(delete_accounts_future, create_or_update_accounts_future)?; Ok(()) } - -pub(crate) async fn store_accounts_from_genesis( - pool: Database, - accounts_models: Vec, -) -> anyhow::Result<()> { - info!( - target: crate::INDEXER_FOR_EXPLORER, - "Adding/updating accounts from genesis..." - ); - - crate::await_retry_or_panic!( - diesel::insert_into(schema::accounts::table) - .values(accounts_models.clone()) - .on_conflict_do_nothing() - .execute_async(&pool), - 10, - "Accounts were stored from genesis".to_string(), - &accounts_models - ); - - Ok(()) -} - -pub(crate) async fn get_lockup_account_ids_at_block_height( - pool: &actix_diesel::Database, - block_height: &near_lake_framework::near_indexer_primitives::types::BlockHeight, -) -> anyhow::Result> { - // Diesel does not support named joins - // https://github.com/diesel-rs/diesel/pull/2254 - // Raw SQL (diesel-1.4.7/src/query_builder/functions.rs:464) does not support async methods - // So we decided to use view + simple SQL with `where` clause - // Initial SQL statement: - // let raw_sql: String = format!(" - // SELECT accounts.account_id, blocks_start.block_height, blocks_end.block_height - // FROM accounts - // LEFT JOIN receipts AS receipts_start ON accounts.created_by_receipt_id = receipts_start.receipt_id - // LEFT JOIN blocks AS blocks_start ON receipts_start.included_in_block_hash = blocks_start.block_hash - // LEFT JOIN receipts AS receipts_end ON accounts.deleted_by_receipt_id = receipts_end.receipt_id - // LEFT JOIN blocks AS blocks_end ON receipts_end.included_in_block_hash = blocks_end.block_hash - // WHERE accounts.account_id like '%.lockup.near' - // AND (blocks_start.block_height IS NULL OR blocks_start.block_height <= {0}) - // AND (blocks_end.block_height IS NULL OR blocks_end.block_height >= {0}); - // ", block_height); - - schema::aggregated__lockups::table - .select(schema::aggregated__lockups::dsl::account_id) - .filter( - schema::aggregated__lockups::dsl::creation_block_height - .is_null() - .or(schema::aggregated__lockups::dsl::creation_block_height - .le(BigDecimal::from(*block_height))), - ) - .filter( - schema::aggregated__lockups::dsl::deletion_block_height - .is_null() - .or(schema::aggregated__lockups::dsl::deletion_block_height - .ge(BigDecimal::from(*block_height))), - ) - .get_results_async::(pool) - .await - .with_context(|| format!( - "DB error while collecting lockup account ids for block_height {}", - block_height - ) - ) - .map(|results| { - results - .into_iter() - .map(|account_id_string| - near_lake_framework::near_indexer_primitives::types::AccountId::try_from(account_id_string) - .expect("Selecting lockup account ids bumped into the account_id which is not valid; that should never happen")) - .collect() - }) -} diff --git a/src/db_adapters/aggregated/circulating_supply.rs b/src/db_adapters/aggregated/circulating_supply.rs deleted file mode 100644 index 9842ee29..00000000 --- a/src/db_adapters/aggregated/circulating_supply.rs +++ /dev/null @@ -1,61 +0,0 @@ -use actix_diesel::dsl::AsyncRunQueryDsl; -use bigdecimal::BigDecimal; -use diesel::{ExpressionMethods, PgConnection, QueryDsl}; -use tracing::error; - -use crate::models::aggregated::circulating_supply::CirculatingSupply; -use crate::schema; - -pub(crate) async fn add_circulating_supply( - pool: &actix_diesel::Database, - stats: &CirculatingSupply, -) { - let mut interval = crate::INTERVAL; - loop { - match diesel::insert_into(schema::aggregated__circulating_supply::table) - .values(stats.to_owned()) - .on_conflict_do_nothing() - .execute_async(pool) - .await - { - Ok(_) => { - break; - } - Err(async_error) => { - error!( - target: crate::AGGREGATED, - "Error occurred while Circulating Supply was adding to database. Retrying in {} milliseconds... \n {:#?}", - interval.as_millis(), - async_error - ); - tokio::time::sleep(interval).await; - if interval < crate::MAX_DELAY_TIME { - interval *= 2; - } - } - } - } -} - -pub(crate) async fn get_precomputed_circulating_supply_for_timestamp( - pool: &actix_diesel::Database, - timestamp: u64, -) -> anyhow::Result> { - let supply = schema::aggregated__circulating_supply::table - .select(schema::aggregated__circulating_supply::dsl::circulating_tokens_supply) - .filter( - schema::aggregated__circulating_supply::dsl::computed_at_block_timestamp - .eq(BigDecimal::from(timestamp)), - ) - .get_optional_result_async::(pool) - .await; - - match supply { - Ok(Some(value)) => match value.to_string().parse::() { - Ok(res) => Ok(Some(res)), - Err(_) => anyhow::bail!("`circulating_tokens_supply` expected to be u128"), - }, - Ok(None) => Ok(None), - Err(err) => anyhow::bail!("DB Error: {}", err), - } -} diff --git a/src/db_adapters/aggregated/mod.rs b/src/db_adapters/aggregated/mod.rs deleted file mode 100644 index eecd50a6..00000000 --- a/src/db_adapters/aggregated/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) mod circulating_supply; diff --git a/src/db_adapters/assets/events.rs b/src/db_adapters/assets/events.rs index 51b84b33..40f1184b 100644 --- a/src/db_adapters/assets/events.rs +++ b/src/db_adapters/assets/events.rs @@ -8,7 +8,7 @@ use super::event_types; pub(crate) async fn store_events( pool: &Database, - streamer_message: &near_lake_framework::near_indexer_primitives:: StreamerMessage, + streamer_message: &near_lake_framework::near_indexer_primitives::StreamerMessage, ) -> anyhow::Result<()> { let futures = streamer_message.shards.iter().map(|shard| { collect_and_store_events(pool, shard, streamer_message.block.header.timestamp) diff --git a/src/db_adapters/assets/fungible_token_events.rs b/src/db_adapters/assets/fungible_token_events.rs index 9a55b608..4ff917d0 100644 --- a/src/db_adapters/assets/fungible_token_events.rs +++ b/src/db_adapters/assets/fungible_token_events.rs @@ -29,7 +29,7 @@ pub(crate) async fn store_ft_events( 10, "FungibleTokenEvent were adding to database".to_string(), &ft_events_chunk, - &detect_ft_db_error + detect_ft_db_error ); } diff --git a/src/db_adapters/assets/non_fungible_token_events.rs b/src/db_adapters/assets/non_fungible_token_events.rs index 7e6bee0e..4b8b303b 100644 --- a/src/db_adapters/assets/non_fungible_token_events.rs +++ b/src/db_adapters/assets/non_fungible_token_events.rs @@ -29,7 +29,7 @@ pub(crate) async fn store_nft_events( 10, "NonFungibleTokenEvent were adding to database".to_string(), &nft_events_chunk, - &detect_nft_db_error + detect_nft_db_error ); } diff --git a/src/db_adapters/blocks.rs b/src/db_adapters/blocks.rs index f2a22653..ddd9518c 100644 --- a/src/db_adapters/blocks.rs +++ b/src/db_adapters/blocks.rs @@ -1,5 +1,4 @@ use actix_diesel::dsl::AsyncRunQueryDsl; -use anyhow::Context; use bigdecimal::{BigDecimal, ToPrimitive}; use diesel::{ExpressionMethods, PgConnection, QueryDsl}; @@ -28,26 +27,13 @@ pub(crate) async fn store_block( /// Gets the latest block's height from database pub(crate) async fn latest_block_height( pool: &actix_diesel::Database, -) -> Result, String> { +) -> anyhow::Result> { tracing::debug!(target: crate::INDEXER_FOR_EXPLORER, "fetching latest"); Ok(schema::blocks::table .select((schema::blocks::dsl::block_height,)) .order(schema::blocks::dsl::block_height.desc()) .limit(1) - .get_optional_result_async::<(bigdecimal::BigDecimal,)>(pool) - .await - .map_err(|err| format!("DB Error: {}", err))? + .get_optional_result_async::<(BigDecimal,)>(pool) + .await? .and_then(|(block_height,)| block_height.to_u64())) } - -pub(crate) async fn get_latest_block_before_timestamp( - pool: &actix_diesel::Database, - timestamp: u64, -) -> anyhow::Result { - Ok(schema::blocks::table - .filter(schema::blocks::dsl::block_timestamp.le(BigDecimal::from(timestamp))) - .order(schema::blocks::dsl::block_timestamp.desc()) - .first_async::(pool) - .await - .context("DB Error")?) -} diff --git a/src/db_adapters/genesis.rs b/src/db_adapters/genesis.rs deleted file mode 100644 index 8dfd941d..00000000 --- a/src/db_adapters/genesis.rs +++ /dev/null @@ -1,129 +0,0 @@ -use actix_diesel::Database; -use diesel::PgConnection; - -use crate::db_adapters::access_keys::store_access_keys_from_genesis; -use crate::db_adapters::accounts::store_accounts_from_genesis; - -/// This is an ugly hack that allows to execute an async body on a specified actix runtime. -/// You should only call it from a separate thread! -/// -/// ```ignore -/// async fn some_async_function() { -/// let current_actix_system = actix::System::current(); -/// tokio::tasks::spawn_blocking(move || { -/// let x = vec![0, 1, 2]; -/// x.map(|i| { -/// block_on(current_actix_system, async move { -/// reqwest::get(...).await.text().await -/// }) -/// }); -/// } -/// } -fn block_on( - actix_arbiter: &actix_rt::ArbiterHandle, - f: Fut, -) -> Result -where - T: Send + 'static, - Fut: std::future::Future + Send + 'static, -{ - let (tx, rx) = std::sync::mpsc::channel(); - actix_arbiter.spawn(async move { - let result = f.await; - let _ = tx.send(result); - }); - rx.recv() -} - -/// Iterates over GenesisRecords and stores selected ones (Accounts, AccessKeys) -/// to database. -/// Separately stores records divided in portions by 5000 to optimize -/// memory usage and minimize database queries -pub(crate) async fn store_genesis_records( - pool: Database, - near_config: near_indexer::NearConfig, -) -> anyhow::Result<()> { - tracing::info!( - target: crate::INDEXER_FOR_EXPLORER, - "Storing genesis records to database...", - ); - let genesis_height = near_config.genesis.config.genesis_height; - - // Remember the current actix runtime thread in order to be able to - // schedule async function on it from the thread that processes genesis in - // a blocking way. - let actix_system = actix::System::current(); - // Spawn the blocking genesis processing on a separate thread - tokio::task::spawn_blocking(move || { - let actix_arbiter = actix_system.arbiter(); - - let mut accounts_to_store: Vec = vec![]; - let mut access_keys_to_store: Vec = vec![]; - - near_config.genesis.for_each_record(|record| { - if accounts_to_store.len() == 5_000 { - let mut accounts_to_store_chunk = vec![]; - std::mem::swap(&mut accounts_to_store, &mut accounts_to_store_chunk); - let pool = pool.clone(); - block_on( - actix_arbiter, - store_accounts_from_genesis(pool, accounts_to_store_chunk), - ) - .expect("storing accounts from genesis failed") - .expect("storing accounts from genesis failed"); - } - if access_keys_to_store.len() == 5_000 { - let mut access_keys_to_store_chunk = vec![]; - std::mem::swap(&mut access_keys_to_store, &mut access_keys_to_store_chunk); - let pool = pool.clone(); - block_on( - actix_arbiter, - store_access_keys_from_genesis(pool, access_keys_to_store_chunk), - ) - .expect("storing access keys from genesis failed") - .expect("storing access keys from genesis failed"); - } - - match record { - near_primitives::state_record::StateRecord::Account { - account_id, - .. - } => { - accounts_to_store.push(crate::models::accounts::Account::new_from_genesis( - account_id, - genesis_height, - )); - } - near_primitives::state_record::StateRecord::AccessKey { - account_id, - public_key, - access_key, - } => { - access_keys_to_store.push(crate::models::access_keys::AccessKey::from_genesis( - public_key, - account_id, - access_key, - genesis_height, - )); - } - _ => {} - }; - }); - - let fut = || async move { - store_accounts_from_genesis(pool.clone(), accounts_to_store).await?; - store_access_keys_from_genesis(pool, access_keys_to_store).await?; - anyhow::Result::<()>::Ok(()) - }; - block_on(actix_arbiter, fut()) - .expect("storing leftover accounts and access keys from genesis failed") - .expect("storing leftover accounts and access keys from genesis failed"); - }) - .await?; - - tracing::info!( - target: crate::INDEXER_FOR_EXPLORER, - "Genesis records has been stored.", - ); - Ok(()) -} diff --git a/src/db_adapters/mod.rs b/src/db_adapters/mod.rs index 13177b1e..b8684dc6 100644 --- a/src/db_adapters/mod.rs +++ b/src/db_adapters/mod.rs @@ -1,12 +1,10 @@ pub(crate) mod access_keys; pub(crate) mod account_changes; pub(crate) mod accounts; -pub(crate) mod aggregated; pub(crate) mod assets; pub(crate) mod blocks; pub(crate) mod chunks; pub(crate) mod execution_outcomes; -//pub(crate) mod genesis; pub(crate) mod receipts; pub(crate) mod transactions; diff --git a/src/db_adapters/receipts.rs b/src/db_adapters/receipts.rs index 48393455..d327b8c4 100644 --- a/src/db_adapters/receipts.rs +++ b/src/db_adapters/receipts.rs @@ -51,8 +51,9 @@ async fn store_chunk_receipts( strict_mode: bool, receipts_cache: crate::ReceiptsCache, ) -> anyhow::Result<()> { - let mut skipping_receipt_ids = - std::collections::HashSet::::new(); + let mut skipping_receipt_ids = std::collections::HashSet::< + near_lake_framework::near_indexer_primitives::CryptoHash, + >::new(); let tx_hashes_for_receipts = find_tx_hashes_for_receipts( pool, @@ -172,7 +173,9 @@ async fn find_tx_hashes_for_receipts( // add receipt-transaction pairs from the cache to the response tx_hashes_for_receipts.extend(receipts.iter().filter_map(|receipt| { match receipt.receipt { - near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { .. } => receipts_cache_lock + near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { + .. + } => receipts_cache_lock .cache_get(&crate::ReceiptOrDataId::ReceiptId(receipt.receipt_id)) .map(|parent_transaction_hash| { ( @@ -180,7 +183,10 @@ async fn find_tx_hashes_for_receipts( parent_transaction_hash.clone(), ) }), - near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Data { data_id, .. } => { + near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Data { + data_id, + .. + } => { // Pair DataId:ParentTransactionHash won't be used after this moment // We want to clean it up to prevent our cache from growing receipts_cache_lock @@ -199,9 +205,10 @@ async fn find_tx_hashes_for_receipts( // discard the Receipts already in cache from the attempts to search receipts.retain(|r| match r.receipt { - near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Data { data_id, .. } => { - !tx_hashes_for_receipts.contains_key(&crate::ReceiptOrDataId::DataId(data_id)) - } + near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Data { + data_id, + .. + } => !tx_hashes_for_receipts.contains_key(&crate::ReceiptOrDataId::DataId(data_id)), near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { .. } => { !tx_hashes_for_receipts.contains_key(&crate::ReceiptOrDataId::ReceiptId(r.receipt_id)) } @@ -223,9 +230,10 @@ async fn find_tx_hashes_for_receipts( let data_ids: Vec = receipts .iter() .filter_map(|r| match r.receipt { - near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Data { data_id, .. } => { - Some(data_id.to_string()) - } + near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Data { + data_id, + .. + } => Some(data_id.to_string()), _ => None, }) .collect(); @@ -295,7 +303,8 @@ async fn find_tx_hashes_for_receipts( crate::ParentTransactionHashString, )> = receipts .iter() - .filter_map(|r| match r.receipt { + .filter_map(|r| { + match r.receipt { near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Data { data_id, .. } => tx_hashes_for_data_id_via_data_output_hashmap @@ -307,6 +316,7 @@ async fn find_tx_hashes_for_receipts( ) }), _ => None, + } }) .collect(); @@ -362,8 +372,10 @@ async fn find_tx_hashes_for_receipts( |(receipt_id_string, transaction_hash_string)| { ( crate::ReceiptOrDataId::ReceiptId( - near_lake_framework::near_indexer_primitives::CryptoHash::from_str(&receipt_id_string) - .expect("Failed to convert String to CryptoHash"), + near_lake_framework::near_indexer_primitives::CryptoHash::from_str( + &receipt_id_string, + ) + .expect("Failed to convert String to CryptoHash"), ), transaction_hash_string, ) @@ -412,8 +424,10 @@ async fn find_tx_hashes_for_receipts( |(receipt_id_string, transaction_hash_string)| { ( crate::ReceiptOrDataId::ReceiptId( - near_lake_framework::near_indexer_primitives::CryptoHash::from_str(&receipt_id_string) - .expect("Failed to convert String to CryptoHash"), + near_lake_framework::near_indexer_primitives::CryptoHash::from_str( + &receipt_id_string, + ) + .expect("Failed to convert String to CryptoHash"), ), transaction_hash_string, ) @@ -482,7 +496,8 @@ async fn store_receipt_actions( .iter() .filter_map(|receipt| { if let near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { - actions, .. + actions, + .. } = &receipt.receipt { Some(actions.iter().enumerate().map(move |(index, action)| { diff --git a/src/db_adapters/transactions.rs b/src/db_adapters/transactions.rs index 1523e9bd..3e794fc3 100644 --- a/src/db_adapters/transactions.rs +++ b/src/db_adapters/transactions.rs @@ -24,11 +24,10 @@ pub(crate) async fn store_transactions( tried_to_insert_transactions_count += chunk.transactions.len(); store_chunk_transactions( pool, - chunk - .transactions - .iter() - .enumerate() - .collect::>(), + chunk.transactions.iter().enumerate().collect::>(), &chunk.header.chunk_hash, block_hash, block_timestamp, @@ -88,17 +87,20 @@ async fn collect_converted_to_receipt_ids( pool: &actix_diesel::Database, block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, ) -> anyhow::Result> { - Ok(schema::transactions::table + schema::transactions::table .select(schema::transactions::dsl::converted_into_receipt_id) .filter(schema::transactions::dsl::included_in_block_hash.eq(block_hash.to_string())) .get_results_async::(pool) .await - .context("DB Error")?) + .context("DB Error") } async fn store_chunk_transactions( pool: &actix_diesel::Database, - transactions: Vec<(usize, &near_lake_framework::near_indexer_primitives::IndexerTransactionWithOutcome)>, + transactions: Vec<( + usize, + &near_lake_framework::near_indexer_primitives::IndexerTransactionWithOutcome, + )>, chunk_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, block_timestamp: u64, diff --git a/src/main.rs b/src/main.rs index fc93fd77..adf17149 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,20 +1,16 @@ use clap::Parser; -use std::convert::TryFrom; #[macro_use] extern crate diesel; -use actix_diesel::Database; pub use cached::SizedCache; use diesel::PgConnection; use futures::future::try_join_all; use futures::{try_join, StreamExt}; -use tokio::sync::{mpsc, Mutex}; -use tracing::{debug, info, warn}; -use tracing_subscriber::EnvFilter; +use tokio::sync::Mutex; +use tracing::{debug, info}; use crate::configs::Opts; -// mod aggregated; mod configs; mod db_adapters; mod models; @@ -24,7 +20,7 @@ mod retriable; // Categories for logging const INDEXER_FOR_EXPLORER: &str = "indexer_for_explorer"; -const AGGREGATED: &str = "aggregated"; +// const AGGREGATED: &str = "aggregated"; const INTERVAL: std::time::Duration = std::time::Duration::from_millis(100); const MAX_DELAY_TIME: std::time::Duration = std::time::Duration::from_secs(120); @@ -150,29 +146,27 @@ async fn handle_message( assets_events_future )?; } + Ok(()) } -async fn listen_blocks( - stream: mpsc::Receiver, - pool: Database, - concurrency: std::num::NonZeroU16, - strict_mode: bool, - stop_after_number_of_blocks: Option, -) { - if let Some(stop_after_n_blocks) = stop_after_number_of_blocks { - warn!( - target: crate::INDEXER_FOR_EXPLORER, - "Indexer will stop after indexing {} blocks", stop_after_n_blocks, - ); - } - if !strict_mode { - warn!( - target: crate::INDEXER_FOR_EXPLORER, - "Indexer is starting in NON-STRICT mode", - ); - } - info!(target: crate::INDEXER_FOR_EXPLORER, "Stream has started"); +#[actix::main] +async fn main() -> anyhow::Result<()> { + // We use it to automatically search the for root certificates to perform HTTPS calls + // (sending telemetry and downloading genesis) + openssl_probe::init_ssl_cert_env_vars(); + + dotenv::dotenv().ok(); + + let opts: Opts = Opts::parse(); + + configs::init_tracing(opts.debug)?; + + // We establish connection as early as possible as an additional sanity check. + // Indexer should fail if .env file with credentials is missing/wrong + let pool = models::establish_connection(&opts.database_url); + + let strict_mode = !opts.non_strict_mode; // We want to prevent unnecessary SELECT queries to the database to find // the Transaction hash for the Receipt. @@ -182,8 +176,15 @@ async fn listen_blocks( let receipts_cache: ReceiptsCache = std::sync::Arc::new(Mutex::new(SizedCache::with_size(100_000))); - let handle_messages = - tokio_stream::wrappers::ReceiverStream::new(stream).map(|streamer_message| { + let config: near_lake_framework::LakeConfig = opts.to_lake_config().await; + let (sender, stream) = near_lake_framework::streamer(config); + + tracing::info!( + target: INDEXER_FOR_EXPLORER, + "Starting Indexer for Explorer (lake)...", + ); + let mut handlers = tokio_stream::wrappers::ReceiverStream::new(stream) + .map(|streamer_message| { info!( target: crate::INDEXER_FOR_EXPLORER, "Block height {}", &streamer_message.block.header.height @@ -194,102 +195,16 @@ async fn listen_blocks( strict_mode, std::sync::Arc::clone(&receipts_cache), ) - }); - let mut handle_messages = if let Some(stop_after_n_blocks) = stop_after_number_of_blocks { - handle_messages - .take(stop_after_n_blocks.get()) - .boxed_local() - } else { - handle_messages.boxed_local() - } - .buffer_unordered(usize::from(concurrency.get())); - - while let Some(_handled_message) = handle_messages.next().await {} - // Graceful shutdown - info!( - target: crate::INDEXER_FOR_EXPLORER, - "Indexer will be shutdown gracefully in 7 seconds...", - ); - drop(handle_messages); - tokio::time::sleep(std::time::Duration::from_secs(7)).await; -} - -fn main() -> anyhow::Result<()> { - // We use it to automatically search the for root certificates to perform HTTPS calls - // (sending telemetry and downloading genesis) - openssl_probe::init_ssl_cert_env_vars(); - - // We establish connection as early as possible as an additional sanity check. - // Indexer should fail if .env file with credentials is missing/wrong - let pool = models::establish_connection(); - - let opts: Opts = Opts::parse(); - - let mut env_filter = EnvFilter::new( - "tokio_reactor=info,near=info,stats=info,telemetry=info,indexer=info,aggregated=info,near_lake_framework=info", - ); + }) + .buffer_unordered(1usize); - if opts.debug { - env_filter = env_filter.add_directive( - "indexer_for_explorer=debug" - .parse() - .expect("Failed to parse directive"), - ); - } else { - env_filter = env_filter.add_directive( - "indexer_for_explorer=info" - .parse() - .expect("Failed to parse directive"), - ); - }; + while let Some(_handle_message) = handlers.next().await {} + drop(handlers); // close the channel so the sender will stop - if let Ok(rust_log) = std::env::var("RUST_LOG") { - if !rust_log.is_empty() { - for directive in rust_log.split(',').filter_map(|s| match s.parse() { - Ok(directive) => Some(directive), - Err(err) => { - eprintln!("Ignoring directive `{}`: {}", s, err); - None - } - }) { - env_filter = env_filter.add_directive(directive); - } - } + // propagate errors from the sender + match sender.await { + Ok(Ok(())) => Ok(()), + Ok(Err(e)) => Err(e), + Err(e) => Err(anyhow::Error::from(e)), // JoinError } - - tracing_subscriber::fmt::Subscriber::builder() - .with_env_filter(env_filter) - .with_writer(std::io::stderr) - .init(); - - let config = near_lake_framework::LakeConfigBuilder::default() - .s3_bucket_name(opts.s3_bucket_name.clone()) - .s3_region_name(opts.s3_region_name.clone()) - .start_block_height(opts.start_block_height) // want to start from the freshest - .build()?; - let system = actix::System::new(); - system.block_on(async move { - let (lake_handle, stream) = near_lake_framework::streamer(config); - - listen_blocks( - stream, - pool.clone(), - opts.concurrency, - !opts.non_strict_mode, - None, - ) - .await; - - actix::System::current().stop(); - - // propagate errors from the sender - match lake_handle.await { - Ok(Ok(())) => Ok(()), - Ok(Err(e)) => Err(e), - Err(e) => Err(anyhow::Error::from(e)), // JoinError - } - })?; - - system.run()?; - Ok(()) } diff --git a/src/models/account_changes.rs b/src/models/account_changes.rs index c69c8ab1..5ca69f78 100644 --- a/src/models/account_changes.rs +++ b/src/models/account_changes.rs @@ -27,8 +27,10 @@ impl AccountChange { changed_in_block_timestamp: u64, index_in_block: i32, ) -> Option { - let near_lake_framework::near_indexer_primitives::views::StateChangeWithCauseView { cause, value } = - state_change_with_cause; + let near_lake_framework::near_indexer_primitives::views::StateChangeWithCauseView { + cause, + value, + } = state_change_with_cause; let (account_id, account): ( String, diff --git a/src/models/enums.rs b/src/models/enums.rs index ccc32b76..eb849281 100644 --- a/src/models/enums.rs +++ b/src/models/enums.rs @@ -10,10 +10,16 @@ pub enum ReceiptKind { } impl From<&near_lake_framework::near_indexer_primitives::views::ReceiptEnumView> for ReceiptKind { - fn from(receipt_enum_view: &near_lake_framework::near_indexer_primitives::views::ReceiptEnumView) -> Self { + fn from( + receipt_enum_view: &near_lake_framework::near_indexer_primitives::views::ReceiptEnumView, + ) -> Self { match receipt_enum_view { - near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { .. } => Self::Action, - near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Data { .. } => Self::Data, + near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { + .. + } => Self::Action, + near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Data { + .. + } => Self::Data, } } } @@ -44,8 +50,12 @@ pub enum ExecutionOutcomeStatus { SuccessReceiptId, } -impl From for ExecutionOutcomeStatus { - fn from(status: near_lake_framework::near_indexer_primitives::views::ExecutionStatusView) -> Self { +impl From + for ExecutionOutcomeStatus +{ + fn from( + status: near_lake_framework::near_indexer_primitives::views::ExecutionStatusView, + ) -> Self { match status { near_lake_framework::near_indexer_primitives::views::ExecutionStatusView::Unknown => Self::Unknown, near_lake_framework::near_indexer_primitives::views::ExecutionStatusView::Failure(_) => Self::Failure, @@ -70,8 +80,12 @@ pub enum AccessKeyPermission { FunctionCall, } -impl From<&near_lake_framework::near_indexer_primitives::views::AccessKeyPermissionView> for AccessKeyPermission { - fn from(item: &near_lake_framework::near_indexer_primitives::views::AccessKeyPermissionView) -> Self { +impl From<&near_lake_framework::near_indexer_primitives::views::AccessKeyPermissionView> + for AccessKeyPermission +{ + fn from( + item: &near_lake_framework::near_indexer_primitives::views::AccessKeyPermissionView, + ) -> Self { match item { near_lake_framework::near_indexer_primitives::views::AccessKeyPermissionView::FunctionCall { .. @@ -86,12 +100,10 @@ impl From<&near_lake_framework::near_indexer_primitives::views::AccessKeyPermiss impl From<&near_primitives::account::AccessKeyPermission> for AccessKeyPermission { fn from(item: &near_primitives::account::AccessKeyPermission) -> Self { match item { - near_primitives::account::AccessKeyPermission::FunctionCall { - .. - } => Self::FunctionCall, - near_primitives::account::AccessKeyPermission::FullAccess => { - Self::FullAccess + near_primitives::account::AccessKeyPermission::FunctionCall { .. } => { + Self::FunctionCall } + near_primitives::account::AccessKeyPermission::FullAccess => Self::FullAccess, } } } @@ -112,7 +124,9 @@ pub enum StateChangeReasonKind { Resharding, } -impl From<&near_lake_framework::near_indexer_primitives::views::StateChangeCauseView> for StateChangeReasonKind { +impl From<&near_lake_framework::near_indexer_primitives::views::StateChangeCauseView> + for StateChangeReasonKind +{ fn from( state_change_cause_view: &near_lake_framework::near_indexer_primitives::views::StateChangeCauseView, ) -> Self { diff --git a/src/models/mod.rs b/src/models/mod.rs index 0e90295c..5f235c2e 100644 --- a/src/models/mod.rs +++ b/src/models/mod.rs @@ -1,7 +1,4 @@ -use std::env; - use diesel::PgConnection; -use dotenv::dotenv; pub use access_keys::AccessKey; pub use account_changes::AccountChange; @@ -20,7 +17,6 @@ pub(crate) use serializers::extract_action_type_and_value_from_action_view; pub mod access_keys; pub mod account_changes; pub mod accounts; -pub mod aggregated; pub mod assets; pub mod blocks; pub mod chunks; @@ -30,16 +26,8 @@ pub mod receipts; mod serializers; pub mod transactions; -/// Get database credentials from `.env` or fail -pub(crate) fn get_database_credentials() -> String { - dotenv().ok(); - - env::var("DATABASE_URL").expect("DATABASE_URL must be set in .env file") -} - -pub(crate) fn establish_connection() -> actix_diesel::Database { - let database_url = get_database_credentials(); +pub(crate) fn establish_connection(database_url: &str) -> actix_diesel::Database { actix_diesel::Database::builder() .pool_max_size(30) - .open(&database_url) + .open(database_url) } diff --git a/src/models/receipts.rs b/src/models/receipts.rs index 0f02c5c5..2eb5722e 100644 --- a/src/models/receipts.rs +++ b/src/models/receipts.rs @@ -62,8 +62,10 @@ impl TryFrom<&near_lake_framework::near_indexer_primitives::views::ReceiptView> fn try_from( receipt_view: &near_lake_framework::near_indexer_primitives::views::ReceiptView, ) -> Result { - if let near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Data { data_id, data } = - &receipt_view.receipt + if let near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Data { + data_id, + data, + } = &receipt_view.receipt { Ok(Self { receipt_id: receipt_view.receipt_id.to_string(), diff --git a/src/models/serializers.rs b/src/models/serializers.rs index a448ee3e..ac83cc54 100644 --- a/src/models/serializers.rs +++ b/src/models/serializers.rs @@ -1,8 +1,8 @@ use serde::{Deserialize, Serialize}; use serde_json::json; -use near_primitives::serialize::option_u128_dec_format; use near_lake_framework::near_indexer_primitives::views::ActionView; +use near_primitives::serialize::option_u128_dec_format; use crate::models::enums::ActionKind; @@ -15,7 +15,9 @@ pub(crate) struct AccessKeyView { } impl From<&near_lake_framework::near_indexer_primitives::views::AccessKeyView> for AccessKeyView { - fn from(access_key_view: &near_lake_framework::near_indexer_primitives::views::AccessKeyView) -> Self { + fn from( + access_key_view: &near_lake_framework::near_indexer_primitives::views::AccessKeyView, + ) -> Self { Self { nonce: access_key_view.nonce, permission: access_key_view.permission.clone().into(), @@ -44,7 +46,9 @@ pub(crate) enum AccessKeyPermissionView { impl From for AccessKeyPermissionView { - fn from(permission: near_lake_framework::near_indexer_primitives::views::AccessKeyPermissionView) -> Self { + fn from( + permission: near_lake_framework::near_indexer_primitives::views::AccessKeyPermissionView, + ) -> Self { match permission { near_lake_framework::near_indexer_primitives::views::AccessKeyPermissionView::FullAccess => { Self::FullAccess From 52d5046c9610f91307835a90c39ba91ffa80b63a Mon Sep 17 00:00:00 2001 From: Bohdan Khorolets Date: Fri, 19 Aug 2022 14:41:28 +0300 Subject: [PATCH 06/42] chore: Update test.yml (#3) * Update test.yml * update rust version * update ubuntu version in the workflows * add libpq-dev to cross Dockerfile --- .github/workflows/release.yml | 2 +- .github/workflows/test.yml | 8 ++++---- Cargo.toml | 1 + cross/x86_64-unknown-linux-gnu/Dockerfile | 1 + rust-toolchain | 1 - 5 files changed, 7 insertions(+), 6 deletions(-) delete mode 100644 rust-toolchain diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 831ee039..183435eb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -59,7 +59,7 @@ jobs: matrix: include: - build: linux-x86_64 - os: ubuntu-18.04 + os: ubuntu-latest rust: stable target: x86_64-unknown-linux-gnu diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3a44e32e..e142a235 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,9 +1,9 @@ name: ci on: pull_request: - branches: [main] + branches: [master-lake] push: - branches: [main] + branches: [master-lake] # schedule: # - cron: '00 01 * * *' jobs: @@ -27,7 +27,7 @@ jobs: matrix: include: - build: linux-x86_64 - os: ubuntu-18.04 + os: ubuntu-latest rust: stable target: x86_64-unknown-linux-gnu @@ -69,7 +69,7 @@ jobs: rustfmt: name: rustfmt - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@v2 diff --git a/Cargo.toml b/Cargo.toml index 160ef076..8f985d97 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ name = "indexer-explorer-lake" version = "0.10.0" authors = ["Near Inc "] edition = "2021" +rust-version = "1.62.1" [dependencies] actix = "0.13.0" diff --git a/cross/x86_64-unknown-linux-gnu/Dockerfile b/cross/x86_64-unknown-linux-gnu/Dockerfile index 87bfb42a..089e4a06 100644 --- a/cross/x86_64-unknown-linux-gnu/Dockerfile +++ b/cross/x86_64-unknown-linux-gnu/Dockerfile @@ -9,6 +9,7 @@ RUN apt-get update -qq && \ g++ \ pkg-config \ libssl-dev \ + libpq-dev \ curl \ llvm \ clang diff --git a/rust-toolchain b/rust-toolchain deleted file mode 100644 index 91951fd8..00000000 --- a/rust-toolchain +++ /dev/null @@ -1 +0,0 @@ -1.61.0 From 118e55e0179cadbfbe524666f492f57808f928fe Mon Sep 17 00:00:00 2001 From: Bohdan Khorolets Date: Thu, 15 Sep 2022 13:56:51 +0300 Subject: [PATCH 07/42] chore: Upgrade near-lake-framework to 0.5.2 (#4) --- CHANGELOG.md | 4 ++++ Cargo.lock | 6 +++--- Cargo.toml | 4 ++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55753126..e19cf752 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.10.1 + +* Upgrade `near-lake-framework` to `0.5.2` + ## 0.10.0 * Upgrade `near-lake-framework` to `0.5.1` diff --git a/Cargo.lock b/Cargo.lock index f851fc44..689de8fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1582,7 +1582,7 @@ dependencies = [ [[package]] name = "indexer-explorer-lake" -version = "0.10.0" +version = "0.10.1" dependencies = [ "actix", "actix-diesel", @@ -1925,9 +1925,9 @@ dependencies = [ [[package]] name = "near-lake-framework" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e64c96682214091519e78f991cb17c1991f0c1b32d2f9bd9977fe71ede4e6e5" +checksum = "5984129964c79102a489ccd9f2af31b4bb8e9f78b0e757f480cb13d426e1915f" dependencies = [ "anyhow", "async-stream", diff --git a/Cargo.toml b/Cargo.toml index 8f985d97..bfc003d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "indexer-explorer-lake" -version = "0.10.0" +version = "0.10.1" authors = ["Near Inc "] edition = "2021" rust-version = "1.62.1" @@ -39,6 +39,6 @@ uint = { version = "0.8.3", default-features = false } actix-diesel = { git = "https://github.com/frol/actix-diesel", rev = "3a001986c89dfabfc3c448d8bae28525101b4992" } near-jsonrpc-client = "0.4.0-beta.0" -near-lake-framework = "0.5.1" +near-lake-framework = "0.5.2" near-primitives = "0.14" near-crypto = "0.14" From 4575c7f91f9a6c23e669ea1e7e183810966cd137 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Tue, 18 Oct 2022 02:40:00 +1300 Subject: [PATCH 08/42] fix: Avoid recreating access key on transfer to implicit account (#5) * fix: Avoid recreating access key on transfer to implicit account * refactor: Remove `AccountDeletion` logic as its also covered by `AccessKeyDeletion` * chore: Update `CHANGELOG.md` * refactor: use `near_lake_framework` types rather than `near_primitives` --- CHANGELOG.md | 4 + src/db_adapters/access_keys.rs | 172 +++++++++------------------------ src/main.rs | 2 +- 3 files changed, 49 insertions(+), 129 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e19cf752..5d888805 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.10.2 + +* Avoid recreating access key on transfer to implicit account + ## 0.10.1 * Upgrade `near-lake-framework` to `0.5.2` diff --git a/src/db_adapters/access_keys.rs b/src/db_adapters/access_keys.rs index 7adfa013..1a75542f 100644 --- a/src/db_adapters/access_keys.rs +++ b/src/db_adapters/access_keys.rs @@ -1,9 +1,7 @@ use std::collections::HashMap; -use std::convert::TryFrom; use actix_diesel::dsl::AsyncRunQueryDsl; -use bigdecimal::BigDecimal; use diesel::{ExpressionMethods, PgConnection, QueryDsl}; use futures::try_join; @@ -12,108 +10,58 @@ use crate::schema; pub(crate) async fn handle_access_keys( pool: &actix_diesel::Database, - outcomes: &[near_lake_framework::near_indexer_primitives::IndexerExecutionOutcomeWithReceipt], + state_changes: &[near_lake_framework::near_indexer_primitives::views::StateChangeWithCauseView], block_height: near_lake_framework::near_indexer_primitives::types::BlockHeight, ) -> anyhow::Result<()> { - if outcomes.is_empty() { + if state_changes.is_empty() { return Ok(()); } - let successful_receipts = outcomes - .iter() - .filter(|outcome_with_receipt| { - matches!( - outcome_with_receipt.execution_outcome.outcome.status, - near_lake_framework::near_indexer_primitives::views::ExecutionStatusView::SuccessValue(_) - | near_lake_framework::near_indexer_primitives::views::ExecutionStatusView::SuccessReceiptId(_) - ) - }) - .map(|outcome_with_receipt| &outcome_with_receipt.receipt); let mut access_keys = HashMap::<(String, String), models::access_keys::AccessKey>::new(); - let mut deleted_accounts = HashMap::::new(); - for receipt in successful_receipts { - if let near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { - actions, - .. - } = &receipt.receipt + for state_change in state_changes { + if let near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::ReceiptProcessing { receipt_hash } = + state_change.cause { - for action in actions { - match action { - near_lake_framework::near_indexer_primitives::views::ActionView::DeleteAccount { .. } => { - deleted_accounts.insert( - receipt.receiver_id.to_string(), - receipt.receipt_id.to_string(), - ); - access_keys - .iter_mut() - .filter(|((_, receiver_id), _)| { - receiver_id == receipt.receiver_id.as_ref() - }) - .for_each(|(_, access_key)| { - access_key.deleted_by_receipt_id = - Some(receipt.receipt_id.to_string()); - }); - } - near_lake_framework::near_indexer_primitives::views::ActionView::AddKey { - public_key, - access_key, - } => { - access_keys.insert( - (public_key.to_string(), receipt.receiver_id.to_string()), - models::access_keys::AccessKey::from_action_view( - public_key, - &receipt.receiver_id, - access_key, - &receipt.receipt_id, - block_height, - ), - ); - } - near_lake_framework::near_indexer_primitives::views::ActionView::DeleteKey { public_key } => { - access_keys - .entry((public_key.to_string(), receipt.receiver_id.to_string())) - .and_modify(|existing_access_key| { - existing_access_key.deleted_by_receipt_id = - Some(receipt.receipt_id.to_string()); - }) - .or_insert_with(|| models::access_keys::AccessKey { - public_key: public_key.to_string(), - account_id: receipt.receiver_id.to_string(), - created_by_receipt_id: None, - deleted_by_receipt_id: Some(receipt.receipt_id.to_string()), - // this is a workaround to avoid additional struct with optional field - // permission_kind is not supposed to change on delete action - permission_kind: models::enums::AccessKeyPermission::FullAccess, - last_update_block_height: block_height.into(), - }); - } - near_lake_framework::near_indexer_primitives::views::ActionView::Transfer { .. } => { - if receipt.receiver_id.len() != 64usize { - continue; - } - if let Ok(public_key_bytes) = hex::decode(receipt.receiver_id.as_ref()) { - if let Ok(public_key) = - near_crypto::ED25519PublicKey::try_from(&public_key_bytes[..]) - { - access_keys.insert( - (near_crypto::PublicKey::from(public_key.clone()).to_string(), receipt.receiver_id.to_string()), - models::access_keys::AccessKey::from_action_view( - &near_crypto::PublicKey::from(public_key.clone()), - &receipt.receiver_id, - &near_lake_framework::near_indexer_primitives::views::AccessKeyView { - nonce: 0, - permission: near_lake_framework::near_indexer_primitives::views::AccessKeyPermissionView::FullAccess - }, - &receipt.receipt_id, - block_height, - ), - ); - } - } - } - _ => continue, + match &state_change.value { + near_lake_framework::near_indexer_primitives::views::StateChangeValueView::AccessKeyUpdate { + account_id, + public_key, + access_key, + } => { + access_keys.insert( + (public_key.to_string(), account_id.to_string()), + models::access_keys::AccessKey::from_action_view( + public_key, + &account_id, + access_key, + &receipt_hash, + block_height, + ), + ); } + near_lake_framework::near_indexer_primitives::views::StateChangeValueView::AccessKeyDeletion { + account_id, + public_key, + } => { + access_keys + .entry((public_key.to_string(), account_id.to_string())) + .and_modify(|existing_access_key| { + existing_access_key.deleted_by_receipt_id = + Some(receipt_hash.to_string()); + }) + .or_insert_with(|| models::access_keys::AccessKey { + public_key: public_key.to_string(), + account_id: account_id.to_string(), + created_by_receipt_id: None, + deleted_by_receipt_id: Some(receipt_hash.to_string()), + // this is a workaround to avoid additional struct with optional field + // permission_kind is not supposed to change on delete action + permission_kind: models::enums::AccessKeyPermission::FullAccess, + last_update_block_height: block_height.into(), + }); + } + _ => continue, } } } @@ -126,34 +74,6 @@ pub(crate) async fn handle_access_keys( .cloned() .partition(|model| model.created_by_receipt_id.is_some()); - let delete_access_keys_for_deleted_accounts = async { - let last_update_block_height: BigDecimal = block_height.into(); - for (account_id, deleted_by_receipt_id) in deleted_accounts { - let target = schema::access_keys::table - .filter(schema::access_keys::dsl::deleted_by_receipt_id.is_null()) - .filter( - schema::access_keys::dsl::last_update_block_height - .lt(last_update_block_height.clone()), - ) - .filter(schema::access_keys::dsl::account_id.eq(account_id)); - - crate::await_retry_or_panic!( - diesel::update(target.clone()) - .set(( - schema::access_keys::dsl::deleted_by_receipt_id - .eq(deleted_by_receipt_id.clone()), - schema::access_keys::dsl::last_update_block_height - .eq(last_update_block_height.clone()), - )) - .execute_async(pool), - 10, - "AccessKeys were deleting".to_string(), - &deleted_by_receipt_id - ); - } - Ok(()) - }; - let update_access_keys_future = async { for value in access_keys_to_update { let target = schema::access_keys::table @@ -220,11 +140,7 @@ pub(crate) async fn handle_access_keys( Ok(()) }; - try_join!( - delete_access_keys_for_deleted_accounts, - update_access_keys_future, - add_access_keys_future - )?; + try_join!(update_access_keys_future, add_access_keys_future)?; Ok(()) } diff --git a/src/main.rs b/src/main.rs index adf17149..d118d1bf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -116,7 +116,7 @@ async fn handle_message( let futures = streamer_message.shards.iter().map(|shard| { db_adapters::access_keys::handle_access_keys( pool, - &shard.receipt_execution_outcomes, + &shard.state_changes, streamer_message.block.header.height, ) }); From 2473ddfc002e7eceb8af615a32e978ff74f2ae46 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Wed, 2 Nov 2022 19:26:12 +1300 Subject: [PATCH 09/42] build: Migrate to Cargo Workspaces (#6) * build: Create Cargo workspace and empty `indexer` crate * build: Migrate crate root to workspaces --- Cargo.toml | 48 ++----------------- CHANGELOG.md => indexer/CHANGELOG.md | 0 indexer/Cargo.toml | 44 +++++++++++++++++ diesel.toml => indexer/diesel.toml | 0 {migrations => indexer/migrations}/.gitkeep | 0 .../down.sql | 0 .../up.sql | 0 .../2020-12-07-153402_initial_schema/down.sql | 0 .../2020-12-07-153402_initial_schema/up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../2021-10-04-100000_assets_nft/down.sql | 0 .../2021-10-04-100000_assets_nft/up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../2022-01-12-100000_assets_ft/down.sql | 0 .../2022-01-12-100000_assets_ft/up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../up.sql | 0 {src => indexer/src}/configs.rs | 0 .../src}/db_adapters/access_keys.rs | 0 .../src}/db_adapters/account_changes.rs | 0 {src => indexer/src}/db_adapters/accounts.rs | 0 .../src}/db_adapters/assets/event_types.rs | 0 .../src}/db_adapters/assets/events.rs | 0 .../assets/fungible_token_events.rs | 0 .../src}/db_adapters/assets/mod.rs | 0 .../assets/non_fungible_token_events.rs | 0 {src => indexer/src}/db_adapters/blocks.rs | 0 {src => indexer/src}/db_adapters/chunks.rs | 0 .../src}/db_adapters/execution_outcomes.rs | 0 {src => indexer/src}/db_adapters/mod.rs | 0 {src => indexer/src}/db_adapters/receipts.rs | 0 .../src}/db_adapters/transactions.rs | 0 {src => indexer/src}/main.rs | 0 {src => indexer/src}/models/access_keys.rs | 0 .../src}/models/account_changes.rs | 0 {src => indexer/src}/models/accounts.rs | 0 .../models/aggregated/circulating_supply.rs | 0 {src => indexer/src}/models/aggregated/mod.rs | 0 .../models/assets/fungible_token_events.rs | 0 {src => indexer/src}/models/assets/mod.rs | 0 .../assets/non_fungible_token_events.rs | 0 {src => indexer/src}/models/blocks.rs | 0 {src => indexer/src}/models/chunks.rs | 0 {src => indexer/src}/models/enums.rs | 0 .../src}/models/execution_outcomes.rs | 0 {src => indexer/src}/models/mod.rs | 0 {src => indexer/src}/models/receipts.rs | 0 {src => indexer/src}/models/serializers.rs | 0 {src => indexer/src}/models/transactions.rs | 0 {src => indexer/src}/retriable.rs | 0 {src => indexer/src}/schema.patch | 0 {src => indexer/src}/schema.rs | 0 82 files changed, 48 insertions(+), 44 deletions(-) rename CHANGELOG.md => indexer/CHANGELOG.md (100%) create mode 100644 indexer/Cargo.toml rename diesel.toml => indexer/diesel.toml (100%) rename {migrations => indexer/migrations}/.gitkeep (100%) rename {migrations => indexer/migrations}/00000000000000_diesel_initial_setup/down.sql (100%) rename {migrations => indexer/migrations}/00000000000000_diesel_initial_setup/up.sql (100%) rename {migrations => indexer/migrations}/2020-12-07-153402_initial_schema/down.sql (100%) rename {migrations => indexer/migrations}/2020-12-07-153402_initial_schema/up.sql (100%) rename {migrations => indexer/migrations}/2021-01-14-170424_index-receipt-originated-from-transaction-hash/down.sql (100%) rename {migrations => indexer/migrations}/2021-01-14-170424_index-receipt-originated-from-transaction-hash/up.sql (100%) rename {migrations => indexer/migrations}/2021-01-20-152056_index-transactions-receiver-account-id/down.sql (100%) rename {migrations => indexer/migrations}/2021-01-20-152056_index-transactions-receiver-account-id/up.sql (100%) rename {migrations => indexer/migrations}/2021-03-11-123839_index-action-kind-and-status/down.sql (100%) rename {migrations => indexer/migrations}/2021-03-11-123839_index-action-kind-and-status/up.sql (100%) rename {migrations => indexer/migrations}/2021-04-22-090505_execution_outcomes_replace_chunk_relation_with_shard_id/down.sql (100%) rename {migrations => indexer/migrations}/2021-04-22-090505_execution_outcomes_replace_chunk_relation_with_shard_id/up.sql (100%) rename {migrations => indexer/migrations}/2021-04-28-154439_denormalize_action_receipt_actions/down.sql (100%) rename {migrations => indexer/migrations}/2021-04-28-154439_denormalize_action_receipt_actions/up.sql (100%) rename {migrations => indexer/migrations}/2021-05-06-093622_additional_indexes_for_action_receipt_actions/down.sql (100%) rename {migrations => indexer/migrations}/2021-05-06-093622_additional_indexes_for_action_receipt_actions/up.sql (100%) rename {migrations => indexer/migrations}/2021-05-07-115559_convert_args_base64_to_args_json/down.sql (100%) rename {migrations => indexer/migrations}/2021-05-07-115559_convert_args_base64_to_args_json/up.sql (100%) rename {migrations => indexer/migrations}/2021-05-10-084700_indexer_on_args_field/down.sql (100%) rename {migrations => indexer/migrations}/2021-05-10-084700_indexer_on_args_field/up.sql (100%) rename {migrations => indexer/migrations}/2021-05-27-154211_account_changes_unique_idx/down.sql (100%) rename {migrations => indexer/migrations}/2021-05-27-154211_account_changes_unique_idx/up.sql (100%) rename {migrations => indexer/migrations}/2021-06-02-173100_add_migration_state_change_reason_kind/down.sql (100%) rename {migrations => indexer/migrations}/2021-06-02-173100_add_migration_state_change_reason_kind/up.sql (100%) rename {migrations => indexer/migrations}/2021-08-02-183200_transactions_sorting_idx/down.sql (100%) rename {migrations => indexer/migrations}/2021-08-02-183200_transactions_sorting_idx/up.sql (100%) rename {migrations => indexer/migrations}/2021-08-04-151515_circulating_supply_table/down.sql (100%) rename {migrations => indexer/migrations}/2021-08-04-151515_circulating_supply_table/up.sql (100%) rename {migrations => indexer/migrations}/2021-08-06-123500_account_changes_ordering_column/down.sql (100%) rename {migrations => indexer/migrations}/2021-08-06-123500_account_changes_ordering_column/up.sql (100%) rename {migrations => indexer/migrations}/2021-08-11-163800_account_changes_ordering_idx/down.sql (100%) rename {migrations => indexer/migrations}/2021-08-11-163800_account_changes_ordering_idx/up.sql (100%) rename {migrations => indexer/migrations}/2021-10-04-100000_assets_nft/down.sql (100%) rename {migrations => indexer/migrations}/2021-10-04-100000_assets_nft/up.sql (100%) rename {migrations => indexer/migrations}/2021-10-14-180948_add_resharding_state_change_reason_kind/down.sql (100%) rename {migrations => indexer/migrations}/2021-10-14-180948_add_resharding_state_change_reason_kind/up.sql (100%) rename {migrations => indexer/migrations}/2022-01-12-100000_assets_ft/down.sql (100%) rename {migrations => indexer/migrations}/2022-01-12-100000_assets_ft/up.sql (100%) rename {migrations => indexer/migrations}/2022-01-26-184200_drop_duplicated_index_execution_outcome/down.sql (100%) rename {migrations => indexer/migrations}/2022-01-26-184200_drop_duplicated_index_execution_outcome/up.sql (100%) rename {migrations => indexer/migrations}/2022-01-26-184201_index_action_receipt_actions/down.sql (100%) rename {migrations => indexer/migrations}/2022-01-26-184201_index_action_receipt_actions/up.sql (100%) rename {src => indexer/src}/configs.rs (100%) rename {src => indexer/src}/db_adapters/access_keys.rs (100%) rename {src => indexer/src}/db_adapters/account_changes.rs (100%) rename {src => indexer/src}/db_adapters/accounts.rs (100%) rename {src => indexer/src}/db_adapters/assets/event_types.rs (100%) rename {src => indexer/src}/db_adapters/assets/events.rs (100%) rename {src => indexer/src}/db_adapters/assets/fungible_token_events.rs (100%) rename {src => indexer/src}/db_adapters/assets/mod.rs (100%) rename {src => indexer/src}/db_adapters/assets/non_fungible_token_events.rs (100%) rename {src => indexer/src}/db_adapters/blocks.rs (100%) rename {src => indexer/src}/db_adapters/chunks.rs (100%) rename {src => indexer/src}/db_adapters/execution_outcomes.rs (100%) rename {src => indexer/src}/db_adapters/mod.rs (100%) rename {src => indexer/src}/db_adapters/receipts.rs (100%) rename {src => indexer/src}/db_adapters/transactions.rs (100%) rename {src => indexer/src}/main.rs (100%) rename {src => indexer/src}/models/access_keys.rs (100%) rename {src => indexer/src}/models/account_changes.rs (100%) rename {src => indexer/src}/models/accounts.rs (100%) rename {src => indexer/src}/models/aggregated/circulating_supply.rs (100%) rename {src => indexer/src}/models/aggregated/mod.rs (100%) rename {src => indexer/src}/models/assets/fungible_token_events.rs (100%) rename {src => indexer/src}/models/assets/mod.rs (100%) rename {src => indexer/src}/models/assets/non_fungible_token_events.rs (100%) rename {src => indexer/src}/models/blocks.rs (100%) rename {src => indexer/src}/models/chunks.rs (100%) rename {src => indexer/src}/models/enums.rs (100%) rename {src => indexer/src}/models/execution_outcomes.rs (100%) rename {src => indexer/src}/models/mod.rs (100%) rename {src => indexer/src}/models/receipts.rs (100%) rename {src => indexer/src}/models/serializers.rs (100%) rename {src => indexer/src}/models/transactions.rs (100%) rename {src => indexer/src}/retriable.rs (100%) rename {src => indexer/src}/schema.patch (100%) rename {src => indexer/src}/schema.rs (100%) diff --git a/Cargo.toml b/Cargo.toml index bfc003d4..16bdcec9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,44 +1,4 @@ -[package] -name = "indexer-explorer-lake" -version = "0.10.1" -authors = ["Near Inc "] -edition = "2021" -rust-version = "1.62.1" - -[dependencies] -actix = "0.13.0" -actix-rt = "2.2.0" -anyhow = "1.0.51" -aws-types = "0.13.0" -aws-sdk-s3 = "0.13.0" -base64 = "0.11" -bigdecimal = "=0.1.0" -borsh = "0.7.1" -cached = "0.23.0" -chrono = "0.4.19" -clap = { version = "3.1.6", features = ["color", "derive", "env"] } -diesel = { version = "1.4.7", features = ["postgres", "numeric", "serde_json"] } -# Using hacky diesel-derive-enum https://github.com/adwhit/diesel-derive-enum/issues/52 -diesel-derive-enum = { git = "https://github.com/khorolets/diesel-derive-enum.git", branch = "lookup-hack", features = ["postgres"] } -dotenv = "0.15.0" -futures = "0.3.5" -hex = "0.4" -itertools = "0.10.3" -# syn version conflict, replace with crates.io version once released -near-sdk = { git = "https://github.com/near/near-sdk-rs", rev="03487c184d37b0382dd9bd41c57466acad58fc1f" } -num-traits = "0.2.11" -openssl-probe = { version = "0.1.2" } -r2d2 = "0.8.8" -serde = { version = "1", features = ["derive"] } -serde_json = "1.0.55" -tokio = { version = "1.1", features = ["sync", "time"] } -tokio-stream = { version = "0.1" } -tracing = "0.1.13" -tracing-subscriber = "0.2.4" -uint = { version = "0.8.3", default-features = false } - -actix-diesel = { git = "https://github.com/frol/actix-diesel", rev = "3a001986c89dfabfc3c448d8bae28525101b4992" } -near-jsonrpc-client = "0.4.0-beta.0" -near-lake-framework = "0.5.2" -near-primitives = "0.14" -near-crypto = "0.14" +[workspace] +members = [ + "indexer" +] diff --git a/CHANGELOG.md b/indexer/CHANGELOG.md similarity index 100% rename from CHANGELOG.md rename to indexer/CHANGELOG.md diff --git a/indexer/Cargo.toml b/indexer/Cargo.toml new file mode 100644 index 00000000..bfc003d4 --- /dev/null +++ b/indexer/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "indexer-explorer-lake" +version = "0.10.1" +authors = ["Near Inc "] +edition = "2021" +rust-version = "1.62.1" + +[dependencies] +actix = "0.13.0" +actix-rt = "2.2.0" +anyhow = "1.0.51" +aws-types = "0.13.0" +aws-sdk-s3 = "0.13.0" +base64 = "0.11" +bigdecimal = "=0.1.0" +borsh = "0.7.1" +cached = "0.23.0" +chrono = "0.4.19" +clap = { version = "3.1.6", features = ["color", "derive", "env"] } +diesel = { version = "1.4.7", features = ["postgres", "numeric", "serde_json"] } +# Using hacky diesel-derive-enum https://github.com/adwhit/diesel-derive-enum/issues/52 +diesel-derive-enum = { git = "https://github.com/khorolets/diesel-derive-enum.git", branch = "lookup-hack", features = ["postgres"] } +dotenv = "0.15.0" +futures = "0.3.5" +hex = "0.4" +itertools = "0.10.3" +# syn version conflict, replace with crates.io version once released +near-sdk = { git = "https://github.com/near/near-sdk-rs", rev="03487c184d37b0382dd9bd41c57466acad58fc1f" } +num-traits = "0.2.11" +openssl-probe = { version = "0.1.2" } +r2d2 = "0.8.8" +serde = { version = "1", features = ["derive"] } +serde_json = "1.0.55" +tokio = { version = "1.1", features = ["sync", "time"] } +tokio-stream = { version = "0.1" } +tracing = "0.1.13" +tracing-subscriber = "0.2.4" +uint = { version = "0.8.3", default-features = false } + +actix-diesel = { git = "https://github.com/frol/actix-diesel", rev = "3a001986c89dfabfc3c448d8bae28525101b4992" } +near-jsonrpc-client = "0.4.0-beta.0" +near-lake-framework = "0.5.2" +near-primitives = "0.14" +near-crypto = "0.14" diff --git a/diesel.toml b/indexer/diesel.toml similarity index 100% rename from diesel.toml rename to indexer/diesel.toml diff --git a/migrations/.gitkeep b/indexer/migrations/.gitkeep similarity index 100% rename from migrations/.gitkeep rename to indexer/migrations/.gitkeep diff --git a/migrations/00000000000000_diesel_initial_setup/down.sql b/indexer/migrations/00000000000000_diesel_initial_setup/down.sql similarity index 100% rename from migrations/00000000000000_diesel_initial_setup/down.sql rename to indexer/migrations/00000000000000_diesel_initial_setup/down.sql diff --git a/migrations/00000000000000_diesel_initial_setup/up.sql b/indexer/migrations/00000000000000_diesel_initial_setup/up.sql similarity index 100% rename from migrations/00000000000000_diesel_initial_setup/up.sql rename to indexer/migrations/00000000000000_diesel_initial_setup/up.sql diff --git a/migrations/2020-12-07-153402_initial_schema/down.sql b/indexer/migrations/2020-12-07-153402_initial_schema/down.sql similarity index 100% rename from migrations/2020-12-07-153402_initial_schema/down.sql rename to indexer/migrations/2020-12-07-153402_initial_schema/down.sql diff --git a/migrations/2020-12-07-153402_initial_schema/up.sql b/indexer/migrations/2020-12-07-153402_initial_schema/up.sql similarity index 100% rename from migrations/2020-12-07-153402_initial_schema/up.sql rename to indexer/migrations/2020-12-07-153402_initial_schema/up.sql diff --git a/migrations/2021-01-14-170424_index-receipt-originated-from-transaction-hash/down.sql b/indexer/migrations/2021-01-14-170424_index-receipt-originated-from-transaction-hash/down.sql similarity index 100% rename from migrations/2021-01-14-170424_index-receipt-originated-from-transaction-hash/down.sql rename to indexer/migrations/2021-01-14-170424_index-receipt-originated-from-transaction-hash/down.sql diff --git a/migrations/2021-01-14-170424_index-receipt-originated-from-transaction-hash/up.sql b/indexer/migrations/2021-01-14-170424_index-receipt-originated-from-transaction-hash/up.sql similarity index 100% rename from migrations/2021-01-14-170424_index-receipt-originated-from-transaction-hash/up.sql rename to indexer/migrations/2021-01-14-170424_index-receipt-originated-from-transaction-hash/up.sql diff --git a/migrations/2021-01-20-152056_index-transactions-receiver-account-id/down.sql b/indexer/migrations/2021-01-20-152056_index-transactions-receiver-account-id/down.sql similarity index 100% rename from migrations/2021-01-20-152056_index-transactions-receiver-account-id/down.sql rename to indexer/migrations/2021-01-20-152056_index-transactions-receiver-account-id/down.sql diff --git a/migrations/2021-01-20-152056_index-transactions-receiver-account-id/up.sql b/indexer/migrations/2021-01-20-152056_index-transactions-receiver-account-id/up.sql similarity index 100% rename from migrations/2021-01-20-152056_index-transactions-receiver-account-id/up.sql rename to indexer/migrations/2021-01-20-152056_index-transactions-receiver-account-id/up.sql diff --git a/migrations/2021-03-11-123839_index-action-kind-and-status/down.sql b/indexer/migrations/2021-03-11-123839_index-action-kind-and-status/down.sql similarity index 100% rename from migrations/2021-03-11-123839_index-action-kind-and-status/down.sql rename to indexer/migrations/2021-03-11-123839_index-action-kind-and-status/down.sql diff --git a/migrations/2021-03-11-123839_index-action-kind-and-status/up.sql b/indexer/migrations/2021-03-11-123839_index-action-kind-and-status/up.sql similarity index 100% rename from migrations/2021-03-11-123839_index-action-kind-and-status/up.sql rename to indexer/migrations/2021-03-11-123839_index-action-kind-and-status/up.sql diff --git a/migrations/2021-04-22-090505_execution_outcomes_replace_chunk_relation_with_shard_id/down.sql b/indexer/migrations/2021-04-22-090505_execution_outcomes_replace_chunk_relation_with_shard_id/down.sql similarity index 100% rename from migrations/2021-04-22-090505_execution_outcomes_replace_chunk_relation_with_shard_id/down.sql rename to indexer/migrations/2021-04-22-090505_execution_outcomes_replace_chunk_relation_with_shard_id/down.sql diff --git a/migrations/2021-04-22-090505_execution_outcomes_replace_chunk_relation_with_shard_id/up.sql b/indexer/migrations/2021-04-22-090505_execution_outcomes_replace_chunk_relation_with_shard_id/up.sql similarity index 100% rename from migrations/2021-04-22-090505_execution_outcomes_replace_chunk_relation_with_shard_id/up.sql rename to indexer/migrations/2021-04-22-090505_execution_outcomes_replace_chunk_relation_with_shard_id/up.sql diff --git a/migrations/2021-04-28-154439_denormalize_action_receipt_actions/down.sql b/indexer/migrations/2021-04-28-154439_denormalize_action_receipt_actions/down.sql similarity index 100% rename from migrations/2021-04-28-154439_denormalize_action_receipt_actions/down.sql rename to indexer/migrations/2021-04-28-154439_denormalize_action_receipt_actions/down.sql diff --git a/migrations/2021-04-28-154439_denormalize_action_receipt_actions/up.sql b/indexer/migrations/2021-04-28-154439_denormalize_action_receipt_actions/up.sql similarity index 100% rename from migrations/2021-04-28-154439_denormalize_action_receipt_actions/up.sql rename to indexer/migrations/2021-04-28-154439_denormalize_action_receipt_actions/up.sql diff --git a/migrations/2021-05-06-093622_additional_indexes_for_action_receipt_actions/down.sql b/indexer/migrations/2021-05-06-093622_additional_indexes_for_action_receipt_actions/down.sql similarity index 100% rename from migrations/2021-05-06-093622_additional_indexes_for_action_receipt_actions/down.sql rename to indexer/migrations/2021-05-06-093622_additional_indexes_for_action_receipt_actions/down.sql diff --git a/migrations/2021-05-06-093622_additional_indexes_for_action_receipt_actions/up.sql b/indexer/migrations/2021-05-06-093622_additional_indexes_for_action_receipt_actions/up.sql similarity index 100% rename from migrations/2021-05-06-093622_additional_indexes_for_action_receipt_actions/up.sql rename to indexer/migrations/2021-05-06-093622_additional_indexes_for_action_receipt_actions/up.sql diff --git a/migrations/2021-05-07-115559_convert_args_base64_to_args_json/down.sql b/indexer/migrations/2021-05-07-115559_convert_args_base64_to_args_json/down.sql similarity index 100% rename from migrations/2021-05-07-115559_convert_args_base64_to_args_json/down.sql rename to indexer/migrations/2021-05-07-115559_convert_args_base64_to_args_json/down.sql diff --git a/migrations/2021-05-07-115559_convert_args_base64_to_args_json/up.sql b/indexer/migrations/2021-05-07-115559_convert_args_base64_to_args_json/up.sql similarity index 100% rename from migrations/2021-05-07-115559_convert_args_base64_to_args_json/up.sql rename to indexer/migrations/2021-05-07-115559_convert_args_base64_to_args_json/up.sql diff --git a/migrations/2021-05-10-084700_indexer_on_args_field/down.sql b/indexer/migrations/2021-05-10-084700_indexer_on_args_field/down.sql similarity index 100% rename from migrations/2021-05-10-084700_indexer_on_args_field/down.sql rename to indexer/migrations/2021-05-10-084700_indexer_on_args_field/down.sql diff --git a/migrations/2021-05-10-084700_indexer_on_args_field/up.sql b/indexer/migrations/2021-05-10-084700_indexer_on_args_field/up.sql similarity index 100% rename from migrations/2021-05-10-084700_indexer_on_args_field/up.sql rename to indexer/migrations/2021-05-10-084700_indexer_on_args_field/up.sql diff --git a/migrations/2021-05-27-154211_account_changes_unique_idx/down.sql b/indexer/migrations/2021-05-27-154211_account_changes_unique_idx/down.sql similarity index 100% rename from migrations/2021-05-27-154211_account_changes_unique_idx/down.sql rename to indexer/migrations/2021-05-27-154211_account_changes_unique_idx/down.sql diff --git a/migrations/2021-05-27-154211_account_changes_unique_idx/up.sql b/indexer/migrations/2021-05-27-154211_account_changes_unique_idx/up.sql similarity index 100% rename from migrations/2021-05-27-154211_account_changes_unique_idx/up.sql rename to indexer/migrations/2021-05-27-154211_account_changes_unique_idx/up.sql diff --git a/migrations/2021-06-02-173100_add_migration_state_change_reason_kind/down.sql b/indexer/migrations/2021-06-02-173100_add_migration_state_change_reason_kind/down.sql similarity index 100% rename from migrations/2021-06-02-173100_add_migration_state_change_reason_kind/down.sql rename to indexer/migrations/2021-06-02-173100_add_migration_state_change_reason_kind/down.sql diff --git a/migrations/2021-06-02-173100_add_migration_state_change_reason_kind/up.sql b/indexer/migrations/2021-06-02-173100_add_migration_state_change_reason_kind/up.sql similarity index 100% rename from migrations/2021-06-02-173100_add_migration_state_change_reason_kind/up.sql rename to indexer/migrations/2021-06-02-173100_add_migration_state_change_reason_kind/up.sql diff --git a/migrations/2021-08-02-183200_transactions_sorting_idx/down.sql b/indexer/migrations/2021-08-02-183200_transactions_sorting_idx/down.sql similarity index 100% rename from migrations/2021-08-02-183200_transactions_sorting_idx/down.sql rename to indexer/migrations/2021-08-02-183200_transactions_sorting_idx/down.sql diff --git a/migrations/2021-08-02-183200_transactions_sorting_idx/up.sql b/indexer/migrations/2021-08-02-183200_transactions_sorting_idx/up.sql similarity index 100% rename from migrations/2021-08-02-183200_transactions_sorting_idx/up.sql rename to indexer/migrations/2021-08-02-183200_transactions_sorting_idx/up.sql diff --git a/migrations/2021-08-04-151515_circulating_supply_table/down.sql b/indexer/migrations/2021-08-04-151515_circulating_supply_table/down.sql similarity index 100% rename from migrations/2021-08-04-151515_circulating_supply_table/down.sql rename to indexer/migrations/2021-08-04-151515_circulating_supply_table/down.sql diff --git a/migrations/2021-08-04-151515_circulating_supply_table/up.sql b/indexer/migrations/2021-08-04-151515_circulating_supply_table/up.sql similarity index 100% rename from migrations/2021-08-04-151515_circulating_supply_table/up.sql rename to indexer/migrations/2021-08-04-151515_circulating_supply_table/up.sql diff --git a/migrations/2021-08-06-123500_account_changes_ordering_column/down.sql b/indexer/migrations/2021-08-06-123500_account_changes_ordering_column/down.sql similarity index 100% rename from migrations/2021-08-06-123500_account_changes_ordering_column/down.sql rename to indexer/migrations/2021-08-06-123500_account_changes_ordering_column/down.sql diff --git a/migrations/2021-08-06-123500_account_changes_ordering_column/up.sql b/indexer/migrations/2021-08-06-123500_account_changes_ordering_column/up.sql similarity index 100% rename from migrations/2021-08-06-123500_account_changes_ordering_column/up.sql rename to indexer/migrations/2021-08-06-123500_account_changes_ordering_column/up.sql diff --git a/migrations/2021-08-11-163800_account_changes_ordering_idx/down.sql b/indexer/migrations/2021-08-11-163800_account_changes_ordering_idx/down.sql similarity index 100% rename from migrations/2021-08-11-163800_account_changes_ordering_idx/down.sql rename to indexer/migrations/2021-08-11-163800_account_changes_ordering_idx/down.sql diff --git a/migrations/2021-08-11-163800_account_changes_ordering_idx/up.sql b/indexer/migrations/2021-08-11-163800_account_changes_ordering_idx/up.sql similarity index 100% rename from migrations/2021-08-11-163800_account_changes_ordering_idx/up.sql rename to indexer/migrations/2021-08-11-163800_account_changes_ordering_idx/up.sql diff --git a/migrations/2021-10-04-100000_assets_nft/down.sql b/indexer/migrations/2021-10-04-100000_assets_nft/down.sql similarity index 100% rename from migrations/2021-10-04-100000_assets_nft/down.sql rename to indexer/migrations/2021-10-04-100000_assets_nft/down.sql diff --git a/migrations/2021-10-04-100000_assets_nft/up.sql b/indexer/migrations/2021-10-04-100000_assets_nft/up.sql similarity index 100% rename from migrations/2021-10-04-100000_assets_nft/up.sql rename to indexer/migrations/2021-10-04-100000_assets_nft/up.sql diff --git a/migrations/2021-10-14-180948_add_resharding_state_change_reason_kind/down.sql b/indexer/migrations/2021-10-14-180948_add_resharding_state_change_reason_kind/down.sql similarity index 100% rename from migrations/2021-10-14-180948_add_resharding_state_change_reason_kind/down.sql rename to indexer/migrations/2021-10-14-180948_add_resharding_state_change_reason_kind/down.sql diff --git a/migrations/2021-10-14-180948_add_resharding_state_change_reason_kind/up.sql b/indexer/migrations/2021-10-14-180948_add_resharding_state_change_reason_kind/up.sql similarity index 100% rename from migrations/2021-10-14-180948_add_resharding_state_change_reason_kind/up.sql rename to indexer/migrations/2021-10-14-180948_add_resharding_state_change_reason_kind/up.sql diff --git a/migrations/2022-01-12-100000_assets_ft/down.sql b/indexer/migrations/2022-01-12-100000_assets_ft/down.sql similarity index 100% rename from migrations/2022-01-12-100000_assets_ft/down.sql rename to indexer/migrations/2022-01-12-100000_assets_ft/down.sql diff --git a/migrations/2022-01-12-100000_assets_ft/up.sql b/indexer/migrations/2022-01-12-100000_assets_ft/up.sql similarity index 100% rename from migrations/2022-01-12-100000_assets_ft/up.sql rename to indexer/migrations/2022-01-12-100000_assets_ft/up.sql diff --git a/migrations/2022-01-26-184200_drop_duplicated_index_execution_outcome/down.sql b/indexer/migrations/2022-01-26-184200_drop_duplicated_index_execution_outcome/down.sql similarity index 100% rename from migrations/2022-01-26-184200_drop_duplicated_index_execution_outcome/down.sql rename to indexer/migrations/2022-01-26-184200_drop_duplicated_index_execution_outcome/down.sql diff --git a/migrations/2022-01-26-184200_drop_duplicated_index_execution_outcome/up.sql b/indexer/migrations/2022-01-26-184200_drop_duplicated_index_execution_outcome/up.sql similarity index 100% rename from migrations/2022-01-26-184200_drop_duplicated_index_execution_outcome/up.sql rename to indexer/migrations/2022-01-26-184200_drop_duplicated_index_execution_outcome/up.sql diff --git a/migrations/2022-01-26-184201_index_action_receipt_actions/down.sql b/indexer/migrations/2022-01-26-184201_index_action_receipt_actions/down.sql similarity index 100% rename from migrations/2022-01-26-184201_index_action_receipt_actions/down.sql rename to indexer/migrations/2022-01-26-184201_index_action_receipt_actions/down.sql diff --git a/migrations/2022-01-26-184201_index_action_receipt_actions/up.sql b/indexer/migrations/2022-01-26-184201_index_action_receipt_actions/up.sql similarity index 100% rename from migrations/2022-01-26-184201_index_action_receipt_actions/up.sql rename to indexer/migrations/2022-01-26-184201_index_action_receipt_actions/up.sql diff --git a/src/configs.rs b/indexer/src/configs.rs similarity index 100% rename from src/configs.rs rename to indexer/src/configs.rs diff --git a/src/db_adapters/access_keys.rs b/indexer/src/db_adapters/access_keys.rs similarity index 100% rename from src/db_adapters/access_keys.rs rename to indexer/src/db_adapters/access_keys.rs diff --git a/src/db_adapters/account_changes.rs b/indexer/src/db_adapters/account_changes.rs similarity index 100% rename from src/db_adapters/account_changes.rs rename to indexer/src/db_adapters/account_changes.rs diff --git a/src/db_adapters/accounts.rs b/indexer/src/db_adapters/accounts.rs similarity index 100% rename from src/db_adapters/accounts.rs rename to indexer/src/db_adapters/accounts.rs diff --git a/src/db_adapters/assets/event_types.rs b/indexer/src/db_adapters/assets/event_types.rs similarity index 100% rename from src/db_adapters/assets/event_types.rs rename to indexer/src/db_adapters/assets/event_types.rs diff --git a/src/db_adapters/assets/events.rs b/indexer/src/db_adapters/assets/events.rs similarity index 100% rename from src/db_adapters/assets/events.rs rename to indexer/src/db_adapters/assets/events.rs diff --git a/src/db_adapters/assets/fungible_token_events.rs b/indexer/src/db_adapters/assets/fungible_token_events.rs similarity index 100% rename from src/db_adapters/assets/fungible_token_events.rs rename to indexer/src/db_adapters/assets/fungible_token_events.rs diff --git a/src/db_adapters/assets/mod.rs b/indexer/src/db_adapters/assets/mod.rs similarity index 100% rename from src/db_adapters/assets/mod.rs rename to indexer/src/db_adapters/assets/mod.rs diff --git a/src/db_adapters/assets/non_fungible_token_events.rs b/indexer/src/db_adapters/assets/non_fungible_token_events.rs similarity index 100% rename from src/db_adapters/assets/non_fungible_token_events.rs rename to indexer/src/db_adapters/assets/non_fungible_token_events.rs diff --git a/src/db_adapters/blocks.rs b/indexer/src/db_adapters/blocks.rs similarity index 100% rename from src/db_adapters/blocks.rs rename to indexer/src/db_adapters/blocks.rs diff --git a/src/db_adapters/chunks.rs b/indexer/src/db_adapters/chunks.rs similarity index 100% rename from src/db_adapters/chunks.rs rename to indexer/src/db_adapters/chunks.rs diff --git a/src/db_adapters/execution_outcomes.rs b/indexer/src/db_adapters/execution_outcomes.rs similarity index 100% rename from src/db_adapters/execution_outcomes.rs rename to indexer/src/db_adapters/execution_outcomes.rs diff --git a/src/db_adapters/mod.rs b/indexer/src/db_adapters/mod.rs similarity index 100% rename from src/db_adapters/mod.rs rename to indexer/src/db_adapters/mod.rs diff --git a/src/db_adapters/receipts.rs b/indexer/src/db_adapters/receipts.rs similarity index 100% rename from src/db_adapters/receipts.rs rename to indexer/src/db_adapters/receipts.rs diff --git a/src/db_adapters/transactions.rs b/indexer/src/db_adapters/transactions.rs similarity index 100% rename from src/db_adapters/transactions.rs rename to indexer/src/db_adapters/transactions.rs diff --git a/src/main.rs b/indexer/src/main.rs similarity index 100% rename from src/main.rs rename to indexer/src/main.rs diff --git a/src/models/access_keys.rs b/indexer/src/models/access_keys.rs similarity index 100% rename from src/models/access_keys.rs rename to indexer/src/models/access_keys.rs diff --git a/src/models/account_changes.rs b/indexer/src/models/account_changes.rs similarity index 100% rename from src/models/account_changes.rs rename to indexer/src/models/account_changes.rs diff --git a/src/models/accounts.rs b/indexer/src/models/accounts.rs similarity index 100% rename from src/models/accounts.rs rename to indexer/src/models/accounts.rs diff --git a/src/models/aggregated/circulating_supply.rs b/indexer/src/models/aggregated/circulating_supply.rs similarity index 100% rename from src/models/aggregated/circulating_supply.rs rename to indexer/src/models/aggregated/circulating_supply.rs diff --git a/src/models/aggregated/mod.rs b/indexer/src/models/aggregated/mod.rs similarity index 100% rename from src/models/aggregated/mod.rs rename to indexer/src/models/aggregated/mod.rs diff --git a/src/models/assets/fungible_token_events.rs b/indexer/src/models/assets/fungible_token_events.rs similarity index 100% rename from src/models/assets/fungible_token_events.rs rename to indexer/src/models/assets/fungible_token_events.rs diff --git a/src/models/assets/mod.rs b/indexer/src/models/assets/mod.rs similarity index 100% rename from src/models/assets/mod.rs rename to indexer/src/models/assets/mod.rs diff --git a/src/models/assets/non_fungible_token_events.rs b/indexer/src/models/assets/non_fungible_token_events.rs similarity index 100% rename from src/models/assets/non_fungible_token_events.rs rename to indexer/src/models/assets/non_fungible_token_events.rs diff --git a/src/models/blocks.rs b/indexer/src/models/blocks.rs similarity index 100% rename from src/models/blocks.rs rename to indexer/src/models/blocks.rs diff --git a/src/models/chunks.rs b/indexer/src/models/chunks.rs similarity index 100% rename from src/models/chunks.rs rename to indexer/src/models/chunks.rs diff --git a/src/models/enums.rs b/indexer/src/models/enums.rs similarity index 100% rename from src/models/enums.rs rename to indexer/src/models/enums.rs diff --git a/src/models/execution_outcomes.rs b/indexer/src/models/execution_outcomes.rs similarity index 100% rename from src/models/execution_outcomes.rs rename to indexer/src/models/execution_outcomes.rs diff --git a/src/models/mod.rs b/indexer/src/models/mod.rs similarity index 100% rename from src/models/mod.rs rename to indexer/src/models/mod.rs diff --git a/src/models/receipts.rs b/indexer/src/models/receipts.rs similarity index 100% rename from src/models/receipts.rs rename to indexer/src/models/receipts.rs diff --git a/src/models/serializers.rs b/indexer/src/models/serializers.rs similarity index 100% rename from src/models/serializers.rs rename to indexer/src/models/serializers.rs diff --git a/src/models/transactions.rs b/indexer/src/models/transactions.rs similarity index 100% rename from src/models/transactions.rs rename to indexer/src/models/transactions.rs diff --git a/src/retriable.rs b/indexer/src/retriable.rs similarity index 100% rename from src/retriable.rs rename to indexer/src/retriable.rs diff --git a/src/schema.patch b/indexer/src/schema.patch similarity index 100% rename from src/schema.patch rename to indexer/src/schema.patch diff --git a/src/schema.rs b/indexer/src/schema.rs similarity index 100% rename from src/schema.rs rename to indexer/src/schema.rs From a2a1b0c26ea1038c336e6ed6cd7fb34dd6bc5aa8 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Mon, 31 Oct 2022 15:10:56 +1300 Subject: [PATCH 10/42] feat: Create empty `database` library crate --- Cargo.toml | 3 ++- database/Cargo.toml | 8 ++++++++ database/src/lib.rs | 14 ++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 database/Cargo.toml create mode 100644 database/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 16bdcec9..664b7c9a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,4 +1,5 @@ [workspace] members = [ - "indexer" + "indexer", + "database" ] diff --git a/database/Cargo.toml b/database/Cargo.toml new file mode 100644 index 00000000..f4726d77 --- /dev/null +++ b/database/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "database" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/database/src/lib.rs b/database/src/lib.rs new file mode 100644 index 00000000..7d12d9af --- /dev/null +++ b/database/src/lib.rs @@ -0,0 +1,14 @@ +pub fn add(left: usize, right: usize) -> usize { + left + right +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn it_works() { + let result = add(2, 2); + assert_eq!(result, 4); + } +} From 0d97e3b79424c45e67a58b92134b8eb0089047f5 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Mon, 31 Oct 2022 16:04:29 +1300 Subject: [PATCH 11/42] refactor: Extract db logic to `database` lib --- Cargo.lock | 35 +++++++++++++------ database/Cargo.toml | 21 ++++++++++- {indexer => database}/diesel.toml | 0 .../src/db_adapters/access_keys.rs | 0 .../src/db_adapters/account_changes.rs | 0 .../src/db_adapters/accounts.rs | 0 .../src/db_adapters/assets/event_types.rs | 0 .../src/db_adapters/assets/events.rs | 0 .../assets/fungible_token_events.rs | 0 .../src/db_adapters/assets/mod.rs | 0 .../assets/non_fungible_token_events.rs | 0 .../src/db_adapters/blocks.rs | 0 .../src/db_adapters/chunks.rs | 0 .../src/db_adapters/execution_outcomes.rs | 0 {indexer => database}/src/db_adapters/mod.rs | 0 .../src/db_adapters/receipts.rs | 0 .../src/db_adapters/transactions.rs | 0 database/src/lib.rs | 30 ++++++++-------- {indexer => database/src}/migrations/.gitkeep | 0 .../down.sql | 0 .../up.sql | 0 .../2020-12-07-153402_initial_schema/down.sql | 0 .../2020-12-07-153402_initial_schema/up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../2021-10-04-100000_assets_nft/down.sql | 0 .../2021-10-04-100000_assets_nft/up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../2022-01-12-100000_assets_ft/down.sql | 0 .../2022-01-12-100000_assets_ft/up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../src/models/access_keys.rs | 0 .../src/models/account_changes.rs | 0 {indexer => database}/src/models/accounts.rs | 0 .../models/aggregated/circulating_supply.rs | 0 .../src/models/aggregated/mod.rs | 0 .../models/assets/fungible_token_events.rs | 0 .../src/models/assets/mod.rs | 0 .../assets/non_fungible_token_events.rs | 0 {indexer => database}/src/models/blocks.rs | 0 {indexer => database}/src/models/chunks.rs | 0 {indexer => database}/src/models/enums.rs | 0 .../src/models/execution_outcomes.rs | 0 {indexer => database}/src/models/mod.rs | 0 {indexer => database}/src/models/receipts.rs | 0 .../src/models/serializers.rs | 0 .../src/models/transactions.rs | 0 {indexer => database}/src/retriable.rs | 0 {indexer => database}/src/schema.patch | 0 {indexer => database}/src/schema.rs | 0 indexer/Cargo.toml | 14 ++------ indexer/src/configs.rs | 10 +++--- indexer/src/main.rs | 15 ++------ 83 files changed, 70 insertions(+), 55 deletions(-) rename {indexer => database}/diesel.toml (100%) rename {indexer => database}/src/db_adapters/access_keys.rs (100%) rename {indexer => database}/src/db_adapters/account_changes.rs (100%) rename {indexer => database}/src/db_adapters/accounts.rs (100%) rename {indexer => database}/src/db_adapters/assets/event_types.rs (100%) rename {indexer => database}/src/db_adapters/assets/events.rs (100%) rename {indexer => database}/src/db_adapters/assets/fungible_token_events.rs (100%) rename {indexer => database}/src/db_adapters/assets/mod.rs (100%) rename {indexer => database}/src/db_adapters/assets/non_fungible_token_events.rs (100%) rename {indexer => database}/src/db_adapters/blocks.rs (100%) rename {indexer => database}/src/db_adapters/chunks.rs (100%) rename {indexer => database}/src/db_adapters/execution_outcomes.rs (100%) rename {indexer => database}/src/db_adapters/mod.rs (100%) rename {indexer => database}/src/db_adapters/receipts.rs (100%) rename {indexer => database}/src/db_adapters/transactions.rs (100%) rename {indexer => database/src}/migrations/.gitkeep (100%) rename {indexer => database/src}/migrations/00000000000000_diesel_initial_setup/down.sql (100%) rename {indexer => database/src}/migrations/00000000000000_diesel_initial_setup/up.sql (100%) rename {indexer => database/src}/migrations/2020-12-07-153402_initial_schema/down.sql (100%) rename {indexer => database/src}/migrations/2020-12-07-153402_initial_schema/up.sql (100%) rename {indexer => database/src}/migrations/2021-01-14-170424_index-receipt-originated-from-transaction-hash/down.sql (100%) rename {indexer => database/src}/migrations/2021-01-14-170424_index-receipt-originated-from-transaction-hash/up.sql (100%) rename {indexer => database/src}/migrations/2021-01-20-152056_index-transactions-receiver-account-id/down.sql (100%) rename {indexer => database/src}/migrations/2021-01-20-152056_index-transactions-receiver-account-id/up.sql (100%) rename {indexer => database/src}/migrations/2021-03-11-123839_index-action-kind-and-status/down.sql (100%) rename {indexer => database/src}/migrations/2021-03-11-123839_index-action-kind-and-status/up.sql (100%) rename {indexer => database/src}/migrations/2021-04-22-090505_execution_outcomes_replace_chunk_relation_with_shard_id/down.sql (100%) rename {indexer => database/src}/migrations/2021-04-22-090505_execution_outcomes_replace_chunk_relation_with_shard_id/up.sql (100%) rename {indexer => database/src}/migrations/2021-04-28-154439_denormalize_action_receipt_actions/down.sql (100%) rename {indexer => database/src}/migrations/2021-04-28-154439_denormalize_action_receipt_actions/up.sql (100%) rename {indexer => database/src}/migrations/2021-05-06-093622_additional_indexes_for_action_receipt_actions/down.sql (100%) rename {indexer => database/src}/migrations/2021-05-06-093622_additional_indexes_for_action_receipt_actions/up.sql (100%) rename {indexer => database/src}/migrations/2021-05-07-115559_convert_args_base64_to_args_json/down.sql (100%) rename {indexer => database/src}/migrations/2021-05-07-115559_convert_args_base64_to_args_json/up.sql (100%) rename {indexer => database/src}/migrations/2021-05-10-084700_indexer_on_args_field/down.sql (100%) rename {indexer => database/src}/migrations/2021-05-10-084700_indexer_on_args_field/up.sql (100%) rename {indexer => database/src}/migrations/2021-05-27-154211_account_changes_unique_idx/down.sql (100%) rename {indexer => database/src}/migrations/2021-05-27-154211_account_changes_unique_idx/up.sql (100%) rename {indexer => database/src}/migrations/2021-06-02-173100_add_migration_state_change_reason_kind/down.sql (100%) rename {indexer => database/src}/migrations/2021-06-02-173100_add_migration_state_change_reason_kind/up.sql (100%) rename {indexer => database/src}/migrations/2021-08-02-183200_transactions_sorting_idx/down.sql (100%) rename {indexer => database/src}/migrations/2021-08-02-183200_transactions_sorting_idx/up.sql (100%) rename {indexer => database/src}/migrations/2021-08-04-151515_circulating_supply_table/down.sql (100%) rename {indexer => database/src}/migrations/2021-08-04-151515_circulating_supply_table/up.sql (100%) rename {indexer => database/src}/migrations/2021-08-06-123500_account_changes_ordering_column/down.sql (100%) rename {indexer => database/src}/migrations/2021-08-06-123500_account_changes_ordering_column/up.sql (100%) rename {indexer => database/src}/migrations/2021-08-11-163800_account_changes_ordering_idx/down.sql (100%) rename {indexer => database/src}/migrations/2021-08-11-163800_account_changes_ordering_idx/up.sql (100%) rename {indexer => database/src}/migrations/2021-10-04-100000_assets_nft/down.sql (100%) rename {indexer => database/src}/migrations/2021-10-04-100000_assets_nft/up.sql (100%) rename {indexer => database/src}/migrations/2021-10-14-180948_add_resharding_state_change_reason_kind/down.sql (100%) rename {indexer => database/src}/migrations/2021-10-14-180948_add_resharding_state_change_reason_kind/up.sql (100%) rename {indexer => database/src}/migrations/2022-01-12-100000_assets_ft/down.sql (100%) rename {indexer => database/src}/migrations/2022-01-12-100000_assets_ft/up.sql (100%) rename {indexer => database/src}/migrations/2022-01-26-184200_drop_duplicated_index_execution_outcome/down.sql (100%) rename {indexer => database/src}/migrations/2022-01-26-184200_drop_duplicated_index_execution_outcome/up.sql (100%) rename {indexer => database/src}/migrations/2022-01-26-184201_index_action_receipt_actions/down.sql (100%) rename {indexer => database/src}/migrations/2022-01-26-184201_index_action_receipt_actions/up.sql (100%) rename {indexer => database}/src/models/access_keys.rs (100%) rename {indexer => database}/src/models/account_changes.rs (100%) rename {indexer => database}/src/models/accounts.rs (100%) rename {indexer => database}/src/models/aggregated/circulating_supply.rs (100%) rename {indexer => database}/src/models/aggregated/mod.rs (100%) rename {indexer => database}/src/models/assets/fungible_token_events.rs (100%) rename {indexer => database}/src/models/assets/mod.rs (100%) rename {indexer => database}/src/models/assets/non_fungible_token_events.rs (100%) rename {indexer => database}/src/models/blocks.rs (100%) rename {indexer => database}/src/models/chunks.rs (100%) rename {indexer => database}/src/models/enums.rs (100%) rename {indexer => database}/src/models/execution_outcomes.rs (100%) rename {indexer => database}/src/models/mod.rs (100%) rename {indexer => database}/src/models/receipts.rs (100%) rename {indexer => database}/src/models/serializers.rs (100%) rename {indexer => database}/src/models/transactions.rs (100%) rename {indexer => database}/src/retriable.rs (100%) rename {indexer => database}/src/schema.patch (100%) rename {indexer => database}/src/schema.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 689de8fe..a8a96ea4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1209,6 +1209,29 @@ version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77f3309417938f28bf8228fcff79a4a37103981e3e186d2ccd19c74b38f4eb71" +[[package]] +name = "explorer-database" +version = "0.1.0" +dependencies = [ + "actix-diesel", + "anyhow", + "base64 0.11.0", + "bigdecimal", + "cached", + "diesel", + "diesel-derive-enum", + "futures", + "hex", + "near-crypto", + "near-lake-framework", + "near-primitives", + "num-traits", + "serde", + "serde_json", + "tokio", + "tracing", +] + [[package]] name = "fastrand" version = "1.7.0" @@ -1585,33 +1608,23 @@ name = "indexer-explorer-lake" version = "0.10.1" dependencies = [ "actix", - "actix-diesel", "actix-rt", "anyhow", "aws-sdk-s3", "aws-types", - "base64 0.11.0", - "bigdecimal", "borsh 0.7.2", "cached", "chrono", "clap", - "diesel", - "diesel-derive-enum", "dotenv", + "explorer-database", "futures", - "hex", "itertools", - "near-crypto", "near-jsonrpc-client", "near-lake-framework", - "near-primitives", "near-sdk", - "num-traits", "openssl-probe", "r2d2", - "serde", - "serde_json", "tokio", "tokio-stream", "tracing", diff --git a/database/Cargo.toml b/database/Cargo.toml index f4726d77..b5ebf16a 100644 --- a/database/Cargo.toml +++ b/database/Cargo.toml @@ -1,8 +1,27 @@ [package] -name = "database" +name = "explorer-database" version = "0.1.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +anyhow = "1.0.51" +base64 = "0.11" +bigdecimal = "=0.1.0" +diesel = { version = "1.4.7", features = ["postgres", "numeric", "serde_json"] } +# Using hacky diesel-derive-enum https://github.com/adwhit/diesel-derive-enum/issues/52 +diesel-derive-enum = { git = "https://github.com/khorolets/diesel-derive-enum.git", branch = "lookup-hack", features = ["postgres"] } +cached = "0.23.0" +futures = "0.3.5" +hex = "0.4" +num-traits = "0.2.11" +serde = { version = "1", features = ["derive"] } +serde_json = "1.0.55" +tracing = "0.1.13" +tokio = { version = "1.1", features = ["sync", "time"] } + +actix-diesel = { git = "https://github.com/frol/actix-diesel", rev = "3a001986c89dfabfc3c448d8bae28525101b4992" } +near-lake-framework = "0.5.2" +near-primitives = "0.14" +near-crypto = "0.14" diff --git a/indexer/diesel.toml b/database/diesel.toml similarity index 100% rename from indexer/diesel.toml rename to database/diesel.toml diff --git a/indexer/src/db_adapters/access_keys.rs b/database/src/db_adapters/access_keys.rs similarity index 100% rename from indexer/src/db_adapters/access_keys.rs rename to database/src/db_adapters/access_keys.rs diff --git a/indexer/src/db_adapters/account_changes.rs b/database/src/db_adapters/account_changes.rs similarity index 100% rename from indexer/src/db_adapters/account_changes.rs rename to database/src/db_adapters/account_changes.rs diff --git a/indexer/src/db_adapters/accounts.rs b/database/src/db_adapters/accounts.rs similarity index 100% rename from indexer/src/db_adapters/accounts.rs rename to database/src/db_adapters/accounts.rs diff --git a/indexer/src/db_adapters/assets/event_types.rs b/database/src/db_adapters/assets/event_types.rs similarity index 100% rename from indexer/src/db_adapters/assets/event_types.rs rename to database/src/db_adapters/assets/event_types.rs diff --git a/indexer/src/db_adapters/assets/events.rs b/database/src/db_adapters/assets/events.rs similarity index 100% rename from indexer/src/db_adapters/assets/events.rs rename to database/src/db_adapters/assets/events.rs diff --git a/indexer/src/db_adapters/assets/fungible_token_events.rs b/database/src/db_adapters/assets/fungible_token_events.rs similarity index 100% rename from indexer/src/db_adapters/assets/fungible_token_events.rs rename to database/src/db_adapters/assets/fungible_token_events.rs diff --git a/indexer/src/db_adapters/assets/mod.rs b/database/src/db_adapters/assets/mod.rs similarity index 100% rename from indexer/src/db_adapters/assets/mod.rs rename to database/src/db_adapters/assets/mod.rs diff --git a/indexer/src/db_adapters/assets/non_fungible_token_events.rs b/database/src/db_adapters/assets/non_fungible_token_events.rs similarity index 100% rename from indexer/src/db_adapters/assets/non_fungible_token_events.rs rename to database/src/db_adapters/assets/non_fungible_token_events.rs diff --git a/indexer/src/db_adapters/blocks.rs b/database/src/db_adapters/blocks.rs similarity index 100% rename from indexer/src/db_adapters/blocks.rs rename to database/src/db_adapters/blocks.rs diff --git a/indexer/src/db_adapters/chunks.rs b/database/src/db_adapters/chunks.rs similarity index 100% rename from indexer/src/db_adapters/chunks.rs rename to database/src/db_adapters/chunks.rs diff --git a/indexer/src/db_adapters/execution_outcomes.rs b/database/src/db_adapters/execution_outcomes.rs similarity index 100% rename from indexer/src/db_adapters/execution_outcomes.rs rename to database/src/db_adapters/execution_outcomes.rs diff --git a/indexer/src/db_adapters/mod.rs b/database/src/db_adapters/mod.rs similarity index 100% rename from indexer/src/db_adapters/mod.rs rename to database/src/db_adapters/mod.rs diff --git a/indexer/src/db_adapters/receipts.rs b/database/src/db_adapters/receipts.rs similarity index 100% rename from indexer/src/db_adapters/receipts.rs rename to database/src/db_adapters/receipts.rs diff --git a/indexer/src/db_adapters/transactions.rs b/database/src/db_adapters/transactions.rs similarity index 100% rename from indexer/src/db_adapters/transactions.rs rename to database/src/db_adapters/transactions.rs diff --git a/database/src/lib.rs b/database/src/lib.rs index 7d12d9af..fc79eb80 100644 --- a/database/src/lib.rs +++ b/database/src/lib.rs @@ -1,14 +1,16 @@ -pub fn add(left: usize, right: usize) -> usize { - left + right -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn it_works() { - let result = add(2, 2); - assert_eq!(result, 4); - } -} +#[macro_use] +pub extern crate diesel; + +pub use actix_diesel; + +pub mod db_adapters; +pub mod models; + +mod schema; +#[macro_use] +mod retriable; + +const INTERVAL: std::time::Duration = std::time::Duration::from_millis(100); +const INDEXER_FOR_EXPLORER: &str = "indexer_for_explorer"; + +const MAX_DELAY_TIME: std::time::Duration = std::time::Duration::from_secs(120); diff --git a/indexer/migrations/.gitkeep b/database/src/migrations/.gitkeep similarity index 100% rename from indexer/migrations/.gitkeep rename to database/src/migrations/.gitkeep diff --git a/indexer/migrations/00000000000000_diesel_initial_setup/down.sql b/database/src/migrations/00000000000000_diesel_initial_setup/down.sql similarity index 100% rename from indexer/migrations/00000000000000_diesel_initial_setup/down.sql rename to database/src/migrations/00000000000000_diesel_initial_setup/down.sql diff --git a/indexer/migrations/00000000000000_diesel_initial_setup/up.sql b/database/src/migrations/00000000000000_diesel_initial_setup/up.sql similarity index 100% rename from indexer/migrations/00000000000000_diesel_initial_setup/up.sql rename to database/src/migrations/00000000000000_diesel_initial_setup/up.sql diff --git a/indexer/migrations/2020-12-07-153402_initial_schema/down.sql b/database/src/migrations/2020-12-07-153402_initial_schema/down.sql similarity index 100% rename from indexer/migrations/2020-12-07-153402_initial_schema/down.sql rename to database/src/migrations/2020-12-07-153402_initial_schema/down.sql diff --git a/indexer/migrations/2020-12-07-153402_initial_schema/up.sql b/database/src/migrations/2020-12-07-153402_initial_schema/up.sql similarity index 100% rename from indexer/migrations/2020-12-07-153402_initial_schema/up.sql rename to database/src/migrations/2020-12-07-153402_initial_schema/up.sql diff --git a/indexer/migrations/2021-01-14-170424_index-receipt-originated-from-transaction-hash/down.sql b/database/src/migrations/2021-01-14-170424_index-receipt-originated-from-transaction-hash/down.sql similarity index 100% rename from indexer/migrations/2021-01-14-170424_index-receipt-originated-from-transaction-hash/down.sql rename to database/src/migrations/2021-01-14-170424_index-receipt-originated-from-transaction-hash/down.sql diff --git a/indexer/migrations/2021-01-14-170424_index-receipt-originated-from-transaction-hash/up.sql b/database/src/migrations/2021-01-14-170424_index-receipt-originated-from-transaction-hash/up.sql similarity index 100% rename from indexer/migrations/2021-01-14-170424_index-receipt-originated-from-transaction-hash/up.sql rename to database/src/migrations/2021-01-14-170424_index-receipt-originated-from-transaction-hash/up.sql diff --git a/indexer/migrations/2021-01-20-152056_index-transactions-receiver-account-id/down.sql b/database/src/migrations/2021-01-20-152056_index-transactions-receiver-account-id/down.sql similarity index 100% rename from indexer/migrations/2021-01-20-152056_index-transactions-receiver-account-id/down.sql rename to database/src/migrations/2021-01-20-152056_index-transactions-receiver-account-id/down.sql diff --git a/indexer/migrations/2021-01-20-152056_index-transactions-receiver-account-id/up.sql b/database/src/migrations/2021-01-20-152056_index-transactions-receiver-account-id/up.sql similarity index 100% rename from indexer/migrations/2021-01-20-152056_index-transactions-receiver-account-id/up.sql rename to database/src/migrations/2021-01-20-152056_index-transactions-receiver-account-id/up.sql diff --git a/indexer/migrations/2021-03-11-123839_index-action-kind-and-status/down.sql b/database/src/migrations/2021-03-11-123839_index-action-kind-and-status/down.sql similarity index 100% rename from indexer/migrations/2021-03-11-123839_index-action-kind-and-status/down.sql rename to database/src/migrations/2021-03-11-123839_index-action-kind-and-status/down.sql diff --git a/indexer/migrations/2021-03-11-123839_index-action-kind-and-status/up.sql b/database/src/migrations/2021-03-11-123839_index-action-kind-and-status/up.sql similarity index 100% rename from indexer/migrations/2021-03-11-123839_index-action-kind-and-status/up.sql rename to database/src/migrations/2021-03-11-123839_index-action-kind-and-status/up.sql diff --git a/indexer/migrations/2021-04-22-090505_execution_outcomes_replace_chunk_relation_with_shard_id/down.sql b/database/src/migrations/2021-04-22-090505_execution_outcomes_replace_chunk_relation_with_shard_id/down.sql similarity index 100% rename from indexer/migrations/2021-04-22-090505_execution_outcomes_replace_chunk_relation_with_shard_id/down.sql rename to database/src/migrations/2021-04-22-090505_execution_outcomes_replace_chunk_relation_with_shard_id/down.sql diff --git a/indexer/migrations/2021-04-22-090505_execution_outcomes_replace_chunk_relation_with_shard_id/up.sql b/database/src/migrations/2021-04-22-090505_execution_outcomes_replace_chunk_relation_with_shard_id/up.sql similarity index 100% rename from indexer/migrations/2021-04-22-090505_execution_outcomes_replace_chunk_relation_with_shard_id/up.sql rename to database/src/migrations/2021-04-22-090505_execution_outcomes_replace_chunk_relation_with_shard_id/up.sql diff --git a/indexer/migrations/2021-04-28-154439_denormalize_action_receipt_actions/down.sql b/database/src/migrations/2021-04-28-154439_denormalize_action_receipt_actions/down.sql similarity index 100% rename from indexer/migrations/2021-04-28-154439_denormalize_action_receipt_actions/down.sql rename to database/src/migrations/2021-04-28-154439_denormalize_action_receipt_actions/down.sql diff --git a/indexer/migrations/2021-04-28-154439_denormalize_action_receipt_actions/up.sql b/database/src/migrations/2021-04-28-154439_denormalize_action_receipt_actions/up.sql similarity index 100% rename from indexer/migrations/2021-04-28-154439_denormalize_action_receipt_actions/up.sql rename to database/src/migrations/2021-04-28-154439_denormalize_action_receipt_actions/up.sql diff --git a/indexer/migrations/2021-05-06-093622_additional_indexes_for_action_receipt_actions/down.sql b/database/src/migrations/2021-05-06-093622_additional_indexes_for_action_receipt_actions/down.sql similarity index 100% rename from indexer/migrations/2021-05-06-093622_additional_indexes_for_action_receipt_actions/down.sql rename to database/src/migrations/2021-05-06-093622_additional_indexes_for_action_receipt_actions/down.sql diff --git a/indexer/migrations/2021-05-06-093622_additional_indexes_for_action_receipt_actions/up.sql b/database/src/migrations/2021-05-06-093622_additional_indexes_for_action_receipt_actions/up.sql similarity index 100% rename from indexer/migrations/2021-05-06-093622_additional_indexes_for_action_receipt_actions/up.sql rename to database/src/migrations/2021-05-06-093622_additional_indexes_for_action_receipt_actions/up.sql diff --git a/indexer/migrations/2021-05-07-115559_convert_args_base64_to_args_json/down.sql b/database/src/migrations/2021-05-07-115559_convert_args_base64_to_args_json/down.sql similarity index 100% rename from indexer/migrations/2021-05-07-115559_convert_args_base64_to_args_json/down.sql rename to database/src/migrations/2021-05-07-115559_convert_args_base64_to_args_json/down.sql diff --git a/indexer/migrations/2021-05-07-115559_convert_args_base64_to_args_json/up.sql b/database/src/migrations/2021-05-07-115559_convert_args_base64_to_args_json/up.sql similarity index 100% rename from indexer/migrations/2021-05-07-115559_convert_args_base64_to_args_json/up.sql rename to database/src/migrations/2021-05-07-115559_convert_args_base64_to_args_json/up.sql diff --git a/indexer/migrations/2021-05-10-084700_indexer_on_args_field/down.sql b/database/src/migrations/2021-05-10-084700_indexer_on_args_field/down.sql similarity index 100% rename from indexer/migrations/2021-05-10-084700_indexer_on_args_field/down.sql rename to database/src/migrations/2021-05-10-084700_indexer_on_args_field/down.sql diff --git a/indexer/migrations/2021-05-10-084700_indexer_on_args_field/up.sql b/database/src/migrations/2021-05-10-084700_indexer_on_args_field/up.sql similarity index 100% rename from indexer/migrations/2021-05-10-084700_indexer_on_args_field/up.sql rename to database/src/migrations/2021-05-10-084700_indexer_on_args_field/up.sql diff --git a/indexer/migrations/2021-05-27-154211_account_changes_unique_idx/down.sql b/database/src/migrations/2021-05-27-154211_account_changes_unique_idx/down.sql similarity index 100% rename from indexer/migrations/2021-05-27-154211_account_changes_unique_idx/down.sql rename to database/src/migrations/2021-05-27-154211_account_changes_unique_idx/down.sql diff --git a/indexer/migrations/2021-05-27-154211_account_changes_unique_idx/up.sql b/database/src/migrations/2021-05-27-154211_account_changes_unique_idx/up.sql similarity index 100% rename from indexer/migrations/2021-05-27-154211_account_changes_unique_idx/up.sql rename to database/src/migrations/2021-05-27-154211_account_changes_unique_idx/up.sql diff --git a/indexer/migrations/2021-06-02-173100_add_migration_state_change_reason_kind/down.sql b/database/src/migrations/2021-06-02-173100_add_migration_state_change_reason_kind/down.sql similarity index 100% rename from indexer/migrations/2021-06-02-173100_add_migration_state_change_reason_kind/down.sql rename to database/src/migrations/2021-06-02-173100_add_migration_state_change_reason_kind/down.sql diff --git a/indexer/migrations/2021-06-02-173100_add_migration_state_change_reason_kind/up.sql b/database/src/migrations/2021-06-02-173100_add_migration_state_change_reason_kind/up.sql similarity index 100% rename from indexer/migrations/2021-06-02-173100_add_migration_state_change_reason_kind/up.sql rename to database/src/migrations/2021-06-02-173100_add_migration_state_change_reason_kind/up.sql diff --git a/indexer/migrations/2021-08-02-183200_transactions_sorting_idx/down.sql b/database/src/migrations/2021-08-02-183200_transactions_sorting_idx/down.sql similarity index 100% rename from indexer/migrations/2021-08-02-183200_transactions_sorting_idx/down.sql rename to database/src/migrations/2021-08-02-183200_transactions_sorting_idx/down.sql diff --git a/indexer/migrations/2021-08-02-183200_transactions_sorting_idx/up.sql b/database/src/migrations/2021-08-02-183200_transactions_sorting_idx/up.sql similarity index 100% rename from indexer/migrations/2021-08-02-183200_transactions_sorting_idx/up.sql rename to database/src/migrations/2021-08-02-183200_transactions_sorting_idx/up.sql diff --git a/indexer/migrations/2021-08-04-151515_circulating_supply_table/down.sql b/database/src/migrations/2021-08-04-151515_circulating_supply_table/down.sql similarity index 100% rename from indexer/migrations/2021-08-04-151515_circulating_supply_table/down.sql rename to database/src/migrations/2021-08-04-151515_circulating_supply_table/down.sql diff --git a/indexer/migrations/2021-08-04-151515_circulating_supply_table/up.sql b/database/src/migrations/2021-08-04-151515_circulating_supply_table/up.sql similarity index 100% rename from indexer/migrations/2021-08-04-151515_circulating_supply_table/up.sql rename to database/src/migrations/2021-08-04-151515_circulating_supply_table/up.sql diff --git a/indexer/migrations/2021-08-06-123500_account_changes_ordering_column/down.sql b/database/src/migrations/2021-08-06-123500_account_changes_ordering_column/down.sql similarity index 100% rename from indexer/migrations/2021-08-06-123500_account_changes_ordering_column/down.sql rename to database/src/migrations/2021-08-06-123500_account_changes_ordering_column/down.sql diff --git a/indexer/migrations/2021-08-06-123500_account_changes_ordering_column/up.sql b/database/src/migrations/2021-08-06-123500_account_changes_ordering_column/up.sql similarity index 100% rename from indexer/migrations/2021-08-06-123500_account_changes_ordering_column/up.sql rename to database/src/migrations/2021-08-06-123500_account_changes_ordering_column/up.sql diff --git a/indexer/migrations/2021-08-11-163800_account_changes_ordering_idx/down.sql b/database/src/migrations/2021-08-11-163800_account_changes_ordering_idx/down.sql similarity index 100% rename from indexer/migrations/2021-08-11-163800_account_changes_ordering_idx/down.sql rename to database/src/migrations/2021-08-11-163800_account_changes_ordering_idx/down.sql diff --git a/indexer/migrations/2021-08-11-163800_account_changes_ordering_idx/up.sql b/database/src/migrations/2021-08-11-163800_account_changes_ordering_idx/up.sql similarity index 100% rename from indexer/migrations/2021-08-11-163800_account_changes_ordering_idx/up.sql rename to database/src/migrations/2021-08-11-163800_account_changes_ordering_idx/up.sql diff --git a/indexer/migrations/2021-10-04-100000_assets_nft/down.sql b/database/src/migrations/2021-10-04-100000_assets_nft/down.sql similarity index 100% rename from indexer/migrations/2021-10-04-100000_assets_nft/down.sql rename to database/src/migrations/2021-10-04-100000_assets_nft/down.sql diff --git a/indexer/migrations/2021-10-04-100000_assets_nft/up.sql b/database/src/migrations/2021-10-04-100000_assets_nft/up.sql similarity index 100% rename from indexer/migrations/2021-10-04-100000_assets_nft/up.sql rename to database/src/migrations/2021-10-04-100000_assets_nft/up.sql diff --git a/indexer/migrations/2021-10-14-180948_add_resharding_state_change_reason_kind/down.sql b/database/src/migrations/2021-10-14-180948_add_resharding_state_change_reason_kind/down.sql similarity index 100% rename from indexer/migrations/2021-10-14-180948_add_resharding_state_change_reason_kind/down.sql rename to database/src/migrations/2021-10-14-180948_add_resharding_state_change_reason_kind/down.sql diff --git a/indexer/migrations/2021-10-14-180948_add_resharding_state_change_reason_kind/up.sql b/database/src/migrations/2021-10-14-180948_add_resharding_state_change_reason_kind/up.sql similarity index 100% rename from indexer/migrations/2021-10-14-180948_add_resharding_state_change_reason_kind/up.sql rename to database/src/migrations/2021-10-14-180948_add_resharding_state_change_reason_kind/up.sql diff --git a/indexer/migrations/2022-01-12-100000_assets_ft/down.sql b/database/src/migrations/2022-01-12-100000_assets_ft/down.sql similarity index 100% rename from indexer/migrations/2022-01-12-100000_assets_ft/down.sql rename to database/src/migrations/2022-01-12-100000_assets_ft/down.sql diff --git a/indexer/migrations/2022-01-12-100000_assets_ft/up.sql b/database/src/migrations/2022-01-12-100000_assets_ft/up.sql similarity index 100% rename from indexer/migrations/2022-01-12-100000_assets_ft/up.sql rename to database/src/migrations/2022-01-12-100000_assets_ft/up.sql diff --git a/indexer/migrations/2022-01-26-184200_drop_duplicated_index_execution_outcome/down.sql b/database/src/migrations/2022-01-26-184200_drop_duplicated_index_execution_outcome/down.sql similarity index 100% rename from indexer/migrations/2022-01-26-184200_drop_duplicated_index_execution_outcome/down.sql rename to database/src/migrations/2022-01-26-184200_drop_duplicated_index_execution_outcome/down.sql diff --git a/indexer/migrations/2022-01-26-184200_drop_duplicated_index_execution_outcome/up.sql b/database/src/migrations/2022-01-26-184200_drop_duplicated_index_execution_outcome/up.sql similarity index 100% rename from indexer/migrations/2022-01-26-184200_drop_duplicated_index_execution_outcome/up.sql rename to database/src/migrations/2022-01-26-184200_drop_duplicated_index_execution_outcome/up.sql diff --git a/indexer/migrations/2022-01-26-184201_index_action_receipt_actions/down.sql b/database/src/migrations/2022-01-26-184201_index_action_receipt_actions/down.sql similarity index 100% rename from indexer/migrations/2022-01-26-184201_index_action_receipt_actions/down.sql rename to database/src/migrations/2022-01-26-184201_index_action_receipt_actions/down.sql diff --git a/indexer/migrations/2022-01-26-184201_index_action_receipt_actions/up.sql b/database/src/migrations/2022-01-26-184201_index_action_receipt_actions/up.sql similarity index 100% rename from indexer/migrations/2022-01-26-184201_index_action_receipt_actions/up.sql rename to database/src/migrations/2022-01-26-184201_index_action_receipt_actions/up.sql diff --git a/indexer/src/models/access_keys.rs b/database/src/models/access_keys.rs similarity index 100% rename from indexer/src/models/access_keys.rs rename to database/src/models/access_keys.rs diff --git a/indexer/src/models/account_changes.rs b/database/src/models/account_changes.rs similarity index 100% rename from indexer/src/models/account_changes.rs rename to database/src/models/account_changes.rs diff --git a/indexer/src/models/accounts.rs b/database/src/models/accounts.rs similarity index 100% rename from indexer/src/models/accounts.rs rename to database/src/models/accounts.rs diff --git a/indexer/src/models/aggregated/circulating_supply.rs b/database/src/models/aggregated/circulating_supply.rs similarity index 100% rename from indexer/src/models/aggregated/circulating_supply.rs rename to database/src/models/aggregated/circulating_supply.rs diff --git a/indexer/src/models/aggregated/mod.rs b/database/src/models/aggregated/mod.rs similarity index 100% rename from indexer/src/models/aggregated/mod.rs rename to database/src/models/aggregated/mod.rs diff --git a/indexer/src/models/assets/fungible_token_events.rs b/database/src/models/assets/fungible_token_events.rs similarity index 100% rename from indexer/src/models/assets/fungible_token_events.rs rename to database/src/models/assets/fungible_token_events.rs diff --git a/indexer/src/models/assets/mod.rs b/database/src/models/assets/mod.rs similarity index 100% rename from indexer/src/models/assets/mod.rs rename to database/src/models/assets/mod.rs diff --git a/indexer/src/models/assets/non_fungible_token_events.rs b/database/src/models/assets/non_fungible_token_events.rs similarity index 100% rename from indexer/src/models/assets/non_fungible_token_events.rs rename to database/src/models/assets/non_fungible_token_events.rs diff --git a/indexer/src/models/blocks.rs b/database/src/models/blocks.rs similarity index 100% rename from indexer/src/models/blocks.rs rename to database/src/models/blocks.rs diff --git a/indexer/src/models/chunks.rs b/database/src/models/chunks.rs similarity index 100% rename from indexer/src/models/chunks.rs rename to database/src/models/chunks.rs diff --git a/indexer/src/models/enums.rs b/database/src/models/enums.rs similarity index 100% rename from indexer/src/models/enums.rs rename to database/src/models/enums.rs diff --git a/indexer/src/models/execution_outcomes.rs b/database/src/models/execution_outcomes.rs similarity index 100% rename from indexer/src/models/execution_outcomes.rs rename to database/src/models/execution_outcomes.rs diff --git a/indexer/src/models/mod.rs b/database/src/models/mod.rs similarity index 100% rename from indexer/src/models/mod.rs rename to database/src/models/mod.rs diff --git a/indexer/src/models/receipts.rs b/database/src/models/receipts.rs similarity index 100% rename from indexer/src/models/receipts.rs rename to database/src/models/receipts.rs diff --git a/indexer/src/models/serializers.rs b/database/src/models/serializers.rs similarity index 100% rename from indexer/src/models/serializers.rs rename to database/src/models/serializers.rs diff --git a/indexer/src/models/transactions.rs b/database/src/models/transactions.rs similarity index 100% rename from indexer/src/models/transactions.rs rename to database/src/models/transactions.rs diff --git a/indexer/src/retriable.rs b/database/src/retriable.rs similarity index 100% rename from indexer/src/retriable.rs rename to database/src/retriable.rs diff --git a/indexer/src/schema.patch b/database/src/schema.patch similarity index 100% rename from indexer/src/schema.patch rename to database/src/schema.patch diff --git a/indexer/src/schema.rs b/database/src/schema.rs similarity index 100% rename from indexer/src/schema.rs rename to database/src/schema.rs diff --git a/indexer/Cargo.toml b/indexer/Cargo.toml index bfc003d4..edd0a9dd 100644 --- a/indexer/Cargo.toml +++ b/indexer/Cargo.toml @@ -11,34 +11,24 @@ actix-rt = "2.2.0" anyhow = "1.0.51" aws-types = "0.13.0" aws-sdk-s3 = "0.13.0" -base64 = "0.11" -bigdecimal = "=0.1.0" borsh = "0.7.1" cached = "0.23.0" chrono = "0.4.19" clap = { version = "3.1.6", features = ["color", "derive", "env"] } -diesel = { version = "1.4.7", features = ["postgres", "numeric", "serde_json"] } -# Using hacky diesel-derive-enum https://github.com/adwhit/diesel-derive-enum/issues/52 -diesel-derive-enum = { git = "https://github.com/khorolets/diesel-derive-enum.git", branch = "lookup-hack", features = ["postgres"] } dotenv = "0.15.0" futures = "0.3.5" -hex = "0.4" itertools = "0.10.3" # syn version conflict, replace with crates.io version once released near-sdk = { git = "https://github.com/near/near-sdk-rs", rev="03487c184d37b0382dd9bd41c57466acad58fc1f" } -num-traits = "0.2.11" openssl-probe = { version = "0.1.2" } r2d2 = "0.8.8" -serde = { version = "1", features = ["derive"] } -serde_json = "1.0.55" tokio = { version = "1.1", features = ["sync", "time"] } tokio-stream = { version = "0.1" } tracing = "0.1.13" tracing-subscriber = "0.2.4" uint = { version = "0.8.3", default-features = false } -actix-diesel = { git = "https://github.com/frol/actix-diesel", rev = "3a001986c89dfabfc3c448d8bae28525101b4992" } near-jsonrpc-client = "0.4.0-beta.0" near-lake-framework = "0.5.2" -near-primitives = "0.14" -near-crypto = "0.14" + +explorer-database = { path = "../database" } diff --git a/indexer/src/configs.rs b/indexer/src/configs.rs index 4394c1ea..a363b2ca 100644 --- a/indexer/src/configs.rs +++ b/indexer/src/configs.rs @@ -1,6 +1,8 @@ use clap::{Parser, Subcommand}; use tracing_subscriber::EnvFilter; +use explorer_database::{db_adapters, models}; + use near_jsonrpc_client::{methods, JsonRpcClient}; use near_lake_framework::near_indexer_primitives::types::{BlockReference, Finality}; @@ -118,11 +120,9 @@ async fn get_start_block_height(opts: &Opts) -> u64 { match opts.start_options() { StartOptions::FromBlock { height } => *height, StartOptions::FromInterruption => { - let pool = crate::models::establish_connection(&opts.database_url); - let last_indexed_block: u64 = match crate::db_adapters::blocks::latest_block_height( - &pool, - ) - .await + let pool = models::establish_connection(&opts.database_url); + let last_indexed_block: u64 = match db_adapters::blocks::latest_block_height(&pool) + .await { Ok(last_indexed_block) => { if let Some(last_indexed_block) = last_indexed_block { diff --git a/indexer/src/main.rs b/indexer/src/main.rs index d118d1bf..d1024323 100644 --- a/indexer/src/main.rs +++ b/indexer/src/main.rs @@ -1,30 +1,21 @@ use clap::Parser; -#[macro_use] -extern crate diesel; pub use cached::SizedCache; -use diesel::PgConnection; use futures::future::try_join_all; use futures::{try_join, StreamExt}; use tokio::sync::Mutex; use tracing::{debug, info}; +use explorer_database::{db_adapters, models}; + use crate::configs::Opts; mod configs; -mod db_adapters; -mod models; -mod schema; -#[macro_use] -mod retriable; // Categories for logging const INDEXER_FOR_EXPLORER: &str = "indexer_for_explorer"; // const AGGREGATED: &str = "aggregated"; -const INTERVAL: std::time::Duration = std::time::Duration::from_millis(100); -const MAX_DELAY_TIME: std::time::Duration = std::time::Duration::from_secs(120); - #[derive(Clone, Hash, PartialEq, Eq, Debug)] pub enum ReceiptOrDataId { ReceiptId(near_lake_framework::near_indexer_primitives::CryptoHash), @@ -40,7 +31,7 @@ pub type ReceiptsCache = std::sync::Arc>>; async fn handle_message( - pool: &actix_diesel::Database, + pool: &explorer_database::actix_diesel::Database, streamer_message: near_lake_framework::near_indexer_primitives::StreamerMessage, strict_mode: bool, receipts_cache: ReceiptsCache, From fa2f47f4346100543136056cde743d091c5a2f0b Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Mon, 31 Oct 2022 16:15:14 +1300 Subject: [PATCH 12/42] refactor: Change `database` logging target to `explorer_database` --- database/src/db_adapters/assets/events.rs | 4 ++-- database/src/db_adapters/blocks.rs | 2 +- database/src/db_adapters/receipts.rs | 8 ++++---- database/src/lib.rs | 2 +- database/src/retriable.rs | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/database/src/db_adapters/assets/events.rs b/database/src/db_adapters/assets/events.rs index 40f1184b..5698377f 100644 --- a/database/src/db_adapters/assets/events.rs +++ b/database/src/db_adapters/assets/events.rs @@ -33,7 +33,7 @@ pub(crate) async fn detect_db_error( return true; } else if constraint_name == broken_data_constraint_name { warn!( - target: crate::INDEXER_FOR_EXPLORER, + target: crate::EXPLORER_DATABASE, "assets::events: data inconsistency is found" ); } @@ -95,7 +95,7 @@ fn extract_events( Ok(result) => Some(result), Err(err) => { warn!( - target: crate::INDEXER_FOR_EXPLORER, + target: crate::EXPLORER_DATABASE, "Provided event log does not correspond to any of formats defined in NEP. Will ignore this event. \n {:#?} \n{:#?}", err, untrimmed_log, diff --git a/database/src/db_adapters/blocks.rs b/database/src/db_adapters/blocks.rs index ddd9518c..263510d4 100644 --- a/database/src/db_adapters/blocks.rs +++ b/database/src/db_adapters/blocks.rs @@ -28,7 +28,7 @@ pub(crate) async fn store_block( pub(crate) async fn latest_block_height( pool: &actix_diesel::Database, ) -> anyhow::Result> { - tracing::debug!(target: crate::INDEXER_FOR_EXPLORER, "fetching latest"); + tracing::debug!(target: crate::EXPLORER_DATABASE, "fetching latest"); Ok(schema::blocks::table .select((schema::blocks::dsl::block_height,)) .order(schema::blocks::dsl::block_height.desc()) diff --git a/database/src/db_adapters/receipts.rs b/database/src/db_adapters/receipts.rs index d327b8c4..f9992104 100644 --- a/database/src/db_adapters/receipts.rs +++ b/database/src/db_adapters/receipts.rs @@ -92,7 +92,7 @@ async fn store_chunk_receipts( )) } else { warn!( - target: crate::INDEXER_FOR_EXPLORER, + target: crate::EXPLORER_DATABASE, "Skipping Receipt {} as we can't find parent Transaction for it. Happen in block hash {}, chunk hash {}", r.receipt_id.to_string(), block_hash, @@ -218,7 +218,7 @@ async fn find_tx_hashes_for_receipts( } warn!( - target: crate::INDEXER_FOR_EXPLORER, + target: crate::EXPLORER_DATABASE, "Looking for parent transaction hash in database for {} receipts {:#?}", &receipts.len(), &receipts, @@ -281,7 +281,7 @@ async fn find_tx_hashes_for_receipts( } Err(async_error) => { error!( - target: crate::INDEXER_FOR_EXPLORER, + target: crate::EXPLORER_DATABASE, "Error occurred while fetching the parent receipt for Receipt. Retrying in {} milliseconds... \n {:#?}", interval.as_millis(), async_error, @@ -450,7 +450,7 @@ async fn find_tx_hashes_for_receipts( } } warn!( - target: crate::INDEXER_FOR_EXPLORER, + target: crate::EXPLORER_DATABASE, "Going to retry to find parent transactions for receipts in {} milliseconds... \n {:#?}\n block hash {} \nchunk hash {}", find_tx_retry_interval.as_millis(), &receipts, diff --git a/database/src/lib.rs b/database/src/lib.rs index fc79eb80..e01f6efd 100644 --- a/database/src/lib.rs +++ b/database/src/lib.rs @@ -11,6 +11,6 @@ mod schema; mod retriable; const INTERVAL: std::time::Duration = std::time::Duration::from_millis(100); -const INDEXER_FOR_EXPLORER: &str = "indexer_for_explorer"; +const EXPLORER_DATABASE: &str = "explorer_database"; const MAX_DELAY_TIME: std::time::Duration = std::time::Duration::from_secs(120); diff --git a/database/src/retriable.rs b/database/src/retriable.rs index c2d53635..dd0649ab 100644 --- a/database/src/retriable.rs +++ b/database/src/retriable.rs @@ -23,7 +23,7 @@ macro_rules! await_retry_or_panic { })? tracing::error!( - target: crate::INDEXER_FOR_EXPLORER, + target: crate::EXPLORER_DATABASE, "Error occurred during {}: \n{:#?} \n{:#?} \n Retrying in {} milliseconds...", async_error, &$error_message, From a8e27a9d8a2e3d8c1fce9bd651c2de9817517215 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Tue, 1 Nov 2022 10:59:45 +1300 Subject: [PATCH 13/42] refactor: Rename `db_adapters` -> `adapters` --- .../{db_adapters => adapters}/access_keys.rs | 0 .../account_changes.rs | 0 .../src/{db_adapters => adapters}/accounts.rs | 0 .../assets/event_types.rs | 0 .../assets/events.rs | 2 +- .../assets/fungible_token_events.rs | 2 +- .../{db_adapters => adapters}/assets/mod.rs | 0 .../assets/non_fungible_token_events.rs | 2 +- .../src/{db_adapters => adapters}/blocks.rs | 0 .../src/{db_adapters => adapters}/chunks.rs | 0 .../execution_outcomes.rs | 0 database/src/{db_adapters => adapters}/mod.rs | 0 .../src/{db_adapters => adapters}/receipts.rs | 0 .../{db_adapters => adapters}/transactions.rs | 0 database/src/lib.rs | 2 +- indexer/src/configs.rs | 6 ++---- indexer/src/main.rs | 20 +++++++++---------- 17 files changed, 16 insertions(+), 18 deletions(-) rename database/src/{db_adapters => adapters}/access_keys.rs (100%) rename database/src/{db_adapters => adapters}/account_changes.rs (100%) rename database/src/{db_adapters => adapters}/accounts.rs (100%) rename database/src/{db_adapters => adapters}/assets/event_types.rs (100%) rename database/src/{db_adapters => adapters}/assets/events.rs (99%) rename database/src/{db_adapters => adapters}/assets/fungible_token_events.rs (98%) rename database/src/{db_adapters => adapters}/assets/mod.rs (100%) rename database/src/{db_adapters => adapters}/assets/non_fungible_token_events.rs (99%) rename database/src/{db_adapters => adapters}/blocks.rs (100%) rename database/src/{db_adapters => adapters}/chunks.rs (100%) rename database/src/{db_adapters => adapters}/execution_outcomes.rs (100%) rename database/src/{db_adapters => adapters}/mod.rs (100%) rename database/src/{db_adapters => adapters}/receipts.rs (100%) rename database/src/{db_adapters => adapters}/transactions.rs (100%) diff --git a/database/src/db_adapters/access_keys.rs b/database/src/adapters/access_keys.rs similarity index 100% rename from database/src/db_adapters/access_keys.rs rename to database/src/adapters/access_keys.rs diff --git a/database/src/db_adapters/account_changes.rs b/database/src/adapters/account_changes.rs similarity index 100% rename from database/src/db_adapters/account_changes.rs rename to database/src/adapters/account_changes.rs diff --git a/database/src/db_adapters/accounts.rs b/database/src/adapters/accounts.rs similarity index 100% rename from database/src/db_adapters/accounts.rs rename to database/src/adapters/accounts.rs diff --git a/database/src/db_adapters/assets/event_types.rs b/database/src/adapters/assets/event_types.rs similarity index 100% rename from database/src/db_adapters/assets/event_types.rs rename to database/src/adapters/assets/event_types.rs diff --git a/database/src/db_adapters/assets/events.rs b/database/src/adapters/assets/events.rs similarity index 99% rename from database/src/db_adapters/assets/events.rs rename to database/src/adapters/assets/events.rs index 5698377f..4bb96a1c 100644 --- a/database/src/db_adapters/assets/events.rs +++ b/database/src/adapters/assets/events.rs @@ -2,7 +2,7 @@ use actix_diesel::{AsyncError, Database}; use diesel::PgConnection; use tracing::warn; -use crate::db_adapters::assets; +use crate::adapters::assets; use super::event_types; diff --git a/database/src/db_adapters/assets/fungible_token_events.rs b/database/src/adapters/assets/fungible_token_events.rs similarity index 98% rename from database/src/db_adapters/assets/fungible_token_events.rs rename to database/src/adapters/assets/fungible_token_events.rs index 4ff917d0..b1dc170d 100644 --- a/database/src/db_adapters/assets/fungible_token_events.rs +++ b/database/src/adapters/assets/fungible_token_events.rs @@ -3,7 +3,7 @@ use actix_diesel::{AsyncError, Database}; use bigdecimal::BigDecimal; use diesel::PgConnection; -use crate::db_adapters::{assets, CHUNK_SIZE_FOR_BATCH_INSERT}; +use crate::adapters::{assets, CHUNK_SIZE_FOR_BATCH_INSERT}; use crate::models; use crate::schema; diff --git a/database/src/db_adapters/assets/mod.rs b/database/src/adapters/assets/mod.rs similarity index 100% rename from database/src/db_adapters/assets/mod.rs rename to database/src/adapters/assets/mod.rs diff --git a/database/src/db_adapters/assets/non_fungible_token_events.rs b/database/src/adapters/assets/non_fungible_token_events.rs similarity index 99% rename from database/src/db_adapters/assets/non_fungible_token_events.rs rename to database/src/adapters/assets/non_fungible_token_events.rs index 4b8b303b..cc77fae6 100644 --- a/database/src/db_adapters/assets/non_fungible_token_events.rs +++ b/database/src/adapters/assets/non_fungible_token_events.rs @@ -3,7 +3,7 @@ use actix_diesel::{AsyncError, Database}; use bigdecimal::BigDecimal; use diesel::PgConnection; -use crate::db_adapters::{assets, CHUNK_SIZE_FOR_BATCH_INSERT}; +use crate::adapters::{assets, CHUNK_SIZE_FOR_BATCH_INSERT}; use crate::models; use crate::schema; diff --git a/database/src/db_adapters/blocks.rs b/database/src/adapters/blocks.rs similarity index 100% rename from database/src/db_adapters/blocks.rs rename to database/src/adapters/blocks.rs diff --git a/database/src/db_adapters/chunks.rs b/database/src/adapters/chunks.rs similarity index 100% rename from database/src/db_adapters/chunks.rs rename to database/src/adapters/chunks.rs diff --git a/database/src/db_adapters/execution_outcomes.rs b/database/src/adapters/execution_outcomes.rs similarity index 100% rename from database/src/db_adapters/execution_outcomes.rs rename to database/src/adapters/execution_outcomes.rs diff --git a/database/src/db_adapters/mod.rs b/database/src/adapters/mod.rs similarity index 100% rename from database/src/db_adapters/mod.rs rename to database/src/adapters/mod.rs diff --git a/database/src/db_adapters/receipts.rs b/database/src/adapters/receipts.rs similarity index 100% rename from database/src/db_adapters/receipts.rs rename to database/src/adapters/receipts.rs diff --git a/database/src/db_adapters/transactions.rs b/database/src/adapters/transactions.rs similarity index 100% rename from database/src/db_adapters/transactions.rs rename to database/src/adapters/transactions.rs diff --git a/database/src/lib.rs b/database/src/lib.rs index e01f6efd..eff624c7 100644 --- a/database/src/lib.rs +++ b/database/src/lib.rs @@ -3,7 +3,7 @@ pub extern crate diesel; pub use actix_diesel; -pub mod db_adapters; +pub mod adapters; pub mod models; mod schema; diff --git a/indexer/src/configs.rs b/indexer/src/configs.rs index a363b2ca..35e5a666 100644 --- a/indexer/src/configs.rs +++ b/indexer/src/configs.rs @@ -1,7 +1,7 @@ use clap::{Parser, Subcommand}; use tracing_subscriber::EnvFilter; -use explorer_database::{db_adapters, models}; +use explorer_database::{adapters, models}; use near_jsonrpc_client::{methods, JsonRpcClient}; use near_lake_framework::near_indexer_primitives::types::{BlockReference, Finality}; @@ -121,9 +121,7 @@ async fn get_start_block_height(opts: &Opts) -> u64 { StartOptions::FromBlock { height } => *height, StartOptions::FromInterruption => { let pool = models::establish_connection(&opts.database_url); - let last_indexed_block: u64 = match db_adapters::blocks::latest_block_height(&pool) - .await - { + let last_indexed_block: u64 = match adapters::blocks::latest_block_height(&pool).await { Ok(last_indexed_block) => { if let Some(last_indexed_block) = last_indexed_block { last_indexed_block diff --git a/indexer/src/main.rs b/indexer/src/main.rs index d1024323..a199e766 100644 --- a/indexer/src/main.rs +++ b/indexer/src/main.rs @@ -6,7 +6,7 @@ use futures::{try_join, StreamExt}; use tokio::sync::Mutex; use tracing::{debug, info}; -use explorer_database::{db_adapters, models}; +use explorer_database::{adapters, models}; use crate::configs::Opts; @@ -40,10 +40,10 @@ async fn handle_message( target: INDEXER_FOR_EXPLORER, "ReceiptsCache #{} \n {:#?}", streamer_message.block.header.height, &receipts_cache ); - db_adapters::blocks::store_block(pool, &streamer_message.block).await?; + adapters::blocks::store_block(pool, &streamer_message.block).await?; // Chunks - db_adapters::chunks::store_chunks( + adapters::chunks::store_chunks( pool, &streamer_message.shards, &streamer_message.block.header.hash, @@ -51,7 +51,7 @@ async fn handle_message( .await?; // Transactions - let transactions_future = db_adapters::transactions::store_transactions( + let transactions_future = adapters::transactions::store_transactions( pool, &streamer_message.shards, &streamer_message.block.header.hash, @@ -61,7 +61,7 @@ async fn handle_message( ); // Receipts - let receipts_future = db_adapters::receipts::store_receipts( + let receipts_future = adapters::receipts::store_receipts( pool, &streamer_message.shards, &streamer_message.block.header.hash, @@ -78,7 +78,7 @@ async fn handle_message( try_join!(transactions_future, receipts_future)?; // ExecutionOutcomes - let execution_outcomes_future = db_adapters::execution_outcomes::store_execution_outcomes( + let execution_outcomes_future = adapters::execution_outcomes::store_execution_outcomes( pool, &streamer_message.shards, streamer_message.block.header.timestamp, @@ -88,7 +88,7 @@ async fn handle_message( // Accounts let accounts_future = async { let futures = streamer_message.shards.iter().map(|shard| { - db_adapters::accounts::handle_accounts( + adapters::accounts::handle_accounts( pool, &shard.receipt_execution_outcomes, streamer_message.block.header.height, @@ -99,13 +99,13 @@ async fn handle_message( }; // Event-based entities (FT, NFT) - let assets_events_future = db_adapters::assets::events::store_events(pool, &streamer_message); + let assets_events_future = adapters::assets::events::store_events(pool, &streamer_message); if strict_mode { // AccessKeys let access_keys_future = async { let futures = streamer_message.shards.iter().map(|shard| { - db_adapters::access_keys::handle_access_keys( + adapters::access_keys::handle_access_keys( pool, &shard.state_changes, streamer_message.block.header.height, @@ -116,7 +116,7 @@ async fn handle_message( }; // StateChange related to Account - let account_changes_future = db_adapters::account_changes::store_account_changes( + let account_changes_future = adapters::account_changes::store_account_changes( pool, &streamer_message.shards, &streamer_message.block.header.hash, From a4e3eea2a06c2b46095d20b417e0150cd154ea46 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Tue, 1 Nov 2022 11:07:56 +1300 Subject: [PATCH 14/42] refactor: Make current in-use DB funcions public --- database/src/adapters/access_keys.rs | 2 +- database/src/adapters/account_changes.rs | 2 +- database/src/adapters/accounts.rs | 2 +- database/src/adapters/assets/events.rs | 2 +- database/src/adapters/assets/mod.rs | 2 +- database/src/adapters/blocks.rs | 4 ++-- database/src/adapters/chunks.rs | 2 +- database/src/adapters/execution_outcomes.rs | 2 +- database/src/adapters/mod.rs | 18 +++++++++--------- database/src/adapters/receipts.rs | 2 +- database/src/adapters/transactions.rs | 2 +- database/src/models/mod.rs | 2 +- 12 files changed, 21 insertions(+), 21 deletions(-) diff --git a/database/src/adapters/access_keys.rs b/database/src/adapters/access_keys.rs index 1a75542f..3b2daf72 100644 --- a/database/src/adapters/access_keys.rs +++ b/database/src/adapters/access_keys.rs @@ -8,7 +8,7 @@ use futures::try_join; use crate::models; use crate::schema; -pub(crate) async fn handle_access_keys( +pub async fn handle_access_keys( pool: &actix_diesel::Database, state_changes: &[near_lake_framework::near_indexer_primitives::views::StateChangeWithCauseView], block_height: near_lake_framework::near_indexer_primitives::types::BlockHeight, diff --git a/database/src/adapters/account_changes.rs b/database/src/adapters/account_changes.rs index 375ee09a..4635ee96 100644 --- a/database/src/adapters/account_changes.rs +++ b/database/src/adapters/account_changes.rs @@ -6,7 +6,7 @@ use crate::models; use crate::schema; /// Saves state change related to account to database -pub(crate) async fn store_account_changes( +pub async fn store_account_changes( pool: &actix_diesel::Database, shards: &[near_lake_framework::near_indexer_primitives::IndexerShard], block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, diff --git a/database/src/adapters/accounts.rs b/database/src/adapters/accounts.rs index 01a72daa..5d6bd22e 100644 --- a/database/src/adapters/accounts.rs +++ b/database/src/adapters/accounts.rs @@ -9,7 +9,7 @@ use crate::models; use crate::schema; /// Saves new Accounts to database or deletes the ones should be deleted -pub(crate) async fn handle_accounts( +pub async fn handle_accounts( pool: &actix_diesel::Database, outcomes: &[near_lake_framework::near_indexer_primitives::IndexerExecutionOutcomeWithReceipt], block_height: near_lake_framework::near_indexer_primitives::types::BlockHeight, diff --git a/database/src/adapters/assets/events.rs b/database/src/adapters/assets/events.rs index 4bb96a1c..a586f539 100644 --- a/database/src/adapters/assets/events.rs +++ b/database/src/adapters/assets/events.rs @@ -6,7 +6,7 @@ use crate::adapters::assets; use super::event_types; -pub(crate) async fn store_events( +pub async fn store_events( pool: &Database, streamer_message: &near_lake_framework::near_indexer_primitives::StreamerMessage, ) -> anyhow::Result<()> { diff --git a/database/src/adapters/assets/mod.rs b/database/src/adapters/assets/mod.rs index e70925cf..ed6d5f98 100644 --- a/database/src/adapters/assets/mod.rs +++ b/database/src/adapters/assets/mod.rs @@ -1,4 +1,4 @@ mod event_types; -pub(crate) mod events; +pub mod events; pub(crate) mod fungible_token_events; pub(crate) mod non_fungible_token_events; diff --git a/database/src/adapters/blocks.rs b/database/src/adapters/blocks.rs index 263510d4..fa5833e1 100644 --- a/database/src/adapters/blocks.rs +++ b/database/src/adapters/blocks.rs @@ -6,7 +6,7 @@ use crate::models; use crate::schema; /// Saves block to database -pub(crate) async fn store_block( +pub async fn store_block( pool: &actix_diesel::Database, block: &near_lake_framework::near_indexer_primitives::views::BlockView, ) -> anyhow::Result<()> { @@ -25,7 +25,7 @@ pub(crate) async fn store_block( } /// Gets the latest block's height from database -pub(crate) async fn latest_block_height( +pub async fn latest_block_height( pool: &actix_diesel::Database, ) -> anyhow::Result> { tracing::debug!(target: crate::EXPLORER_DATABASE, "fetching latest"); diff --git a/database/src/adapters/chunks.rs b/database/src/adapters/chunks.rs index c177e77a..53f4829d 100644 --- a/database/src/adapters/chunks.rs +++ b/database/src/adapters/chunks.rs @@ -5,7 +5,7 @@ use crate::models; use crate::schema; /// Saves chunks to database -pub(crate) async fn store_chunks( +pub async fn store_chunks( pool: &actix_diesel::Database, shards: &[near_lake_framework::near_indexer_primitives::IndexerShard], block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, diff --git a/database/src/adapters/execution_outcomes.rs b/database/src/adapters/execution_outcomes.rs index 8ade3a53..4fbf3ffc 100644 --- a/database/src/adapters/execution_outcomes.rs +++ b/database/src/adapters/execution_outcomes.rs @@ -6,7 +6,7 @@ use futures::future::try_join_all; use crate::models; use crate::schema; -pub(crate) async fn store_execution_outcomes( +pub async fn store_execution_outcomes( pool: &actix_diesel::Database, shards: &[near_lake_framework::near_indexer_primitives::IndexerShard], block_timestamp: u64, diff --git a/database/src/adapters/mod.rs b/database/src/adapters/mod.rs index b8684dc6..cdb20a09 100644 --- a/database/src/adapters/mod.rs +++ b/database/src/adapters/mod.rs @@ -1,11 +1,11 @@ -pub(crate) mod access_keys; -pub(crate) mod account_changes; -pub(crate) mod accounts; -pub(crate) mod assets; -pub(crate) mod blocks; -pub(crate) mod chunks; -pub(crate) mod execution_outcomes; -pub(crate) mod receipts; -pub(crate) mod transactions; +pub mod access_keys; +pub mod account_changes; +pub mod accounts; +pub mod assets; +pub mod blocks; +pub mod chunks; +pub mod execution_outcomes; +pub mod receipts; +pub mod transactions; const CHUNK_SIZE_FOR_BATCH_INSERT: usize = 500; diff --git a/database/src/adapters/receipts.rs b/database/src/adapters/receipts.rs index f9992104..f3855baa 100644 --- a/database/src/adapters/receipts.rs +++ b/database/src/adapters/receipts.rs @@ -15,7 +15,7 @@ use crate::models; use crate::schema; /// Saves receipts to database -pub(crate) async fn store_receipts( +pub async fn store_receipts( pool: &actix_diesel::Database, shards: &[near_lake_framework::near_indexer_primitives::IndexerShard], block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, diff --git a/database/src/adapters/transactions.rs b/database/src/adapters/transactions.rs index 3e794fc3..5ff76326 100644 --- a/database/src/adapters/transactions.rs +++ b/database/src/adapters/transactions.rs @@ -8,7 +8,7 @@ use crate::models; use crate::schema; /// Saves Transactions to database -pub(crate) async fn store_transactions( +pub async fn store_transactions( pool: &actix_diesel::Database, shards: &[near_lake_framework::near_indexer_primitives::IndexerShard], block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, diff --git a/database/src/models/mod.rs b/database/src/models/mod.rs index 5f235c2e..ce71dd54 100644 --- a/database/src/models/mod.rs +++ b/database/src/models/mod.rs @@ -26,7 +26,7 @@ pub mod receipts; mod serializers; pub mod transactions; -pub(crate) fn establish_connection(database_url: &str) -> actix_diesel::Database { +pub fn establish_connection(database_url: &str) -> actix_diesel::Database { actix_diesel::Database::builder() .pool_max_size(30) .open(database_url) From 73dfd6ab2cc631419f042ca82f68a81031a8eb8e Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Tue, 1 Nov 2022 11:26:49 +1300 Subject: [PATCH 15/42] refactor: Extract cache logic to `database` lib --- database/src/adapters/execution_outcomes.rs | 8 +- database/src/adapters/receipts.rs | 91 ++++++++++++--------- database/src/adapters/transactions.rs | 6 +- database/src/lib.rs | 1 + database/src/receipts_cache.rs | 16 ++++ indexer/src/main.rs | 20 +---- 6 files changed, 81 insertions(+), 61 deletions(-) create mode 100644 database/src/receipts_cache.rs diff --git a/database/src/adapters/execution_outcomes.rs b/database/src/adapters/execution_outcomes.rs index 4fbf3ffc..d7a3aeda 100644 --- a/database/src/adapters/execution_outcomes.rs +++ b/database/src/adapters/execution_outcomes.rs @@ -10,7 +10,7 @@ pub async fn store_execution_outcomes( pool: &actix_diesel::Database, shards: &[near_lake_framework::near_indexer_primitives::IndexerShard], block_timestamp: u64, - receipts_cache: crate::ReceiptsCache, + receipts_cache: crate::receipts_cache::ReceiptsCache, ) -> anyhow::Result<()> { let futures = shards.iter().map(|shard| { store_execution_outcomes_for_chunk( @@ -31,7 +31,7 @@ pub async fn store_execution_outcomes_for_chunk( execution_outcomes: &[near_lake_framework::near_indexer_primitives::IndexerExecutionOutcomeWithReceipt], shard_id: near_lake_framework::near_indexer_primitives::types::ShardId, block_timestamp: u64, - receipts_cache: crate::ReceiptsCache, + receipts_cache: crate::receipts_cache::ReceiptsCache, ) -> anyhow::Result<()> { let mut outcome_models: Vec = vec![]; let mut outcome_receipt_models: Vec = @@ -42,7 +42,7 @@ pub async fn store_execution_outcomes_for_chunk( // remove it from cache once found as it is not expected to observe the Receipt for // second time let parent_transaction_hash = receipts_cache_lock.cache_remove( - &crate::ReceiptOrDataId::ReceiptId(outcome.execution_outcome.id), + &crate::receipts_cache::ReceiptOrDataId::ReceiptId(outcome.execution_outcome.id), ); let model = models::execution_outcomes::ExecutionOutcome::from_execution_outcome( @@ -66,7 +66,7 @@ pub async fn store_execution_outcomes_for_chunk( // could find their parents in cache if let Some(transaction_hash) = &parent_transaction_hash { receipts_cache_lock.cache_set( - crate::ReceiptOrDataId::ReceiptId(*receipt_id), + crate::receipts_cache::ReceiptOrDataId::ReceiptId(*receipt_id), transaction_hash.clone(), ); } diff --git a/database/src/adapters/receipts.rs b/database/src/adapters/receipts.rs index f3855baa..cc6d8c42 100644 --- a/database/src/adapters/receipts.rs +++ b/database/src/adapters/receipts.rs @@ -21,7 +21,7 @@ pub async fn store_receipts( block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, block_timestamp: u64, strict_mode: bool, - receipts_cache: crate::ReceiptsCache, + receipts_cache: crate::receipts_cache::ReceiptsCache, ) -> anyhow::Result<()> { let futures = shards .iter() @@ -49,7 +49,7 @@ async fn store_chunk_receipts( chunk_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, block_timestamp: u64, strict_mode: bool, - receipts_cache: crate::ReceiptsCache, + receipts_cache: crate::receipts_cache::ReceiptsCache, ) -> anyhow::Result<()> { let mut skipping_receipt_ids = std::collections::HashSet::< near_lake_framework::near_indexer_primitives::CryptoHash, @@ -75,10 +75,10 @@ async fn store_chunk_receipts( // In case of Data Receipt we are looking for DataId let receipt_or_data_id = match r.receipt { near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { .. } => { - crate::ReceiptOrDataId::ReceiptId(r.receipt_id) + crate::receipts_cache::ReceiptOrDataId::ReceiptId(r.receipt_id) } near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Data { data_id, .. } => { - crate::ReceiptOrDataId::DataId(data_id) + crate::receipts_cache::ReceiptOrDataId::DataId(data_id) } }; if let Some(transaction_hash) = tx_hashes_for_receipts.get(&receipt_or_data_id) { @@ -115,12 +115,12 @@ async fn store_chunk_receipts( } = &receipt.receipt { if !output_data_receivers.is_empty() { - if let Some(transaction_hash) = tx_hashes_for_receipts - .get(&crate::ReceiptOrDataId::ReceiptId(receipt.receipt_id)) - { + if let Some(transaction_hash) = tx_hashes_for_receipts.get( + &crate::receipts_cache::ReceiptOrDataId::ReceiptId(receipt.receipt_id), + ) { for data_receiver in output_data_receivers { receipts_cache_lock.cache_set( - crate::ReceiptOrDataId::DataId(data_receiver.data_id), + crate::receipts_cache::ReceiptOrDataId::DataId(data_receiver.data_id), transaction_hash.clone(), ); } @@ -162,11 +162,16 @@ async fn find_tx_hashes_for_receipts( strict_mode: bool, block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, chunk_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, - receipts_cache: crate::ReceiptsCache, -) -> anyhow::Result> { + receipts_cache: crate::receipts_cache::ReceiptsCache, +) -> anyhow::Result< + HashMap< + crate::receipts_cache::ReceiptOrDataId, + crate::receipts_cache::ParentTransactionHashString, + >, +> { let mut tx_hashes_for_receipts: HashMap< - crate::ReceiptOrDataId, - crate::ParentTransactionHashString, + crate::receipts_cache::ReceiptOrDataId, + crate::receipts_cache::ParentTransactionHashString, > = HashMap::new(); let mut receipts_cache_lock = receipts_cache.lock().await; @@ -176,10 +181,12 @@ async fn find_tx_hashes_for_receipts( near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { .. } => receipts_cache_lock - .cache_get(&crate::ReceiptOrDataId::ReceiptId(receipt.receipt_id)) + .cache_get(&crate::receipts_cache::ReceiptOrDataId::ReceiptId( + receipt.receipt_id, + )) .map(|parent_transaction_hash| { ( - crate::ReceiptOrDataId::ReceiptId(receipt.receipt_id), + crate::receipts_cache::ReceiptOrDataId::ReceiptId(receipt.receipt_id), parent_transaction_hash.clone(), ) }), @@ -190,10 +197,10 @@ async fn find_tx_hashes_for_receipts( // Pair DataId:ParentTransactionHash won't be used after this moment // We want to clean it up to prevent our cache from growing receipts_cache_lock - .cache_remove(&crate::ReceiptOrDataId::DataId(data_id)) + .cache_remove(&crate::receipts_cache::ReceiptOrDataId::DataId(data_id)) .map(|parent_transaction_hash| { ( - crate::ReceiptOrDataId::DataId(data_id), + crate::receipts_cache::ReceiptOrDataId::DataId(data_id), parent_transaction_hash, ) }) @@ -208,9 +215,12 @@ async fn find_tx_hashes_for_receipts( near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Data { data_id, .. - } => !tx_hashes_for_receipts.contains_key(&crate::ReceiptOrDataId::DataId(data_id)), + } => !tx_hashes_for_receipts + .contains_key(&crate::receipts_cache::ReceiptOrDataId::DataId(data_id)), near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { .. } => { - !tx_hashes_for_receipts.contains_key(&crate::ReceiptOrDataId::ReceiptId(r.receipt_id)) + !tx_hashes_for_receipts.contains_key( + &crate::receipts_cache::ReceiptOrDataId::ReceiptId(r.receipt_id), + ) } }); if receipts.is_empty() { @@ -240,8 +250,8 @@ async fn find_tx_hashes_for_receipts( if !data_ids.is_empty() { let mut interval = crate::INTERVAL; let tx_hashes_for_data_id_via_data_output: Vec<( - crate::ReceiptOrDataId, - crate::ParentTransactionHashString, + crate::receipts_cache::ReceiptOrDataId, + crate::receipts_cache::ParentTransactionHashString, )> = loop { match schema::action_receipt_output_data::table .inner_join( @@ -267,7 +277,7 @@ async fn find_tx_hashes_for_receipts( .map( |(receipt_id_string, transaction_hash_string): (String, String)| { ( - crate::ReceiptOrDataId::DataId( + crate::receipts_cache::ReceiptOrDataId::DataId( near_lake_framework::near_indexer_primitives::CryptoHash::from_str( &receipt_id_string, ) @@ -294,13 +304,15 @@ async fn find_tx_hashes_for_receipts( } }; - let mut tx_hashes_for_data_id_via_data_output_hashmap = - HashMap::::new(); + let mut tx_hashes_for_data_id_via_data_output_hashmap = HashMap::< + crate::receipts_cache::ReceiptOrDataId, + crate::receipts_cache::ParentTransactionHashString, + >::new(); tx_hashes_for_data_id_via_data_output_hashmap .extend(tx_hashes_for_data_id_via_data_output); let tx_hashes_for_receipts_via_data_output: Vec<( - crate::ReceiptOrDataId, - crate::ParentTransactionHashString, + crate::receipts_cache::ReceiptOrDataId, + crate::receipts_cache::ParentTransactionHashString, )> = receipts .iter() .filter_map(|r| { @@ -308,10 +320,10 @@ async fn find_tx_hashes_for_receipts( near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Data { data_id, .. } => tx_hashes_for_data_id_via_data_output_hashmap - .get(&crate::ReceiptOrDataId::DataId(data_id)) + .get(&crate::receipts_cache::ReceiptOrDataId::DataId(data_id)) .map(|tx_hash| { ( - crate::ReceiptOrDataId::ReceiptId(r.receipt_id), + crate::receipts_cache::ReceiptOrDataId::ReceiptId(r.receipt_id), tx_hash.to_string(), ) }), @@ -328,12 +340,13 @@ async fn find_tx_hashes_for_receipts( } receipts.retain(|r| { - !tx_hashes_for_receipts - .contains_key(&crate::ReceiptOrDataId::ReceiptId(r.receipt_id)) + !tx_hashes_for_receipts.contains_key( + &crate::receipts_cache::ReceiptOrDataId::ReceiptId(r.receipt_id), + ) }); } - let tx_hashes_for_receipts_via_outcomes: Vec<(String, crate::ParentTransactionHashString)> = + let tx_hashes_for_receipts_via_outcomes: Vec<(String, crate::receipts_cache::ParentTransactionHashString)> = crate::await_retry_or_panic!( schema::execution_outcome_receipts::table .inner_join( @@ -360,7 +373,7 @@ async fn find_tx_hashes_for_receipts( schema::execution_outcome_receipts::dsl::produced_receipt_id, schema::receipts::dsl::originated_from_transaction_hash, )) - .load_async::<(String, crate::ParentTransactionHashString)>(pool), + .load_async::<(String, crate::receipts_cache::ParentTransactionHashString)>(pool), 10, "Parent Transaction for Receipts were fetched".to_string(), &receipts @@ -371,7 +384,7 @@ async fn find_tx_hashes_for_receipts( tx_hashes_for_receipts.extend(tx_hashes_for_receipts_via_outcomes.into_iter().map( |(receipt_id_string, transaction_hash_string)| { ( - crate::ReceiptOrDataId::ReceiptId( + crate::receipts_cache::ReceiptOrDataId::ReceiptId( near_lake_framework::near_indexer_primitives::CryptoHash::from_str( &receipt_id_string, ) @@ -387,12 +400,14 @@ async fn find_tx_hashes_for_receipts( } receipts.retain(|r| { - !tx_hashes_for_receipts.contains_key(&crate::ReceiptOrDataId::ReceiptId(r.receipt_id)) + !tx_hashes_for_receipts.contains_key( + &crate::receipts_cache::ReceiptOrDataId::ReceiptId(r.receipt_id), + ) }); let tx_hashes_for_receipt_via_transactions: Vec<( String, - crate::ParentTransactionHashString, + crate::receipts_cache::ParentTransactionHashString, )> = crate::await_retry_or_panic!( schema::transactions::table .filter( @@ -412,7 +427,7 @@ async fn find_tx_hashes_for_receipts( schema::transactions::dsl::converted_into_receipt_id, schema::transactions::dsl::transaction_hash, )) - .load_async::<(String, crate::ParentTransactionHashString)>(pool), + .load_async::<(String, crate::receipts_cache::ParentTransactionHashString)>(pool), 10, "Parent Transaction for ExecutionOutcome were fetched".to_string(), &receipts @@ -423,7 +438,7 @@ async fn find_tx_hashes_for_receipts( tx_hashes_for_receipts.extend(tx_hashes_for_receipt_via_transactions.into_iter().map( |(receipt_id_string, transaction_hash_string)| { ( - crate::ReceiptOrDataId::ReceiptId( + crate::receipts_cache::ReceiptOrDataId::ReceiptId( near_lake_framework::near_indexer_primitives::CryptoHash::from_str( &receipt_id_string, ) @@ -439,7 +454,9 @@ async fn find_tx_hashes_for_receipts( } receipts.retain(|r| { - !tx_hashes_for_receipts.contains_key(&crate::ReceiptOrDataId::ReceiptId(r.receipt_id)) + !tx_hashes_for_receipts.contains_key( + &crate::receipts_cache::ReceiptOrDataId::ReceiptId(r.receipt_id), + ) }); if !strict_mode { diff --git a/database/src/adapters/transactions.rs b/database/src/adapters/transactions.rs index 5ff76326..24b81c2d 100644 --- a/database/src/adapters/transactions.rs +++ b/database/src/adapters/transactions.rs @@ -14,7 +14,7 @@ pub async fn store_transactions( block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, block_timestamp: u64, block_height: near_lake_framework::near_indexer_primitives::types::BlockHeight, - receipts_cache: crate::ReceiptsCache, + receipts_cache: crate::receipts_cache::ReceiptsCache, ) -> anyhow::Result<()> { let mut tried_to_insert_transactions_count = 0; let tx_futures = shards @@ -106,7 +106,7 @@ async fn store_chunk_transactions( block_timestamp: u64, // hack for supporting duplicated transaction hashes. Empty for most of transactions transaction_hash_suffix: &str, - receipts_cache: crate::ReceiptsCache, + receipts_cache: crate::receipts_cache::ReceiptsCache, ) -> anyhow::Result<()> { let mut receipts_cache_lock = receipts_cache.lock().await; @@ -128,7 +128,7 @@ async fn store_chunk_transactions( // Later, while Receipt will be looking for a parent Transaction hash // it will be able to find it in the ReceiptsCache receipts_cache_lock.cache_set( - crate::ReceiptOrDataId::ReceiptId(*converted_into_receipt_id), + crate::receipts_cache::ReceiptOrDataId::ReceiptId(*converted_into_receipt_id), transaction_hash.clone(), ); diff --git a/database/src/lib.rs b/database/src/lib.rs index eff624c7..142241c2 100644 --- a/database/src/lib.rs +++ b/database/src/lib.rs @@ -5,6 +5,7 @@ pub use actix_diesel; pub mod adapters; pub mod models; +pub mod receipts_cache; mod schema; #[macro_use] diff --git a/database/src/receipts_cache.rs b/database/src/receipts_cache.rs new file mode 100644 index 00000000..523905e8 --- /dev/null +++ b/database/src/receipts_cache.rs @@ -0,0 +1,16 @@ +use cached::SizedCache; +use tokio::sync::Mutex; + +#[derive(Clone, Hash, PartialEq, Eq, Debug)] +pub enum ReceiptOrDataId { + ReceiptId(near_lake_framework::near_indexer_primitives::CryptoHash), + DataId(near_lake_framework::near_indexer_primitives::CryptoHash), +} +// Creating type aliases to make HashMap types for cache more explicit +pub type ParentTransactionHashString = String; +// Introducing a simple cache for Receipts to find their parent Transactions without +// touching the database +// The key is ReceiptID +// The value is TransactionHash (the very parent of the Receipt) +pub type ReceiptsCache = + std::sync::Arc>>; diff --git a/indexer/src/main.rs b/indexer/src/main.rs index a199e766..47fa942f 100644 --- a/indexer/src/main.rs +++ b/indexer/src/main.rs @@ -6,7 +6,7 @@ use futures::{try_join, StreamExt}; use tokio::sync::Mutex; use tracing::{debug, info}; -use explorer_database::{adapters, models}; +use explorer_database::{adapters, models, receipts_cache}; use crate::configs::Opts; @@ -16,25 +16,11 @@ mod configs; const INDEXER_FOR_EXPLORER: &str = "indexer_for_explorer"; // const AGGREGATED: &str = "aggregated"; -#[derive(Clone, Hash, PartialEq, Eq, Debug)] -pub enum ReceiptOrDataId { - ReceiptId(near_lake_framework::near_indexer_primitives::CryptoHash), - DataId(near_lake_framework::near_indexer_primitives::CryptoHash), -} -// Creating type aliases to make HashMap types for cache more explicit -pub type ParentTransactionHashString = String; -// Introducing a simple cache for Receipts to find their parent Transactions without -// touching the database -// The key is ReceiptID -// The value is TransactionHash (the very parent of the Receipt) -pub type ReceiptsCache = - std::sync::Arc>>; - async fn handle_message( pool: &explorer_database::actix_diesel::Database, streamer_message: near_lake_framework::near_indexer_primitives::StreamerMessage, strict_mode: bool, - receipts_cache: ReceiptsCache, + receipts_cache: receipts_cache::ReceiptsCache, ) -> anyhow::Result<()> { debug!( target: INDEXER_FOR_EXPLORER, @@ -164,7 +150,7 @@ async fn main() -> anyhow::Result<()> { // Later we need to find the Receipt which is a parent to underlying Receipts. // Receipt ID will of the child will be stored as key and parent Transaction hash/Receipt ID // will be stored as a value - let receipts_cache: ReceiptsCache = + let receipts_cache: receipts_cache::ReceiptsCache = std::sync::Arc::new(Mutex::new(SizedCache::with_size(100_000))); let config: near_lake_framework::LakeConfig = opts.to_lake_config().await; From d1cde8b1d016e1c07b62063ef016aa281f7308b7 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Thu, 3 Nov 2022 08:25:33 +1300 Subject: [PATCH 16/42] refactor: Replace `near-lake-framework` with `near-indexer-primitives` --- Cargo.lock | 2 +- database/Cargo.toml | 2 +- database/src/adapters/access_keys.rs | 13 +- database/src/adapters/account_changes.rs | 8 +- database/src/adapters/accounts.rs | 26 ++- database/src/adapters/assets/events.rs | 6 +- .../adapters/assets/fungible_token_events.rs | 8 +- .../assets/non_fungible_token_events.rs | 8 +- database/src/adapters/blocks.rs | 2 +- database/src/adapters/chunks.rs | 4 +- database/src/adapters/execution_outcomes.rs | 6 +- database/src/adapters/receipts.rs | 184 ++++++++---------- database/src/adapters/transactions.rs | 21 +- database/src/models/access_keys.rs | 12 +- database/src/models/account_changes.rs | 44 ++--- database/src/models/accounts.rs | 10 +- database/src/models/blocks.rs | 4 +- database/src/models/chunks.rs | 4 +- database/src/models/enums.rs | 72 +++---- database/src/models/execution_outcomes.rs | 2 +- database/src/models/receipts.rs | 26 ++- database/src/models/serializers.rs | 28 +-- database/src/models/transactions.rs | 8 +- database/src/receipts_cache.rs | 4 +- 24 files changed, 227 insertions(+), 277 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a8a96ea4..f0ab510d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1223,7 +1223,7 @@ dependencies = [ "futures", "hex", "near-crypto", - "near-lake-framework", + "near-indexer-primitives", "near-primitives", "num-traits", "serde", diff --git a/database/Cargo.toml b/database/Cargo.toml index b5ebf16a..456de9bf 100644 --- a/database/Cargo.toml +++ b/database/Cargo.toml @@ -22,6 +22,6 @@ tracing = "0.1.13" tokio = { version = "1.1", features = ["sync", "time"] } actix-diesel = { git = "https://github.com/frol/actix-diesel", rev = "3a001986c89dfabfc3c448d8bae28525101b4992" } -near-lake-framework = "0.5.2" +near-indexer-primitives = "0.14" near-primitives = "0.14" near-crypto = "0.14" diff --git a/database/src/adapters/access_keys.rs b/database/src/adapters/access_keys.rs index 3b2daf72..2427727b 100644 --- a/database/src/adapters/access_keys.rs +++ b/database/src/adapters/access_keys.rs @@ -10,8 +10,8 @@ use crate::schema; pub async fn handle_access_keys( pool: &actix_diesel::Database, - state_changes: &[near_lake_framework::near_indexer_primitives::views::StateChangeWithCauseView], - block_height: near_lake_framework::near_indexer_primitives::types::BlockHeight, + state_changes: &[near_indexer_primitives::views::StateChangeWithCauseView], + block_height: near_indexer_primitives::types::BlockHeight, ) -> anyhow::Result<()> { if state_changes.is_empty() { return Ok(()); @@ -20,11 +20,12 @@ pub async fn handle_access_keys( let mut access_keys = HashMap::<(String, String), models::access_keys::AccessKey>::new(); for state_change in state_changes { - if let near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::ReceiptProcessing { receipt_hash } = - state_change.cause + if let near_indexer_primitives::views::StateChangeCauseView::ReceiptProcessing { + receipt_hash, + } = state_change.cause { match &state_change.value { - near_lake_framework::near_indexer_primitives::views::StateChangeValueView::AccessKeyUpdate { + near_indexer_primitives::views::StateChangeValueView::AccessKeyUpdate { account_id, public_key, access_key, @@ -40,7 +41,7 @@ pub async fn handle_access_keys( ), ); } - near_lake_framework::near_indexer_primitives::views::StateChangeValueView::AccessKeyDeletion { + near_indexer_primitives::views::StateChangeValueView::AccessKeyDeletion { account_id, public_key, } => { diff --git a/database/src/adapters/account_changes.rs b/database/src/adapters/account_changes.rs index 4635ee96..cc9c0312 100644 --- a/database/src/adapters/account_changes.rs +++ b/database/src/adapters/account_changes.rs @@ -8,8 +8,8 @@ use crate::schema; /// Saves state change related to account to database pub async fn store_account_changes( pool: &actix_diesel::Database, - shards: &[near_lake_framework::near_indexer_primitives::IndexerShard], - block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, + shards: &[near_indexer_primitives::IndexerShard], + block_hash: &near_indexer_primitives::CryptoHash, block_timestamp: u64, ) -> anyhow::Result<()> { let futures = shards.iter().map(|shard| { @@ -21,8 +21,8 @@ pub async fn store_account_changes( async fn store_account_changes_for_chunk( pool: &actix_diesel::Database, - state_changes: &[near_lake_framework::near_indexer_primitives::views::StateChangeWithCauseView], - block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, + state_changes: &[near_indexer_primitives::views::StateChangeWithCauseView], + block_hash: &near_indexer_primitives::CryptoHash, block_timestamp: u64, ) -> anyhow::Result<()> { if state_changes.is_empty() { diff --git a/database/src/adapters/accounts.rs b/database/src/adapters/accounts.rs index 5d6bd22e..e894f6de 100644 --- a/database/src/adapters/accounts.rs +++ b/database/src/adapters/accounts.rs @@ -11,8 +11,8 @@ use crate::schema; /// Saves new Accounts to database or deletes the ones should be deleted pub async fn handle_accounts( pool: &actix_diesel::Database, - outcomes: &[near_lake_framework::near_indexer_primitives::IndexerExecutionOutcomeWithReceipt], - block_height: near_lake_framework::near_indexer_primitives::types::BlockHeight, + outcomes: &[near_indexer_primitives::IndexerExecutionOutcomeWithReceipt], + block_height: near_indexer_primitives::types::BlockHeight, ) -> anyhow::Result<()> { if outcomes.is_empty() { return Ok(()); @@ -22,26 +22,22 @@ pub async fn handle_accounts( .filter(|outcome_with_receipt| { matches!( outcome_with_receipt.execution_outcome.outcome.status, - near_lake_framework::near_indexer_primitives::views::ExecutionStatusView::SuccessValue(_) - | near_lake_framework::near_indexer_primitives::views::ExecutionStatusView::SuccessReceiptId(_) + near_indexer_primitives::views::ExecutionStatusView::SuccessValue(_) + | near_indexer_primitives::views::ExecutionStatusView::SuccessReceiptId(_) ) }) .map(|outcome_with_receipt| &outcome_with_receipt.receipt); - let mut accounts = HashMap::< - near_lake_framework::near_indexer_primitives::types::AccountId, - models::accounts::Account, - >::new(); + let mut accounts = + HashMap::::new(); for receipt in successful_receipts { - if let near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { - actions, - .. - } = &receipt.receipt + if let near_indexer_primitives::views::ReceiptEnumView::Action { actions, .. } = + &receipt.receipt { for action in actions { match action { - near_lake_framework::near_indexer_primitives::views::ActionView::CreateAccount => { + near_indexer_primitives::views::ActionView::CreateAccount => { accounts.insert( receipt.receiver_id.clone(), models::accounts::Account::new_from_receipt( @@ -51,7 +47,7 @@ pub async fn handle_accounts( ), ); } - near_lake_framework::near_indexer_primitives::views::ActionView::Transfer { .. } => { + near_indexer_primitives::views::ActionView::Transfer { .. } => { if receipt.receiver_id.len() == 64usize { accounts.insert( receipt.receiver_id.clone(), @@ -63,7 +59,7 @@ pub async fn handle_accounts( ); } } - near_lake_framework::near_indexer_primitives::views::ActionView::DeleteAccount { .. } => { + near_indexer_primitives::views::ActionView::DeleteAccount { .. } => { accounts .entry(receipt.receiver_id.clone()) .and_modify(|existing_account| { diff --git a/database/src/adapters/assets/events.rs b/database/src/adapters/assets/events.rs index a586f539..f1a0e6c1 100644 --- a/database/src/adapters/assets/events.rs +++ b/database/src/adapters/assets/events.rs @@ -8,7 +8,7 @@ use super::event_types; pub async fn store_events( pool: &Database, - streamer_message: &near_lake_framework::near_indexer_primitives::StreamerMessage, + streamer_message: &near_indexer_primitives::StreamerMessage, ) -> anyhow::Result<()> { let futures = streamer_message.shards.iter().map(|shard| { collect_and_store_events(pool, shard, streamer_message.block.header.timestamp) @@ -43,7 +43,7 @@ pub(crate) async fn detect_db_error( async fn collect_and_store_events( pool: &Database, - shard: &near_lake_framework::near_indexer_primitives::IndexerShard, + shard: &near_indexer_primitives::IndexerShard, block_timestamp: u64, ) -> anyhow::Result<()> { let mut ft_events_with_outcomes = Vec::new(); @@ -80,7 +80,7 @@ async fn collect_and_store_events( } fn extract_events( - outcome: &near_lake_framework::near_indexer_primitives::IndexerExecutionOutcomeWithReceipt, + outcome: &near_indexer_primitives::IndexerExecutionOutcomeWithReceipt, ) -> Vec { let prefix = "EVENT_JSON:"; outcome.execution_outcome.outcome.logs.iter().filter_map(|untrimmed_log| { diff --git a/database/src/adapters/assets/fungible_token_events.rs b/database/src/adapters/assets/fungible_token_events.rs index b1dc170d..9ad99dba 100644 --- a/database/src/adapters/assets/fungible_token_events.rs +++ b/database/src/adapters/assets/fungible_token_events.rs @@ -11,11 +11,11 @@ use super::event_types; pub(crate) async fn store_ft_events( pool: &Database, - shard: &near_lake_framework::near_indexer_primitives::IndexerShard, + shard: &near_indexer_primitives::IndexerShard, block_timestamp: u64, events_with_outcomes: &[( assets::event_types::Nep141Event, - &near_lake_framework::near_indexer_primitives::IndexerExecutionOutcomeWithReceipt, + &near_indexer_primitives::IndexerExecutionOutcomeWithReceipt, )], ) -> anyhow::Result<()> { let ft_events = compose_ft_db_events(events_with_outcomes, block_timestamp, &shard.shard_id); @@ -48,10 +48,10 @@ async fn detect_ft_db_error(async_error: &AsyncError) -> fn compose_ft_db_events( events_with_outcomes: &[( assets::event_types::Nep141Event, - &near_lake_framework::near_indexer_primitives::IndexerExecutionOutcomeWithReceipt, + &near_indexer_primitives::IndexerExecutionOutcomeWithReceipt, )], block_timestamp: u64, - shard_id: &near_lake_framework::near_indexer_primitives::types::ShardId, + shard_id: &near_indexer_primitives::types::ShardId, ) -> Vec { let mut ft_events = Vec::new(); for (event, outcome) in events_with_outcomes { diff --git a/database/src/adapters/assets/non_fungible_token_events.rs b/database/src/adapters/assets/non_fungible_token_events.rs index cc77fae6..652fbeec 100644 --- a/database/src/adapters/assets/non_fungible_token_events.rs +++ b/database/src/adapters/assets/non_fungible_token_events.rs @@ -11,11 +11,11 @@ use super::event_types; pub(crate) async fn store_nft_events( pool: &Database, - shard: &near_lake_framework::near_indexer_primitives::IndexerShard, + shard: &near_indexer_primitives::IndexerShard, block_timestamp: u64, events_with_outcomes: &[( assets::event_types::Nep171Event, - &near_lake_framework::near_indexer_primitives::IndexerExecutionOutcomeWithReceipt, + &near_indexer_primitives::IndexerExecutionOutcomeWithReceipt, )], ) -> anyhow::Result<()> { let nft_events = compose_nft_db_events(events_with_outcomes, block_timestamp, &shard.shard_id); @@ -48,10 +48,10 @@ async fn detect_nft_db_error(async_error: &AsyncError) -> fn compose_nft_db_events( events_with_outcomes: &[( assets::event_types::Nep171Event, - &near_lake_framework::near_indexer_primitives::IndexerExecutionOutcomeWithReceipt, + &near_indexer_primitives::IndexerExecutionOutcomeWithReceipt, )], block_timestamp: u64, - shard_id: &near_lake_framework::near_indexer_primitives::types::ShardId, + shard_id: &near_indexer_primitives::types::ShardId, ) -> Vec { let mut nft_events = Vec::new(); for (event, outcome) in events_with_outcomes { diff --git a/database/src/adapters/blocks.rs b/database/src/adapters/blocks.rs index fa5833e1..9558e76c 100644 --- a/database/src/adapters/blocks.rs +++ b/database/src/adapters/blocks.rs @@ -8,7 +8,7 @@ use crate::schema; /// Saves block to database pub async fn store_block( pool: &actix_diesel::Database, - block: &near_lake_framework::near_indexer_primitives::views::BlockView, + block: &near_indexer_primitives::views::BlockView, ) -> anyhow::Result<()> { let block_model = models::blocks::Block::from(block); diff --git a/database/src/adapters/chunks.rs b/database/src/adapters/chunks.rs index 53f4829d..f1cdb7c9 100644 --- a/database/src/adapters/chunks.rs +++ b/database/src/adapters/chunks.rs @@ -7,8 +7,8 @@ use crate::schema; /// Saves chunks to database pub async fn store_chunks( pool: &actix_diesel::Database, - shards: &[near_lake_framework::near_indexer_primitives::IndexerShard], - block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, + shards: &[near_indexer_primitives::IndexerShard], + block_hash: &near_indexer_primitives::CryptoHash, ) -> anyhow::Result<()> { if shards.is_empty() { return Ok(()); diff --git a/database/src/adapters/execution_outcomes.rs b/database/src/adapters/execution_outcomes.rs index d7a3aeda..084d4793 100644 --- a/database/src/adapters/execution_outcomes.rs +++ b/database/src/adapters/execution_outcomes.rs @@ -8,7 +8,7 @@ use crate::schema; pub async fn store_execution_outcomes( pool: &actix_diesel::Database, - shards: &[near_lake_framework::near_indexer_primitives::IndexerShard], + shards: &[near_indexer_primitives::IndexerShard], block_timestamp: u64, receipts_cache: crate::receipts_cache::ReceiptsCache, ) -> anyhow::Result<()> { @@ -28,8 +28,8 @@ pub async fn store_execution_outcomes( /// Saves ExecutionOutcome to database and then saves ExecutionOutcomesReceipts pub async fn store_execution_outcomes_for_chunk( pool: &actix_diesel::Database, - execution_outcomes: &[near_lake_framework::near_indexer_primitives::IndexerExecutionOutcomeWithReceipt], - shard_id: near_lake_framework::near_indexer_primitives::types::ShardId, + execution_outcomes: &[near_indexer_primitives::IndexerExecutionOutcomeWithReceipt], + shard_id: near_indexer_primitives::types::ShardId, block_timestamp: u64, receipts_cache: crate::receipts_cache::ReceiptsCache, ) -> anyhow::Result<()> { diff --git a/database/src/adapters/receipts.rs b/database/src/adapters/receipts.rs index cc6d8c42..3bcef678 100644 --- a/database/src/adapters/receipts.rs +++ b/database/src/adapters/receipts.rs @@ -17,8 +17,8 @@ use crate::schema; /// Saves receipts to database pub async fn store_receipts( pool: &actix_diesel::Database, - shards: &[near_lake_framework::near_indexer_primitives::IndexerShard], - block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, + shards: &[near_indexer_primitives::IndexerShard], + block_hash: &near_indexer_primitives::CryptoHash, block_timestamp: u64, strict_mode: bool, receipts_cache: crate::receipts_cache::ReceiptsCache, @@ -44,16 +44,15 @@ pub async fn store_receipts( async fn store_chunk_receipts( pool: &actix_diesel::Database, - receipts: &[near_lake_framework::near_indexer_primitives::views::ReceiptView], - block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, - chunk_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, + receipts: &[near_indexer_primitives::views::ReceiptView], + block_hash: &near_indexer_primitives::CryptoHash, + chunk_hash: &near_indexer_primitives::CryptoHash, block_timestamp: u64, strict_mode: bool, receipts_cache: crate::receipts_cache::ReceiptsCache, ) -> anyhow::Result<()> { - let mut skipping_receipt_ids = std::collections::HashSet::< - near_lake_framework::near_indexer_primitives::CryptoHash, - >::new(); + let mut skipping_receipt_ids = + std::collections::HashSet::::new(); let tx_hashes_for_receipts = find_tx_hashes_for_receipts( pool, @@ -74,10 +73,10 @@ async fn store_chunk_receipts( // In case of Action Receipt we are looking for ReceiptId // In case of Data Receipt we are looking for DataId let receipt_or_data_id = match r.receipt { - near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { .. } => { + near_indexer_primitives::views::ReceiptEnumView::Action { .. } => { crate::receipts_cache::ReceiptOrDataId::ReceiptId(r.receipt_id) } - near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Data { data_id, .. } => { + near_indexer_primitives::views::ReceiptEnumView::Data { data_id, .. } => { crate::receipts_cache::ReceiptOrDataId::DataId(data_id) } }; @@ -109,7 +108,7 @@ async fn store_chunk_receipts( // to ReceiptsCache let mut receipts_cache_lock = receipts_cache.lock().await; for receipt in receipts { - if let near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { + if let near_indexer_primitives::views::ReceiptEnumView::Action { output_data_receivers, .. } = &receipt.receipt @@ -134,15 +133,15 @@ async fn store_chunk_receipts( save_receipts(pool, receipt_models).await?; let (action_receipts, data_receipts): ( - Vec<&near_lake_framework::near_indexer_primitives::views::ReceiptView>, - Vec<&near_lake_framework::near_indexer_primitives::views::ReceiptView>, + Vec<&near_indexer_primitives::views::ReceiptView>, + Vec<&near_indexer_primitives::views::ReceiptView>, ) = receipts .iter() .filter(|r| !skipping_receipt_ids.contains(&r.receipt_id)) .partition(|receipt| { matches!( receipt.receipt, - near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { .. } + near_indexer_primitives::views::ReceiptEnumView::Action { .. } ) }); @@ -158,10 +157,10 @@ async fn store_chunk_receipts( /// Looks for already created parent transaction hash for given receipts async fn find_tx_hashes_for_receipts( pool: &actix_diesel::Database, - mut receipts: Vec, + mut receipts: Vec, strict_mode: bool, - block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, - chunk_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, + block_hash: &near_indexer_primitives::CryptoHash, + chunk_hash: &near_indexer_primitives::CryptoHash, receipts_cache: crate::receipts_cache::ReceiptsCache, ) -> anyhow::Result< HashMap< @@ -178,9 +177,7 @@ async fn find_tx_hashes_for_receipts( // add receipt-transaction pairs from the cache to the response tx_hashes_for_receipts.extend(receipts.iter().filter_map(|receipt| { match receipt.receipt { - near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { - .. - } => receipts_cache_lock + near_indexer_primitives::views::ReceiptEnumView::Action { .. } => receipts_cache_lock .cache_get(&crate::receipts_cache::ReceiptOrDataId::ReceiptId( receipt.receipt_id, )) @@ -190,10 +187,7 @@ async fn find_tx_hashes_for_receipts( parent_transaction_hash.clone(), ) }), - near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Data { - data_id, - .. - } => { + near_indexer_primitives::views::ReceiptEnumView::Data { data_id, .. } => { // Pair DataId:ParentTransactionHash won't be used after this moment // We want to clean it up to prevent our cache from growing receipts_cache_lock @@ -212,16 +206,14 @@ async fn find_tx_hashes_for_receipts( // discard the Receipts already in cache from the attempts to search receipts.retain(|r| match r.receipt { - near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Data { - data_id, - .. - } => !tx_hashes_for_receipts - .contains_key(&crate::receipts_cache::ReceiptOrDataId::DataId(data_id)), - near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { .. } => { - !tx_hashes_for_receipts.contains_key( - &crate::receipts_cache::ReceiptOrDataId::ReceiptId(r.receipt_id), - ) + near_indexer_primitives::views::ReceiptEnumView::Data { data_id, .. } => { + !tx_hashes_for_receipts + .contains_key(&crate::receipts_cache::ReceiptOrDataId::DataId(data_id)) } + near_indexer_primitives::views::ReceiptEnumView::Action { .. } => !tx_hashes_for_receipts + .contains_key(&crate::receipts_cache::ReceiptOrDataId::ReceiptId( + r.receipt_id, + )), }); if receipts.is_empty() { return Ok(tx_hashes_for_receipts); @@ -240,10 +232,9 @@ async fn find_tx_hashes_for_receipts( let data_ids: Vec = receipts .iter() .filter_map(|r| match r.receipt { - near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Data { - data_id, - .. - } => Some(data_id.to_string()), + near_indexer_primitives::views::ReceiptEnumView::Data { data_id, .. } => { + Some(data_id.to_string()) + } _ => None, }) .collect(); @@ -278,7 +269,7 @@ async fn find_tx_hashes_for_receipts( |(receipt_id_string, transaction_hash_string): (String, String)| { ( crate::receipts_cache::ReceiptOrDataId::DataId( - near_lake_framework::near_indexer_primitives::CryptoHash::from_str( + near_indexer_primitives::CryptoHash::from_str( &receipt_id_string, ) .expect("Failed to convert String to CryptoHash"), @@ -315,20 +306,18 @@ async fn find_tx_hashes_for_receipts( crate::receipts_cache::ParentTransactionHashString, )> = receipts .iter() - .filter_map(|r| { - match r.receipt { - near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Data { - data_id, .. - } => tx_hashes_for_data_id_via_data_output_hashmap - .get(&crate::receipts_cache::ReceiptOrDataId::DataId(data_id)) - .map(|tx_hash| { - ( - crate::receipts_cache::ReceiptOrDataId::ReceiptId(r.receipt_id), - tx_hash.to_string(), - ) - }), + .filter_map(|r| match r.receipt { + near_indexer_primitives::views::ReceiptEnumView::Data { data_id, .. } => { + tx_hashes_for_data_id_via_data_output_hashmap + .get(&crate::receipts_cache::ReceiptOrDataId::DataId(data_id)) + .map(|tx_hash| { + ( + crate::receipts_cache::ReceiptOrDataId::ReceiptId(r.receipt_id), + tx_hash.to_string(), + ) + }) + } _ => None, - } }) .collect(); @@ -346,49 +335,47 @@ async fn find_tx_hashes_for_receipts( }); } - let tx_hashes_for_receipts_via_outcomes: Vec<(String, crate::receipts_cache::ParentTransactionHashString)> = - crate::await_retry_or_panic!( - schema::execution_outcome_receipts::table - .inner_join( - schema::receipts::table - .on(schema::execution_outcome_receipts::dsl::executed_receipt_id - .eq(schema::receipts::dsl::receipt_id)), - ) - .filter( - schema::execution_outcome_receipts::dsl::produced_receipt_id.eq(any( - receipts - .clone() - .iter() - .filter(|r| { - matches!( + let tx_hashes_for_receipts_via_outcomes: Vec<( + String, + crate::receipts_cache::ParentTransactionHashString, + )> = crate::await_retry_or_panic!( + schema::execution_outcome_receipts::table + .inner_join( + schema::receipts::table + .on(schema::execution_outcome_receipts::dsl::executed_receipt_id + .eq(schema::receipts::dsl::receipt_id)), + ) + .filter( + schema::execution_outcome_receipts::dsl::produced_receipt_id.eq(any(receipts + .clone() + .iter() + .filter(|r| { + matches!( r.receipt, - near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { .. } + near_indexer_primitives::views::ReceiptEnumView::Action { .. } ) - }) - .map(|r| r.receipt_id.to_string()) - .collect::>() - )), - ) - .select(( - schema::execution_outcome_receipts::dsl::produced_receipt_id, - schema::receipts::dsl::originated_from_transaction_hash, - )) - .load_async::<(String, crate::receipts_cache::ParentTransactionHashString)>(pool), - 10, - "Parent Transaction for Receipts were fetched".to_string(), - &receipts - ) - .unwrap_or_default(); + }) + .map(|r| r.receipt_id.to_string()) + .collect::>())), + ) + .select(( + schema::execution_outcome_receipts::dsl::produced_receipt_id, + schema::receipts::dsl::originated_from_transaction_hash, + )) + .load_async::<(String, crate::receipts_cache::ParentTransactionHashString)>(pool), + 10, + "Parent Transaction for Receipts were fetched".to_string(), + &receipts + ) + .unwrap_or_default(); let found_hashes_len = tx_hashes_for_receipts_via_outcomes.len(); tx_hashes_for_receipts.extend(tx_hashes_for_receipts_via_outcomes.into_iter().map( |(receipt_id_string, transaction_hash_string)| { ( crate::receipts_cache::ReceiptOrDataId::ReceiptId( - near_lake_framework::near_indexer_primitives::CryptoHash::from_str( - &receipt_id_string, - ) - .expect("Failed to convert String to CryptoHash"), + near_indexer_primitives::CryptoHash::from_str(&receipt_id_string) + .expect("Failed to convert String to CryptoHash"), ), transaction_hash_string, ) @@ -417,7 +404,7 @@ async fn find_tx_hashes_for_receipts( .filter(|r| { matches!( r.receipt, - near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { .. } + near_indexer_primitives::views::ReceiptEnumView::Action { .. } ) }) .map(|r| r.receipt_id.to_string()) @@ -439,10 +426,8 @@ async fn find_tx_hashes_for_receipts( |(receipt_id_string, transaction_hash_string)| { ( crate::receipts_cache::ReceiptOrDataId::ReceiptId( - near_lake_framework::near_indexer_primitives::CryptoHash::from_str( - &receipt_id_string, - ) - .expect("Failed to convert String to CryptoHash"), + near_indexer_primitives::CryptoHash::from_str(&receipt_id_string) + .expect("Failed to convert String to CryptoHash"), ), transaction_hash_string, ) @@ -501,7 +486,7 @@ async fn save_receipts( async fn store_receipt_actions( pool: &actix_diesel::Database, - receipts: Vec<&near_lake_framework::near_indexer_primitives::views::ReceiptView>, + receipts: Vec<&near_indexer_primitives::views::ReceiptView>, block_timestamp: u64, ) -> anyhow::Result<()> { let receipt_actions: Vec = receipts @@ -512,10 +497,8 @@ async fn store_receipt_actions( let receipt_action_actions: Vec = receipts .iter() .filter_map(|receipt| { - if let near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { - actions, - .. - } = &receipt.receipt + if let near_indexer_primitives::views::ReceiptEnumView::Action { actions, .. } = + &receipt.receipt { Some(actions.iter().enumerate().map(move |(index, action)| { models::ActionReceiptAction::from_action_view( @@ -537,9 +520,8 @@ async fn store_receipt_actions( let receipt_action_input_data: Vec = receipts .iter() .filter_map(|receipt| { - if let near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { - input_data_ids, - .. + if let near_indexer_primitives::views::ReceiptEnumView::Action { + input_data_ids, .. } = &receipt.receipt { Some(input_data_ids.iter().map(move |data_id| { @@ -558,7 +540,7 @@ async fn store_receipt_actions( let receipt_action_output_data: Vec = receipts .iter() .filter_map(|receipt| { - if let near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { + if let near_indexer_primitives::views::ReceiptEnumView::Action { output_data_receivers, .. } = &receipt.receipt @@ -621,7 +603,7 @@ async fn store_receipt_actions( async fn store_receipt_data( pool: &actix_diesel::Database, - receipts: Vec<&near_lake_framework::near_indexer_primitives::views::ReceiptView>, + receipts: Vec<&near_indexer_primitives::views::ReceiptView>, ) -> anyhow::Result<()> { let receipt_data_models: Vec = receipts .iter() diff --git a/database/src/adapters/transactions.rs b/database/src/adapters/transactions.rs index 24b81c2d..f3f545f6 100644 --- a/database/src/adapters/transactions.rs +++ b/database/src/adapters/transactions.rs @@ -10,10 +10,10 @@ use crate::schema; /// Saves Transactions to database pub async fn store_transactions( pool: &actix_diesel::Database, - shards: &[near_lake_framework::near_indexer_primitives::IndexerShard], - block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, + shards: &[near_indexer_primitives::IndexerShard], + block_hash: &near_indexer_primitives::CryptoHash, block_timestamp: u64, - block_height: near_lake_framework::near_indexer_primitives::types::BlockHeight, + block_height: near_indexer_primitives::types::BlockHeight, receipts_cache: crate::receipts_cache::ReceiptsCache, ) -> anyhow::Result<()> { let mut tried_to_insert_transactions_count = 0; @@ -26,7 +26,7 @@ pub async fn store_transactions( pool, chunk.transactions.iter().enumerate().collect::>(), &chunk.header.chunk_hash, block_hash, @@ -71,7 +71,10 @@ pub async fn store_transactions( .to_string(); !inserted_receipt_ids.contains(converted_into_receipt_id) }) - .collect::>(), + .collect::>(), &chunk.header.chunk_hash, block_hash, block_timestamp, @@ -85,7 +88,7 @@ pub async fn store_transactions( async fn collect_converted_to_receipt_ids( pool: &actix_diesel::Database, - block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, + block_hash: &near_indexer_primitives::CryptoHash, ) -> anyhow::Result> { schema::transactions::table .select(schema::transactions::dsl::converted_into_receipt_id) @@ -99,10 +102,10 @@ async fn store_chunk_transactions( pool: &actix_diesel::Database, transactions: Vec<( usize, - &near_lake_framework::near_indexer_primitives::IndexerTransactionWithOutcome, + &near_indexer_primitives::IndexerTransactionWithOutcome, )>, - chunk_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, - block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, + chunk_hash: &near_indexer_primitives::CryptoHash, + block_hash: &near_indexer_primitives::CryptoHash, block_timestamp: u64, // hack for supporting duplicated transaction hashes. Empty for most of transactions transaction_hash_suffix: &str, diff --git a/database/src/models/access_keys.rs b/database/src/models/access_keys.rs index a6bf8ab6..e2216004 100644 --- a/database/src/models/access_keys.rs +++ b/database/src/models/access_keys.rs @@ -17,10 +17,10 @@ pub struct AccessKey { impl AccessKey { pub fn from_action_view( public_key: &near_crypto::PublicKey, - account_id: &near_lake_framework::near_indexer_primitives::types::AccountId, - access_key: &near_lake_framework::near_indexer_primitives::views::AccessKeyView, - create_by_receipt_id: &near_lake_framework::near_indexer_primitives::CryptoHash, - last_update_block_height: near_lake_framework::near_indexer_primitives::types::BlockHeight, + account_id: &near_indexer_primitives::types::AccountId, + access_key: &near_indexer_primitives::views::AccessKeyView, + create_by_receipt_id: &near_indexer_primitives::CryptoHash, + last_update_block_height: near_indexer_primitives::types::BlockHeight, ) -> Self { Self { public_key: public_key.to_string(), @@ -34,9 +34,9 @@ impl AccessKey { pub fn from_genesis( public_key: &near_crypto::PublicKey, - account_id: &near_lake_framework::near_indexer_primitives::types::AccountId, + account_id: &near_indexer_primitives::types::AccountId, access_key: &near_primitives::account::AccessKey, - last_update_block_height: near_lake_framework::near_indexer_primitives::types::BlockHeight, + last_update_block_height: near_indexer_primitives::types::BlockHeight, ) -> Self { Self { public_key: public_key.to_string(), diff --git a/database/src/models/account_changes.rs b/database/src/models/account_changes.rs index 5ca69f78..affa360a 100644 --- a/database/src/models/account_changes.rs +++ b/database/src/models/account_changes.rs @@ -22,44 +22,40 @@ pub struct AccountChange { impl AccountChange { pub fn from_state_change_with_cause( - state_change_with_cause: &near_lake_framework::near_indexer_primitives::views::StateChangeWithCauseView, - changed_in_block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, + state_change_with_cause: &near_indexer_primitives::views::StateChangeWithCauseView, + changed_in_block_hash: &near_indexer_primitives::CryptoHash, changed_in_block_timestamp: u64, index_in_block: i32, ) -> Option { - let near_lake_framework::near_indexer_primitives::views::StateChangeWithCauseView { - cause, - value, - } = state_change_with_cause; + let near_indexer_primitives::views::StateChangeWithCauseView { cause, value } = + state_change_with_cause; - let (account_id, account): ( - String, - Option<&near_lake_framework::near_indexer_primitives::views::AccountView>, - ) = match value { - near_lake_framework::near_indexer_primitives::views::StateChangeValueView::AccountUpdate { - account_id, - account, - } => (account_id.to_string(), Some(account)), - near_lake_framework::near_indexer_primitives::views::StateChangeValueView::AccountDeletion { - account_id, - } => (account_id.to_string(), None), - _ => return None, - }; + let (account_id, account): (String, Option<&near_indexer_primitives::views::AccountView>) = + match value { + near_indexer_primitives::views::StateChangeValueView::AccountUpdate { + account_id, + account, + } => (account_id.to_string(), Some(account)), + near_indexer_primitives::views::StateChangeValueView::AccountDeletion { + account_id, + } => (account_id.to_string(), None), + _ => return None, + }; Some(Self { affected_account_id: account_id, changed_in_block_timestamp: changed_in_block_timestamp.into(), changed_in_block_hash: changed_in_block_hash.to_string(), - caused_by_transaction_hash: if let near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::TransactionProcessing {tx_hash } = cause { + caused_by_transaction_hash: if let near_indexer_primitives::views::StateChangeCauseView::TransactionProcessing {tx_hash } = cause { Some(tx_hash.to_string()) } else { None }, caused_by_receipt_id: match cause { - near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::ActionReceiptProcessingStarted { receipt_hash} => Some(receipt_hash.to_string()), - near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::ActionReceiptGasReward { receipt_hash } => Some(receipt_hash.to_string()), - near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::ReceiptProcessing { receipt_hash } => Some(receipt_hash.to_string()), - near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::PostponedReceipt { receipt_hash } => Some(receipt_hash.to_string()), + near_indexer_primitives::views::StateChangeCauseView::ActionReceiptProcessingStarted { receipt_hash} => Some(receipt_hash.to_string()), + near_indexer_primitives::views::StateChangeCauseView::ActionReceiptGasReward { receipt_hash } => Some(receipt_hash.to_string()), + near_indexer_primitives::views::StateChangeCauseView::ReceiptProcessing { receipt_hash } => Some(receipt_hash.to_string()), + near_indexer_primitives::views::StateChangeCauseView::PostponedReceipt { receipt_hash } => Some(receipt_hash.to_string()), _ => None, }, update_reason: cause.into(), diff --git a/database/src/models/accounts.rs b/database/src/models/accounts.rs index 6a75aff0..9f5c1379 100644 --- a/database/src/models/accounts.rs +++ b/database/src/models/accounts.rs @@ -14,9 +14,9 @@ pub struct Account { impl Account { pub fn new_from_receipt( - account_id: &near_lake_framework::near_indexer_primitives::types::AccountId, - created_by_receipt_id: &near_lake_framework::near_indexer_primitives::CryptoHash, - last_update_block_height: near_lake_framework::near_indexer_primitives::types::BlockHeight, + account_id: &near_indexer_primitives::types::AccountId, + created_by_receipt_id: &near_indexer_primitives::CryptoHash, + last_update_block_height: near_indexer_primitives::types::BlockHeight, ) -> Self { Self { account_id: account_id.to_string(), @@ -27,8 +27,8 @@ impl Account { } pub fn new_from_genesis( - account_id: &near_lake_framework::near_indexer_primitives::types::AccountId, - last_update_block_height: near_lake_framework::near_indexer_primitives::types::BlockHeight, + account_id: &near_indexer_primitives::types::AccountId, + last_update_block_height: near_indexer_primitives::types::BlockHeight, ) -> Self { Self { account_id: account_id.to_string(), diff --git a/database/src/models/blocks.rs b/database/src/models/blocks.rs index 68ff55dc..97c7ec9d 100644 --- a/database/src/models/blocks.rs +++ b/database/src/models/blocks.rs @@ -16,8 +16,8 @@ pub struct Block { pub author_account_id: String, } -impl From<&near_lake_framework::near_indexer_primitives::views::BlockView> for Block { - fn from(block_view: &near_lake_framework::near_indexer_primitives::views::BlockView) -> Self { +impl From<&near_indexer_primitives::views::BlockView> for Block { + fn from(block_view: &near_indexer_primitives::views::BlockView) -> Self { Self { block_height: block_view.header.height.into(), block_hash: block_view.header.hash.to_string(), diff --git a/database/src/models/chunks.rs b/database/src/models/chunks.rs index c65ba835..de82715e 100644 --- a/database/src/models/chunks.rs +++ b/database/src/models/chunks.rs @@ -16,8 +16,8 @@ pub struct Chunk { impl Chunk { pub fn from_chunk_view( - chunk_view: &near_lake_framework::near_indexer_primitives::IndexerChunkView, - block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, + chunk_view: &near_indexer_primitives::IndexerChunkView, + block_hash: &near_indexer_primitives::CryptoHash, ) -> Self { Self { included_in_block_hash: block_hash.to_string(), diff --git a/database/src/models/enums.rs b/database/src/models/enums.rs index eb849281..3e4f6ef7 100644 --- a/database/src/models/enums.rs +++ b/database/src/models/enums.rs @@ -9,17 +9,11 @@ pub enum ReceiptKind { Data, } -impl From<&near_lake_framework::near_indexer_primitives::views::ReceiptEnumView> for ReceiptKind { - fn from( - receipt_enum_view: &near_lake_framework::near_indexer_primitives::views::ReceiptEnumView, - ) -> Self { +impl From<&near_indexer_primitives::views::ReceiptEnumView> for ReceiptKind { + fn from(receipt_enum_view: &near_indexer_primitives::views::ReceiptEnumView) -> Self { match receipt_enum_view { - near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { - .. - } => Self::Action, - near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Data { - .. - } => Self::Data, + near_indexer_primitives::views::ReceiptEnumView::Action { .. } => Self::Action, + near_indexer_primitives::views::ReceiptEnumView::Data { .. } => Self::Data, } } } @@ -50,19 +44,15 @@ pub enum ExecutionOutcomeStatus { SuccessReceiptId, } -impl From - for ExecutionOutcomeStatus -{ - fn from( - status: near_lake_framework::near_indexer_primitives::views::ExecutionStatusView, - ) -> Self { +impl From for ExecutionOutcomeStatus { + fn from(status: near_indexer_primitives::views::ExecutionStatusView) -> Self { match status { - near_lake_framework::near_indexer_primitives::views::ExecutionStatusView::Unknown => Self::Unknown, - near_lake_framework::near_indexer_primitives::views::ExecutionStatusView::Failure(_) => Self::Failure, - near_lake_framework::near_indexer_primitives::views::ExecutionStatusView::SuccessValue(_) => { + near_indexer_primitives::views::ExecutionStatusView::Unknown => Self::Unknown, + near_indexer_primitives::views::ExecutionStatusView::Failure(_) => Self::Failure, + near_indexer_primitives::views::ExecutionStatusView::SuccessValue(_) => { Self::SuccessValue } - near_lake_framework::near_indexer_primitives::views::ExecutionStatusView::SuccessReceiptId(_) => { + near_indexer_primitives::views::ExecutionStatusView::SuccessReceiptId(_) => { Self::SuccessReceiptId } } @@ -80,19 +70,13 @@ pub enum AccessKeyPermission { FunctionCall, } -impl From<&near_lake_framework::near_indexer_primitives::views::AccessKeyPermissionView> - for AccessKeyPermission -{ - fn from( - item: &near_lake_framework::near_indexer_primitives::views::AccessKeyPermissionView, - ) -> Self { +impl From<&near_indexer_primitives::views::AccessKeyPermissionView> for AccessKeyPermission { + fn from(item: &near_indexer_primitives::views::AccessKeyPermissionView) -> Self { match item { - near_lake_framework::near_indexer_primitives::views::AccessKeyPermissionView::FunctionCall { - .. - } => Self::FunctionCall, - near_lake_framework::near_indexer_primitives::views::AccessKeyPermissionView::FullAccess => { - Self::FullAccess + near_indexer_primitives::views::AccessKeyPermissionView::FunctionCall { .. } => { + Self::FunctionCall } + near_indexer_primitives::views::AccessKeyPermissionView::FullAccess => Self::FullAccess, } } } @@ -124,23 +108,21 @@ pub enum StateChangeReasonKind { Resharding, } -impl From<&near_lake_framework::near_indexer_primitives::views::StateChangeCauseView> - for StateChangeReasonKind -{ +impl From<&near_indexer_primitives::views::StateChangeCauseView> for StateChangeReasonKind { fn from( - state_change_cause_view: &near_lake_framework::near_indexer_primitives::views::StateChangeCauseView, + state_change_cause_view: &near_indexer_primitives::views::StateChangeCauseView, ) -> Self { match state_change_cause_view { - near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::TransactionProcessing { .. } => Self::TransactionProcessing, - near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::ActionReceiptProcessingStarted { .. } => Self::ActionReceiptProcessingStarted, - near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::ActionReceiptGasReward { .. } => Self::ActionReceiptGasReward, - near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::ReceiptProcessing { .. } => Self::ReceiptProcessing, - near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::PostponedReceipt { .. } => Self::PostponedReceipt, - near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::UpdatedDelayedReceipts { .. } => Self::UpdatedDelayedReceipts, - near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::ValidatorAccountsUpdate { .. } => Self::ValidatorAccountsUpdate, - near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::Migration { .. } => Self::Migration, - near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::Resharding { .. } => Self::Resharding, - near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::NotWritableToDisk | near_lake_framework::near_indexer_primitives::views::StateChangeCauseView::InitialState => panic!("Unexpected variant {:?} received", state_change_cause_view), + near_indexer_primitives::views::StateChangeCauseView::TransactionProcessing { .. } => Self::TransactionProcessing, + near_indexer_primitives::views::StateChangeCauseView::ActionReceiptProcessingStarted { .. } => Self::ActionReceiptProcessingStarted, + near_indexer_primitives::views::StateChangeCauseView::ActionReceiptGasReward { .. } => Self::ActionReceiptGasReward, + near_indexer_primitives::views::StateChangeCauseView::ReceiptProcessing { .. } => Self::ReceiptProcessing, + near_indexer_primitives::views::StateChangeCauseView::PostponedReceipt { .. } => Self::PostponedReceipt, + near_indexer_primitives::views::StateChangeCauseView::UpdatedDelayedReceipts { .. } => Self::UpdatedDelayedReceipts, + near_indexer_primitives::views::StateChangeCauseView::ValidatorAccountsUpdate { .. } => Self::ValidatorAccountsUpdate, + near_indexer_primitives::views::StateChangeCauseView::Migration { .. } => Self::Migration, + near_indexer_primitives::views::StateChangeCauseView::Resharding { .. } => Self::Resharding, + near_indexer_primitives::views::StateChangeCauseView::NotWritableToDisk | near_indexer_primitives::views::StateChangeCauseView::InitialState => panic!("Unexpected variant {:?} received", state_change_cause_view), } } } diff --git a/database/src/models/execution_outcomes.rs b/database/src/models/execution_outcomes.rs index 5e1599a3..15ac5918 100644 --- a/database/src/models/execution_outcomes.rs +++ b/database/src/models/execution_outcomes.rs @@ -22,7 +22,7 @@ pub struct ExecutionOutcome { impl ExecutionOutcome { pub fn from_execution_outcome( - execution_outcome: &near_lake_framework::near_indexer_primitives::views::ExecutionOutcomeWithIdView, + execution_outcome: &near_indexer_primitives::views::ExecutionOutcomeWithIdView, index_in_chunk: i32, executed_in_block_timestamp: u64, shard_id: u64, diff --git a/database/src/models/receipts.rs b/database/src/models/receipts.rs index 2eb5722e..e486e74e 100644 --- a/database/src/models/receipts.rs +++ b/database/src/models/receipts.rs @@ -3,7 +3,7 @@ use std::str::FromStr; use bigdecimal::BigDecimal; -use near_lake_framework::near_indexer_primitives::views::DataReceiverView; +use near_indexer_primitives::views::DataReceiverView; use crate::models::enums::{ActionKind, ReceiptKind}; use crate::schema; @@ -27,10 +27,10 @@ pub struct Receipt { impl Receipt { pub fn from_receipt_view( - receipt: &near_lake_framework::near_indexer_primitives::views::ReceiptView, - block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, + receipt: &near_indexer_primitives::views::ReceiptView, + block_hash: &near_indexer_primitives::CryptoHash, transaction_hash: &str, - chunk_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, + chunk_hash: &near_indexer_primitives::CryptoHash, index_in_chunk: i32, block_timestamp: u64, ) -> Self { @@ -56,16 +56,14 @@ pub struct DataReceipt { pub data: Option>, } -impl TryFrom<&near_lake_framework::near_indexer_primitives::views::ReceiptView> for DataReceipt { +impl TryFrom<&near_indexer_primitives::views::ReceiptView> for DataReceipt { type Error = &'static str; fn try_from( - receipt_view: &near_lake_framework::near_indexer_primitives::views::ReceiptView, + receipt_view: &near_indexer_primitives::views::ReceiptView, ) -> Result { - if let near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Data { - data_id, - data, - } = &receipt_view.receipt + if let near_indexer_primitives::views::ReceiptEnumView::Data { data_id, data } = + &receipt_view.receipt { Ok(Self { receipt_id: receipt_view.receipt_id.to_string(), @@ -87,13 +85,13 @@ pub struct ActionReceipt { pub gas_price: BigDecimal, } -impl TryFrom<&near_lake_framework::near_indexer_primitives::views::ReceiptView> for ActionReceipt { +impl TryFrom<&near_indexer_primitives::views::ReceiptView> for ActionReceipt { type Error = &'static str; fn try_from( - receipt_view: &near_lake_framework::near_indexer_primitives::views::ReceiptView, + receipt_view: &near_indexer_primitives::views::ReceiptView, ) -> Result { - if let near_lake_framework::near_indexer_primitives::views::ReceiptEnumView::Action { + if let near_indexer_primitives::views::ReceiptEnumView::Action { signer_id, signer_public_key, gas_price, @@ -129,7 +127,7 @@ impl ActionReceiptAction { pub fn from_action_view( receipt_id: String, index: i32, - action_view: &near_lake_framework::near_indexer_primitives::views::ActionView, + action_view: &near_indexer_primitives::views::ActionView, predecessor_account_id: String, receiver_account_id: String, block_timestamp: u64, diff --git a/database/src/models/serializers.rs b/database/src/models/serializers.rs index ac83cc54..c741fd51 100644 --- a/database/src/models/serializers.rs +++ b/database/src/models/serializers.rs @@ -1,7 +1,7 @@ use serde::{Deserialize, Serialize}; use serde_json::json; -use near_lake_framework::near_indexer_primitives::views::ActionView; +use near_indexer_primitives::views::ActionView; use near_primitives::serialize::option_u128_dec_format; use crate::models::enums::ActionKind; @@ -10,14 +10,12 @@ use crate::models::enums::ActionKind; /// to change serde parameters of serialization. #[derive(Serialize, Deserialize, Debug, Clone)] pub(crate) struct AccessKeyView { - pub nonce: near_lake_framework::near_indexer_primitives::types::Nonce, + pub nonce: near_indexer_primitives::types::Nonce, pub permission: AccessKeyPermissionView, } -impl From<&near_lake_framework::near_indexer_primitives::views::AccessKeyView> for AccessKeyView { - fn from( - access_key_view: &near_lake_framework::near_indexer_primitives::views::AccessKeyView, - ) -> Self { +impl From<&near_indexer_primitives::views::AccessKeyView> for AccessKeyView { + fn from(access_key_view: &near_indexer_primitives::views::AccessKeyView) -> Self { Self { nonce: access_key_view.nonce, permission: access_key_view.permission.clone().into(), @@ -36,24 +34,18 @@ impl From<&near_lake_framework::near_indexer_primitives::views::AccessKeyView> f pub(crate) enum AccessKeyPermissionView { FunctionCall { #[serde(with = "option_u128_dec_format")] - allowance: Option, + allowance: Option, receiver_id: String, method_names: Vec, }, FullAccess, } -impl From - for AccessKeyPermissionView -{ - fn from( - permission: near_lake_framework::near_indexer_primitives::views::AccessKeyPermissionView, - ) -> Self { +impl From for AccessKeyPermissionView { + fn from(permission: near_indexer_primitives::views::AccessKeyPermissionView) -> Self { match permission { - near_lake_framework::near_indexer_primitives::views::AccessKeyPermissionView::FullAccess => { - Self::FullAccess - } - near_lake_framework::near_indexer_primitives::views::AccessKeyPermissionView::FunctionCall { + near_indexer_primitives::views::AccessKeyPermissionView::FullAccess => Self::FullAccess, + near_indexer_primitives::views::AccessKeyPermissionView::FunctionCall { allowance, receiver_id, method_names, @@ -70,7 +62,7 @@ impl From (crate::models::enums::ActionKind, serde_json::Value) { match action_view { ActionView::CreateAccount => (ActionKind::CreateAccount, json!({})), diff --git a/database/src/models/transactions.rs b/database/src/models/transactions.rs index 5ed99a8d..6ba76e66 100644 --- a/database/src/models/transactions.rs +++ b/database/src/models/transactions.rs @@ -26,12 +26,12 @@ pub struct Transaction { impl Transaction { pub fn from_indexer_transaction( - tx: &near_lake_framework::near_indexer_primitives::IndexerTransactionWithOutcome, + tx: &near_indexer_primitives::IndexerTransactionWithOutcome, // hack for supporting duplicated transaction hashes transaction_hash: &str, converted_into_receipt_id: &str, - block_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, - chunk_hash: &near_lake_framework::near_indexer_primitives::CryptoHash, + block_hash: &near_indexer_primitives::CryptoHash, + chunk_hash: &near_indexer_primitives::CryptoHash, block_timestamp: u64, index_in_chunk: i32, ) -> Self { @@ -74,7 +74,7 @@ impl TransactionAction { pub fn from_action_view( transaction_hash: String, index: i32, - action_view: &near_lake_framework::near_indexer_primitives::views::ActionView, + action_view: &near_indexer_primitives::views::ActionView, ) -> Self { let (action_kind, args) = crate::models::extract_action_type_and_value_from_action_view(action_view); diff --git a/database/src/receipts_cache.rs b/database/src/receipts_cache.rs index 523905e8..cd4c2179 100644 --- a/database/src/receipts_cache.rs +++ b/database/src/receipts_cache.rs @@ -3,8 +3,8 @@ use tokio::sync::Mutex; #[derive(Clone, Hash, PartialEq, Eq, Debug)] pub enum ReceiptOrDataId { - ReceiptId(near_lake_framework::near_indexer_primitives::CryptoHash), - DataId(near_lake_framework::near_indexer_primitives::CryptoHash), + ReceiptId(near_indexer_primitives::CryptoHash), + DataId(near_indexer_primitives::CryptoHash), } // Creating type aliases to make HashMap types for cache more explicit pub type ParentTransactionHashString = String; From e9f5d9eed23cc9157e75e5fd8a85b02204ebc5eb Mon Sep 17 00:00:00 2001 From: Vlad Frolov Date: Fri, 4 Nov 2022 14:05:50 +0100 Subject: [PATCH 17/42] docs: Added a warning to the skipping blocks section in troubleshooting guide (#11) --- TROBLESHOOTING.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/TROBLESHOOTING.md b/TROBLESHOOTING.md index 16963631..98b53b41 100644 --- a/TROBLESHOOTING.md +++ b/TROBLESHOOTING.md @@ -76,6 +76,8 @@ Example of not fine indexer logs: For now, we want to try to “skip” that block. In order to achieve this, we would run the indexer syncing process from the block height number following for the one where we have stuck. +**WARNING:** This mode should only be used when there is no other way to get Indexer for Explorer unstuck and we want to trade correctness for livability of Explorer and Wallet for 99.999% users. After the incidents where some of the blocks were skipped, we MUST go back to fix the bug and re-index the borked blocks to ensure full consistency of the database! + If the stuck block height is 47961813 ```bash From 5794def0d01794115b2f9f8e37f79773feda7abf Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Fri, 18 Nov 2022 08:39:23 +1300 Subject: [PATCH 18/42] feat: Introduce standalone binary to calculate circulating supply via JSON RPC (#10) * feat: Add empty `circulating_supply` crate * feat: Add missing db adapaters/models to `database` lib * feat: Add core circulating supply files * refactor: Flatten/merge `circulating_supply` files * fix: Add missing dependencies and fix broken imports Excluding `near-indexer` and `near-client` as these will be replaced * refactor: Make in-use circulating_supply functions public * refactor: Replace `near_indexer::near_primitives` with `near_lake_framework::near_indexer_primitives` * refactor: Instantiate pg pool in `main` * refactor: Replace `ViewClient` calls with `JsonRpcClient` * refactor: `await` circulating_supply in `main` * chore: Rename `tracing` target `aggregated` -> `circulating_supply` * refactor: Extract json rpc url to environment variable * feat: Subscribe to tracing events via `RUST_LOG` env * feat: Improve logging * refactor: Replace `near-indexer-framework` with `near-indexer-primtives` * fix: Base64 decode lockup state before deserializing * feat: Print `anyhow` errors with cause * chore: Add note to ensure API use is updated when upgrading near-core * chore: Use kebab case for workspace crate --- Cargo.lock | 21 ++ Cargo.toml | 3 +- circulating-supply/Cargo.toml | 28 ++ circulating-supply/src/account_details.rs | 59 ++++ circulating-supply/src/lockup.rs | 195 +++++++++++++ circulating-supply/src/lockup_types.rs | 211 ++++++++++++++ circulating-supply/src/main.rs | 267 ++++++++++++++++++ database/src/adapters/accounts.rs | 56 +++- .../adapters/aggregated/circulating_supply.rs | 61 ++++ database/src/adapters/aggregated/mod.rs | 1 + database/src/adapters/blocks.rs | 13 + database/src/adapters/mod.rs | 1 + database/src/models/aggregated/mod.rs | 2 +- database/src/models/mod.rs | 1 + indexer/src/main.rs | 1 - 15 files changed, 916 insertions(+), 4 deletions(-) create mode 100644 circulating-supply/Cargo.toml create mode 100644 circulating-supply/src/account_details.rs create mode 100644 circulating-supply/src/lockup.rs create mode 100644 circulating-supply/src/lockup_types.rs create mode 100644 circulating-supply/src/main.rs create mode 100644 database/src/adapters/aggregated/circulating_supply.rs create mode 100644 database/src/adapters/aggregated/mod.rs diff --git a/Cargo.lock b/Cargo.lock index f0ab510d..32a7353e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -833,6 +833,27 @@ dependencies = [ "generic-array", ] +[[package]] +name = "circulating_supply" +version = "0.1.0" +dependencies = [ + "actix", + "anyhow", + "base64 0.11.0", + "bigdecimal", + "chrono", + "dotenv", + "explorer-database", + "near-indexer-primitives", + "near-jsonrpc-client", + "near-jsonrpc-primitives", + "near-sdk", + "tokio", + "tracing", + "tracing-subscriber", + "uint 0.8.5", +] + [[package]] name = "clap" version = "3.1.18" diff --git a/Cargo.toml b/Cargo.toml index 664b7c9a..49e40e57 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [workspace] members = [ "indexer", - "database" + "database", + "circulating-supply" ] diff --git a/circulating-supply/Cargo.toml b/circulating-supply/Cargo.toml new file mode 100644 index 00000000..64009ae7 --- /dev/null +++ b/circulating-supply/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "circulating-supply" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +actix = "0.13.0" +anyhow = "1.0.51" +bigdecimal = "=0.1.0" +base64 = "0.11" +chrono = "0.4.19" +dotenv = "0.15.0" +# syn version conflict, replace with crates.io version once released +near-sdk = { git = "https://github.com/near/near-sdk-rs", rev="03487c184d37b0382dd9bd41c57466acad58fc1f" } +tokio = { version = "1.1", features = ["sync", "time"] } +tracing = "0.1.13" +tracing-subscriber = "0.2.4" +uint = { version = "0.8.3", default-features = false } + +near-indexer-primitives = "0.14" +near-jsonrpc-client = "0.4.0-beta.0" +# Remove manual base64 decoding of lockup state when upgrading to 0.15 +# https://github.com/khorolets/near-indexer-for-explorer/blob/master-lake/circulating_supply/src/lockup.rs#L60 +near-jsonrpc-primitives = "0.14.0" + +explorer-database = { path = '../database' } diff --git a/circulating-supply/src/account_details.rs b/circulating-supply/src/account_details.rs new file mode 100644 index 00000000..81a48430 --- /dev/null +++ b/circulating-supply/src/account_details.rs @@ -0,0 +1,59 @@ +use anyhow::Context; + +use near_jsonrpc_client::{methods, JsonRpcClient}; +use near_jsonrpc_primitives::types::query::QueryResponseKind; + +pub(crate) async fn get_account_balance( + rpc_client: &JsonRpcClient, + account_id: &near_indexer_primitives::types::AccountId, + block_height: &near_indexer_primitives::types::BlockHeight, +) -> anyhow::Result { + get_account_view_for_block_height(rpc_client, account_id, block_height) + .await + .map(|account| account.amount) + .with_context(|| format!("Unable to get account balance for {}", account_id)) +} + +pub(crate) async fn get_contract_code_hash( + rpc_client: &JsonRpcClient, + account_id: &near_indexer_primitives::types::AccountId, + block_height: &near_indexer_primitives::types::BlockHeight, +) -> anyhow::Result { + get_account_view_for_block_height(rpc_client, account_id, block_height) + .await + .map(|account| account.code_hash) + .with_context(|| format!("Unable to get contract code hash for {}", account_id)) +} + +async fn get_account_view_for_block_height( + rpc_client: &JsonRpcClient, + account_id: &near_indexer_primitives::types::AccountId, + block_height: &near_indexer_primitives::types::BlockHeight, +) -> anyhow::Result { + let block_reference = near_indexer_primitives::types::BlockReference::BlockId( + near_indexer_primitives::types::BlockId::Height(*block_height), + ); + let request = near_indexer_primitives::views::QueryRequest::ViewAccount { + account_id: account_id.clone(), + }; + let query = methods::query::RpcQueryRequest { + block_reference, + request, + }; + + let account_response = rpc_client.call(query).await.with_context(|| { + format!( + "Failed to deliver ViewAccount for account {}, block {}", + account_id, block_height + ) + })?; + + match account_response.kind { + QueryResponseKind::ViewAccount(account) => Ok(account), + _ => anyhow::bail!( + "Failed to extract ViewAccount response for account {}, block {}", + account_id, + block_height + ), + } +} diff --git a/circulating-supply/src/lockup.rs b/circulating-supply/src/lockup.rs new file mode 100644 index 00000000..2a053c07 --- /dev/null +++ b/circulating-supply/src/lockup.rs @@ -0,0 +1,195 @@ +use std::time::Duration; + +use anyhow::Context; + +use near_jsonrpc_client::{methods, JsonRpcClient}; +use near_jsonrpc_primitives::types::query::QueryResponseKind; +use near_sdk::borsh::BorshDeserialize; +use near_sdk::json_types::{U128, U64}; + +use super::lockup_types::{ + LockupContract, TransfersInformation, VestingInformation, VestingSchedule, WrappedBalance, U256, +}; + +// The timestamp (nanos) when transfers were enabled in the Mainnet after community voting +// Tuesday, 13 October 2020 18:38:58.293 +pub(super) const TRANSFERS_ENABLED: Duration = Duration::from_nanos(1602614338293769340); + +pub(super) async fn get_lockup_contract_state( + rpc_client: &JsonRpcClient, + account_id: &near_indexer_primitives::types::AccountId, + block_height: &near_indexer_primitives::types::BlockHeight, +) -> anyhow::Result { + let block_reference = near_indexer_primitives::types::BlockReference::BlockId( + near_indexer_primitives::types::BlockId::Height(*block_height), + ); + let request = near_indexer_primitives::views::QueryRequest::ViewState { + account_id: account_id.clone(), + prefix: vec![].into(), + }; + let query = methods::query::RpcQueryRequest { + block_reference, + request, + }; + + let state_response = rpc_client.call(query).await.with_context(|| { + format!( + "Failed to deliver ViewState for lockup contract {}, block_height {}", + account_id, block_height + ) + })?; + + let view_state_result = match state_response.kind { + QueryResponseKind::ViewState(state) => state, + _ => { + anyhow::bail!( + "Failed to extract ViewState response for lockup contract {}, block_height {}", + account_id, + block_height + ) + } + }; + + let view_state = view_state_result.values.get(0).with_context(|| { + format!( + "Failed to find encoded lockup contract for {}, block_height {}", + account_id, block_height + ) + })?; + + let decoded_view_state = base64::decode(&view_state.value) + .context(format!("Failed to decode lockup state for: {}", account_id))?; + + let mut state = LockupContract::try_from_slice(&decoded_view_state) + .with_context(|| format!("Failed to construct LockupContract for {}", account_id))?; + + // If owner of the lockup account didn't call the + // `check_transfers_vote` contract method we won't be able to + // get proper information based on timestamp, that's why we inject + // the `transfer_timestamp` which is phase2 timestamp + state.lockup_information.transfers_information = TransfersInformation::TransfersEnabled { + transfers_timestamp: U64(TRANSFERS_ENABLED.as_nanos() as u64), + }; + Ok(state) +} + +// The lockup contract implementation had a bug that affected lockup start date. +// https://github.com/near/core-contracts/pull/136 +// For each contract, we should choose the logic based on the binary version of the contract +pub(super) fn is_bug_inside_contract( + code_hash: &near_indexer_primitives::CryptoHash, + account_id: &near_indexer_primitives::types::AccountId, +) -> anyhow::Result { + match &*code_hash.to_string() { + // The first implementation, with the bug + "3kVY9qcVRoW3B5498SMX6R3rtSLiCdmBzKs7zcnzDJ7Q" => Ok(true), + // We have 6 lockups created at 6th of April 2021, assume it's buggy + "DiC9bKCqUHqoYqUXovAnqugiuntHWnM3cAc7KrgaHTu" => Ok(true), + // Another 5 lockups created in May/June 2021, assume they are OK + "Cw7bnyp4B6ypwvgZuMmJtY6rHsxP2D4PC8deqeJ3HP7D" => Ok(false), + // The most fresh one + "4Pfw2RU6e35dUsHQQoFYfwX8KFFvSRNwMSNLXuSFHXrC" => Ok(false), + other => anyhow::bail!( + "Unable to recognise the version of contract {}, code hash {}", + account_id, + other + ), + } +} + +// This is almost a copy of https://github.com/near/core-contracts/blob/master/lockup/src/getters.rs#L64 +impl LockupContract { + /// Returns the amount of tokens that are locked in the account due to lockup or vesting. + pub fn get_locked_amount(&self, timestamp: u64, has_bug: bool) -> WrappedBalance { + let lockup_amount = self.lockup_information.lockup_amount; + if let TransfersInformation::TransfersEnabled { + transfers_timestamp, + } = &self.lockup_information.transfers_information + { + let lockup_timestamp = std::cmp::max( + transfers_timestamp + .0 + .saturating_add(self.lockup_information.lockup_duration), + self.lockup_information.lockup_timestamp.unwrap_or(0), + ); + let block_timestamp = timestamp; + if lockup_timestamp <= block_timestamp { + let unreleased_amount = + if let Some(release_duration) = self.lockup_information.release_duration { + let start_lockup = if has_bug { + transfers_timestamp.0 + } else { + lockup_timestamp + }; + let end_timestamp = start_lockup.saturating_add(release_duration); + if block_timestamp >= end_timestamp { + // Everything is released + 0 + } else { + let time_left = U256::from(end_timestamp - block_timestamp); + let unreleased_amount = U256::from(lockup_amount) * time_left + / U256::from(release_duration); + // The unreleased amount can't be larger than lockup_amount because the + // time_left is smaller than total_time. + unreleased_amount.as_u128() + } + } else { + 0 + }; + + let unvested_amount = match &self.vesting_information { + VestingInformation::VestingSchedule(vs) => { + self.get_unvested_amount(vs.clone(), block_timestamp) + } + VestingInformation::Terminating(terminating) => terminating.unvested_amount, + // Vesting is private, so we can assume the vesting started before lockup date. + _ => U128(0), + }; + return std::cmp::max( + unreleased_amount + .saturating_sub(self.lockup_information.termination_withdrawn_tokens), + unvested_amount.0, + ) + .into(); + } + } + // The entire balance is still locked before the lockup timestamp. + (lockup_amount - self.lockup_information.termination_withdrawn_tokens).into() + } + + /// Returns the amount of tokens that are locked in this account due to vesting schedule. + /// Takes raw vesting schedule, in case the internal vesting schedule is private. + pub fn get_unvested_amount( + &self, + vesting_schedule: VestingSchedule, + block_timestamp: u64, + ) -> WrappedBalance { + let lockup_amount = self.lockup_information.lockup_amount; + match &self.vesting_information { + VestingInformation::Terminating(termination_information) => { + termination_information.unvested_amount + } + VestingInformation::None => U128::from(0), + _ => { + if block_timestamp < vesting_schedule.cliff_timestamp.0 { + // Before the cliff, nothing is vested + lockup_amount.into() + } else if block_timestamp >= vesting_schedule.end_timestamp.0 { + // After the end, everything is vested + 0.into() + } else { + // cannot overflow since block_timestamp < vesting_schedule.end_timestamp + let time_left = U256::from(vesting_schedule.end_timestamp.0 - block_timestamp); + // The total time is positive. Checked at the contract initialization. + let total_time = U256::from( + vesting_schedule.end_timestamp.0 - vesting_schedule.start_timestamp.0, + ); + let unvested_amount = U256::from(lockup_amount) * time_left / total_time; + // The unvested amount can't be larger than lockup_amount because the + // time_left is smaller than total_time. + unvested_amount.as_u128().into() + } + } + } + } +} diff --git a/circulating-supply/src/lockup_types.rs b/circulating-supply/src/lockup_types.rs new file mode 100644 index 00000000..905910b8 --- /dev/null +++ b/circulating-supply/src/lockup_types.rs @@ -0,0 +1,211 @@ +#![allow( + clippy::assign_op_pattern, + clippy::manual_range_contains, + clippy::ptr_offset_with_cast +)] +// Copied from lockup contract code +// https://github.com/near/core-contracts/blob/master/lockup/src/types.rs +// https://github.com/near/core-contracts/blob/master/lockup/src/lib.rs + +use uint::construct_uint; + +use near_sdk::borsh::{self, BorshDeserialize, BorshSerialize}; +use near_sdk::json_types::{Base64VecU8, U128, U64}; +use near_sdk::serde::{Deserialize, Serialize}; +use near_sdk::{AccountId, Balance}; + +construct_uint! { + /// 256-bit unsigned integer. + pub struct U256(4); +} + +/// Raw type for duration in nanoseconds +pub type Duration = u64; +/// Raw type for timestamp in nanoseconds +pub type Timestamp = u64; + +/// Timestamp in nanosecond wrapped into a struct for JSON serialization as a string. +pub type WrappedTimestamp = U64; +/// Balance wrapped into a struct for JSON serialization as a string. +pub type WrappedBalance = U128; + +#[derive(BorshDeserialize, BorshSerialize)] +pub struct LockupContract { + /// The account ID of the owner. + pub owner_account_id: AccountId, + + /// Information about lockup schedule and the amount. + pub lockup_information: LockupInformation, + + /// Information about vesting including schedule or termination status. + pub vesting_information: VestingInformation, + + /// Account ID of the staking pool whitelist contract. + pub staking_pool_whitelist_account_id: AccountId, + + /// Information about staking and delegation. + /// `Some` means the staking information is available and the staking pool contract is selected. + /// `None` means there is no staking pool selected. + pub staking_information: Option, + + /// The account ID that the NEAR Foundation, that has the ability to terminate vesting. + pub foundation_account_id: Option, +} + +/// Contains information about token lockups. +#[derive(BorshDeserialize, BorshSerialize)] +pub struct LockupInformation { + /// The amount in yocto-NEAR tokens locked for this account. + pub lockup_amount: Balance, + /// The amount of tokens that were withdrawn by NEAR foundation due to early termination + /// of vesting. + /// This amount has to be accounted separately from the lockup_amount to make sure + /// linear release is not being affected. + pub termination_withdrawn_tokens: Balance, + /// [deprecated] - the duration in nanoseconds of the lockup period from + /// the moment the transfers are enabled. During this period tokens are locked and + /// the release doesn't start. Instead of this, use `lockup_timestamp` and `release_duration` + pub lockup_duration: Duration, + /// If present, it is the duration when the full lockup amount will be available. The tokens + /// are linearly released from the moment tokens are unlocked, defined by: + /// `max(transfers_timestamp + lockup_duration, lockup_timestamp)`. + /// If not present, the tokens are not locked (though, vesting logic could be used). + pub release_duration: Option, + /// The optional absolute lockup timestamp in nanoseconds which locks the tokens until this + /// timestamp passes. Until this moment the tokens are locked and the release doesn't start. + /// If not present, `transfers_timestamp` will be used. + pub lockup_timestamp: Option, + /// The information about the transfers. Either transfers are already enabled, then it contains + /// the timestamp when they were enabled. Or the transfers are currently disabled and + /// it contains the account ID of the transfer poll contract. + pub transfers_information: TransfersInformation, +} + +/// Contains information about the transfers. Whether transfers are enabled or disabled. +#[derive(BorshDeserialize, BorshSerialize, Deserialize, Serialize, Debug)] +#[serde(crate = "near_sdk::serde")] +pub enum TransfersInformation { + /// The timestamp when the transfers were enabled. + TransfersEnabled { + transfers_timestamp: WrappedTimestamp, + }, + /// The account ID of the transfers poll contract, to check if the transfers are enabled. + /// The lockup period can start only after the transfer voted to be enabled. + /// At the launch of the network transfers are disabled for all lockup contracts, once transfers + /// are enabled, they can't be disabled and don't need to be checked again. + TransfersDisabled { transfer_poll_account_id: AccountId }, +} + +/// Describes the status of transactions with the staking pool contract or terminated unvesting +/// amount withdrawal. +#[derive(BorshDeserialize, BorshSerialize, Deserialize, Serialize, PartialEq)] +#[serde(crate = "near_sdk::serde")] +pub enum TransactionStatus { + /// There are no transactions in progress. + Idle, + /// There is a transaction in progress. + Busy, +} + +/// Contains information about current stake and delegation. +#[derive(BorshDeserialize, BorshSerialize)] +pub struct StakingInformation { + /// The Account ID of the staking pool contract. + pub staking_pool_account_id: AccountId, + + /// Contains status whether there is a transaction in progress. + pub status: TransactionStatus, + + /// The amount of tokens that were deposited from this account to the staking pool. + /// NOTE: The unstaked amount on the staking pool might be higher due to staking rewards. + pub deposit_amount: WrappedBalance, +} + +/// Contains information about vesting schedule. +#[derive(BorshDeserialize, BorshSerialize, Deserialize, Serialize, Clone, PartialEq, Debug)] +#[serde(crate = "near_sdk::serde")] +pub struct VestingSchedule { + /// The timestamp in nanosecond when the vesting starts. E.g. the start date of employment. + pub start_timestamp: WrappedTimestamp, + /// The timestamp in nanosecond when the first part of lockup tokens becomes vested. + /// The remaining tokens will vest continuously until they are fully vested. + /// Example: a 1 year of employment at which moment the 1/4 of tokens become vested. + pub cliff_timestamp: WrappedTimestamp, + /// The timestamp in nanosecond when the vesting ends. + pub end_timestamp: WrappedTimestamp, +} + +/// Initialization argument type to define the vesting schedule +#[derive(Serialize, Deserialize, Debug)] +#[serde(crate = "near_sdk::serde")] +pub enum VestingScheduleOrHash { + /// [deprecated] After transfers are enabled, only public schedule is used. + /// The vesting schedule is private and this is a hash of (vesting_schedule, salt). + /// In JSON, the hash has to be encoded with base64 to a string. + VestingHash(Base64VecU8), + /// The vesting schedule (public) + VestingSchedule(VestingSchedule), +} + +/// Contains information about vesting that contains vesting schedule and termination information. +#[derive(Serialize, BorshDeserialize, BorshSerialize, PartialEq, Clone, Debug)] +#[serde(crate = "near_sdk::serde")] +pub enum VestingInformation { + None, + /// [deprecated] After transfers are enabled, only public schedule is used. + /// Vesting schedule is hashed for privacy and only will be revealed if the NEAR foundation + /// has to terminate vesting. + /// The contract assume the vesting schedule doesn't affect lockup release and duration, because + /// the vesting started before transfers were enabled and the duration is shorter or the same. + VestingHash(Base64VecU8), + /// Explicit vesting schedule. + VestingSchedule(VestingSchedule), + /// The information about the early termination of the vesting schedule. + /// It means the termination of the vesting is currently in progress. + /// Once the unvested amount is transferred out, `VestingInformation` is removed. + Terminating(TerminationInformation), +} + +/// Describes the status of transactions with the staking pool contract or terminated unvesting +/// amount withdrawal. +#[derive( + BorshDeserialize, BorshSerialize, Deserialize, Serialize, PartialEq, Copy, Clone, Debug, +)] +#[serde(crate = "near_sdk::serde")] +pub enum TerminationStatus { + /// Initial stage of the termination in case there are deficit on the account. + VestingTerminatedWithDeficit, + /// A transaction to unstake everything is in progress. + UnstakingInProgress, + /// The transaction to unstake everything from the staking pool has completed. + EverythingUnstaked, + /// A transaction to withdraw everything from the staking pool is in progress. + WithdrawingFromStakingPoolInProgress, + /// Everything is withdrawn from the staking pool. Ready to withdraw out of the account. + ReadyToWithdraw, + /// A transaction to withdraw tokens from the account is in progress. + WithdrawingFromAccountInProgress, +} + +/// Contains information about early termination of the vesting schedule. +#[derive(BorshDeserialize, BorshSerialize, Deserialize, Serialize, PartialEq, Clone, Debug)] +#[serde(crate = "near_sdk::serde")] +pub struct TerminationInformation { + /// The amount of tokens that are unvested and has to be transferred back to NEAR Foundation. + /// These tokens are effectively locked and can't be transferred out and can't be restaked. + pub unvested_amount: WrappedBalance, + + /// The status of the withdrawal. When the unvested amount is in progress of withdrawal the + /// status will be marked as busy, to avoid withdrawing the funds twice. + pub status: TerminationStatus, +} + +/// Contains a vesting schedule with a salt. +#[derive(BorshSerialize, Deserialize, Serialize, Clone, Debug)] +#[serde(crate = "near_sdk::serde")] +pub struct VestingScheduleWithSalt { + /// The vesting schedule + pub vesting_schedule: VestingSchedule, + /// Salt to make the hash unique + pub salt: Base64VecU8, +} diff --git a/circulating-supply/src/main.rs b/circulating-supply/src/main.rs new file mode 100644 index 00000000..7e7402b0 --- /dev/null +++ b/circulating-supply/src/main.rs @@ -0,0 +1,267 @@ +use std::ops::{Add, Sub}; +use std::str::FromStr; +use std::time::{Duration, SystemTime}; + +use anyhow::Context; +use bigdecimal::{BigDecimal, ToPrimitive}; +use chrono::NaiveDateTime; +use tracing::{error, info, warn}; + +use near_jsonrpc_client::{methods, JsonRpcClient}; + +use explorer_database::{adapters, models}; + +mod account_details; +mod lockup; +mod lockup_types; + +const DAY: Duration = Duration::from_secs(60 * 60 * 24); +const RETRY_DURATION: Duration = Duration::from_secs(60 * 60 * 2); + +const CIRCULATING_SUPPLY: &str = "circulating_supply"; + +#[actix::main] +async fn main() { + dotenv::dotenv().ok(); + + tracing_subscriber::fmt::Subscriber::builder() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .init(); + + let pool = models::establish_connection( + &std::env::var("DATABASE_URL") + .expect("DATABASE_URL must be set in either .env or environment "), + ); + + let rpc_client = JsonRpcClient::connect( + std::env::var("RPC_URL").expect("RPC_URL must be set in either .env or environment"), + ); + + info!(target: crate::CIRCULATING_SUPPLY, "Starting calculations"); + + run_circulating_supply_computation(rpc_client, pool).await; +} + +// Compute circulating supply on a daily basis, starting from 13 Oct 2020 +// (Transfers enabled moment on the Mainnet), and put it to the Indexer DB. +// Circulating supply is calculated by the formula: +// total_supply - sum(locked_tokens_on_each_lockup) - sum(locked_foundation_account) +// The value is always computed for the last block in a day (UTC). +pub async fn run_circulating_supply_computation( + rpc_client: JsonRpcClient, + pool: explorer_database::actix_diesel::Database, +) { + // We perform actual computations 00:10 UTC each day to be sure that the data is finalized + let mut day_to_compute = lockup::TRANSFERS_ENABLED + .sub(Duration::from_secs( + lockup::TRANSFERS_ENABLED.as_secs() % DAY.as_secs(), + )) + .add(DAY) + .add(Duration::from_secs(10 * 60)); + + loop { + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .expect("Time went backwards"); + + if now < day_to_compute { + info!( + target: crate::CIRCULATING_SUPPLY, + "Waiting until {} to resume calculations", + NaiveDateTime::from_timestamp(day_to_compute.as_secs() as i64, 0) + ); + tokio::time::sleep_until(tokio::time::Instant::now().add(day_to_compute.sub(now))) + .await; + } + wait_for_loading_needed_blocks(&rpc_client, &day_to_compute).await; + + match check_and_collect_daily_circulating_supply(&rpc_client, &pool, &day_to_compute).await + { + Ok(_) => { + day_to_compute = day_to_compute.add(DAY); + } + Err(err) => { + error!( + target: crate::CIRCULATING_SUPPLY, + "Failed to compute circulating supply for {}: {:#}. Retry in {} hours", + NaiveDateTime::from_timestamp(day_to_compute.as_secs() as i64, 0).date(), + err, + RETRY_DURATION.as_secs() / 60 / 60, + ); + tokio::time::sleep(RETRY_DURATION).await; + } + }; + } +} + +async fn check_and_collect_daily_circulating_supply( + rpc_client: &JsonRpcClient, + pool: &explorer_database::actix_diesel::Database, + request_datetime: &Duration, +) -> anyhow::Result> { + let start_of_day = + request_datetime.as_nanos() - request_datetime.as_nanos() % crate::DAY.as_nanos(); + let printable_date = NaiveDateTime::from_timestamp(request_datetime.as_secs() as i64, 0).date(); + let block = + adapters::blocks::get_latest_block_before_timestamp(pool, start_of_day as u64).await?; + let block_timestamp = block + .block_timestamp + .to_u64() + .context("`block_timestamp` expected to be u64")?; + + match adapters::aggregated::circulating_supply::get_precomputed_circulating_supply_for_timestamp( + pool, + block_timestamp, + ) + .await + { + Ok(None) => { + info!( + target: crate::CIRCULATING_SUPPLY, + "Computing circulating supply for {} (timestamp {})", + printable_date, + block_timestamp + ); + let supply = compute_circulating_supply_for_block(pool, rpc_client, &block).await?; + adapters::aggregated::circulating_supply::add_circulating_supply(pool, &supply).await; + info!( + target: crate::CIRCULATING_SUPPLY, + "Circulating supply for {} (timestamp {}) is {}", + printable_date, + block_timestamp, + supply.circulating_tokens_supply + ); + Ok(Some(supply)) + } + Ok(Some(supply)) => { + info!( + target: crate::CIRCULATING_SUPPLY, + "Circulating supply for {} (timestamp {}) was already computed: {}", + printable_date, + block_timestamp, + supply + ); + Ok(None) + } + Err(err) => Err(err), + } +} + +async fn compute_circulating_supply_for_block( + pool: &explorer_database::actix_diesel::Database, + rpc_client: &JsonRpcClient, + block: &models::Block, +) -> anyhow::Result { + let block_timestamp = block + .block_timestamp + .to_u64() + .context("`block_timestamp` expected to be u64")?; + let block_height = block + .block_height + .to_u64() + .context("`block_height` expected to be u64")?; + let total_supply = block + .total_supply + .to_string() + .parse::() + .context("`total_supply` expected to be u128")?; + + let lockup_account_ids = + adapters::accounts::get_lockup_account_ids_at_block_height(pool, &block_height).await?; + + let mut lockups_locked_tokens: u128 = 0; + let mut unfinished_lockup_contracts_count: i32 = 0; + + for lockup_account_id in &lockup_account_ids { + let state = lockup::get_lockup_contract_state(rpc_client, lockup_account_id, &block_height) + .await + .with_context(|| { + format!( + "Failed to get lockup contract details for {}", + lockup_account_id + ) + })?; + let code_hash = + account_details::get_contract_code_hash(rpc_client, lockup_account_id, &block_height) + .await?; + let is_lockup_with_bug = lockup::is_bug_inside_contract(&code_hash, lockup_account_id)?; + let locked_amount = state + .get_locked_amount(block_timestamp, is_lockup_with_bug) + .0; + lockups_locked_tokens += locked_amount; + if locked_amount > 0 { + unfinished_lockup_contracts_count += 1; + } + } + + // The list is taken from the conversation with Yessin + let foundation_locked_account_ids: [near_indexer_primitives::types::AccountId; 2] = [ + near_indexer_primitives::types::AccountId::from_str("lockup.near") + .expect("lockup.near expected to be a valid AccountId"), + near_indexer_primitives::types::AccountId::from_str("contributors.near") + .expect("contributors.near expected to be a valid AccountId"), + ]; + let mut foundation_locked_tokens: u128 = 0; + for account_id in &foundation_locked_account_ids { + foundation_locked_tokens += + account_details::get_account_balance(rpc_client, account_id, &block_height).await?; + } + + let circulating_supply: u128 = total_supply - foundation_locked_tokens - lockups_locked_tokens; + + Ok(models::aggregated::circulating_supply::CirculatingSupply { + computed_at_block_timestamp: BigDecimal::from(block_timestamp), + computed_at_block_hash: (&block.block_hash).to_string(), + circulating_tokens_supply: BigDecimal::from_str(&circulating_supply.to_string()) + .context("`circulating_tokens_supply` expected to be u128")?, + total_tokens_supply: BigDecimal::from_str(&total_supply.to_string()) + .context("`total_supply` expected to be u128")?, + total_lockup_contracts_count: lockup_account_ids.len() as i32, + unfinished_lockup_contracts_count, + foundation_locked_tokens: BigDecimal::from_str(&foundation_locked_tokens.to_string()) + .context("`foundation_locked_supply` expected to be u128")?, + lockups_locked_tokens: BigDecimal::from_str(&lockups_locked_tokens.to_string()) + .context("`lockups_locked_supply` expected to be u128")?, + }) +} + +async fn wait_for_loading_needed_blocks(rpc_client: &JsonRpcClient, day_to_compute: &Duration) { + loop { + match get_final_block_timestamp(rpc_client).await { + Ok(timestamp) => { + if timestamp > *day_to_compute { + return; + } + warn!( + target: crate::CIRCULATING_SUPPLY, + "Blocks are not loaded to calculate circulating supply for {}. Wait for {} hours", + NaiveDateTime::from_timestamp(day_to_compute.as_secs() as i64, 0).date(), + crate::RETRY_DURATION.as_secs() / 60 / 60, + ); + } + Err(err) => { + error!( + target: crate::CIRCULATING_SUPPLY, + "Failed to get latest block timestamp: {}. Retry in {} hours", + err, + crate::RETRY_DURATION.as_secs() / 60 / 60, + ); + } + } + tokio::time::sleep(crate::RETRY_DURATION).await; + } +} + +async fn get_final_block_timestamp(rpc_client: &JsonRpcClient) -> anyhow::Result { + let block_reference = near_indexer_primitives::types::BlockReference::Finality( + near_indexer_primitives::types::Finality::Final, + ); + let query = methods::block::RpcBlockRequest { block_reference }; + + let block_response = rpc_client + .call(query) + .await + .context("Failed to deliver response")?; + + Ok(Duration::from_nanos(block_response.header.timestamp)) +} diff --git a/database/src/adapters/accounts.rs b/database/src/adapters/accounts.rs index e894f6de..dde4f2c6 100644 --- a/database/src/adapters/accounts.rs +++ b/database/src/adapters/accounts.rs @@ -1,8 +1,10 @@ use std::collections::HashMap; use actix_diesel::dsl::AsyncRunQueryDsl; +use anyhow::Context; +use bigdecimal::BigDecimal; -use diesel::{ExpressionMethods, PgConnection, QueryDsl}; +use diesel::{BoolExpressionMethods, ExpressionMethods, PgConnection, QueryDsl}; use futures::try_join; use crate::models; @@ -195,3 +197,55 @@ pub async fn handle_accounts( try_join!(delete_accounts_future, create_or_update_accounts_future)?; Ok(()) } + +pub async fn get_lockup_account_ids_at_block_height( + pool: &actix_diesel::Database, + block_height: &near_primitives::types::BlockHeight, +) -> anyhow::Result> { + // Diesel does not support named joins + // https://github.com/diesel-rs/diesel/pull/2254 + // Raw SQL (diesel-1.4.7/src/query_builder/functions.rs:464) does not support async methods + // So we decided to use view + simple SQL with `where` clause + // Initial SQL statement: + // let raw_sql: String = format!(" + // SELECT accounts.account_id, blocks_start.block_height, blocks_end.block_height + // FROM accounts + // LEFT JOIN receipts AS receipts_start ON accounts.created_by_receipt_id = receipts_start.receipt_id + // LEFT JOIN blocks AS blocks_start ON receipts_start.included_in_block_hash = blocks_start.block_hash + // LEFT JOIN receipts AS receipts_end ON accounts.deleted_by_receipt_id = receipts_end.receipt_id + // LEFT JOIN blocks AS blocks_end ON receipts_end.included_in_block_hash = blocks_end.block_hash + // WHERE accounts.account_id like '%.lockup.near' + // AND (blocks_start.block_height IS NULL OR blocks_start.block_height <= {0}) + // AND (blocks_end.block_height IS NULL OR blocks_end.block_height >= {0}); + // ", block_height); + + schema::aggregated__lockups::table + .select(schema::aggregated__lockups::dsl::account_id) + .filter( + schema::aggregated__lockups::dsl::creation_block_height + .is_null() + .or(schema::aggregated__lockups::dsl::creation_block_height + .le(BigDecimal::from(*block_height))), + ) + .filter( + schema::aggregated__lockups::dsl::deletion_block_height + .is_null() + .or(schema::aggregated__lockups::dsl::deletion_block_height + .ge(BigDecimal::from(*block_height))), + ) + .get_results_async::(pool) + .await + .with_context(|| format!( + "DB error while collecting lockup account ids for block_height {}", + block_height + ) + ) + .map(|results| { + results + .into_iter() + .map(|account_id_string| + near_primitives::types::AccountId::try_from(account_id_string) + .expect("Selecting lockup account ids bumped into the account_id which is not valid; that should never happen")) + .collect() + }) +} diff --git a/database/src/adapters/aggregated/circulating_supply.rs b/database/src/adapters/aggregated/circulating_supply.rs new file mode 100644 index 00000000..208657d4 --- /dev/null +++ b/database/src/adapters/aggregated/circulating_supply.rs @@ -0,0 +1,61 @@ +use actix_diesel::dsl::AsyncRunQueryDsl; +use bigdecimal::BigDecimal; +use diesel::{ExpressionMethods, PgConnection, QueryDsl}; +use tracing::error; + +use crate::models::aggregated::circulating_supply::CirculatingSupply; +use crate::schema; + +pub async fn add_circulating_supply( + pool: &actix_diesel::Database, + stats: &CirculatingSupply, +) { + let mut interval = crate::INTERVAL; + loop { + match diesel::insert_into(schema::aggregated__circulating_supply::table) + .values(stats.to_owned()) + .on_conflict_do_nothing() + .execute_async(pool) + .await + { + Ok(_) => { + break; + } + Err(async_error) => { + error!( + target: crate::EXPLORER_DATABASE, + "Error occurred while Circulating Supply was adding to database. Retrying in {} milliseconds... \n {:#?}", + interval.as_millis(), + async_error + ); + tokio::time::sleep(interval).await; + if interval < crate::MAX_DELAY_TIME { + interval *= 2; + } + } + } + } +} + +pub async fn get_precomputed_circulating_supply_for_timestamp( + pool: &actix_diesel::Database, + timestamp: u64, +) -> anyhow::Result> { + let supply = schema::aggregated__circulating_supply::table + .select(schema::aggregated__circulating_supply::dsl::circulating_tokens_supply) + .filter( + schema::aggregated__circulating_supply::dsl::computed_at_block_timestamp + .eq(BigDecimal::from(timestamp)), + ) + .get_optional_result_async::(pool) + .await; + + match supply { + Ok(Some(value)) => match value.to_string().parse::() { + Ok(res) => Ok(Some(res)), + Err(_) => anyhow::bail!("`circulating_tokens_supply` expected to be u128"), + }, + Ok(None) => Ok(None), + Err(err) => anyhow::bail!("DB Error: {}", err), + } +} diff --git a/database/src/adapters/aggregated/mod.rs b/database/src/adapters/aggregated/mod.rs new file mode 100644 index 00000000..1f0c3ff1 --- /dev/null +++ b/database/src/adapters/aggregated/mod.rs @@ -0,0 +1 @@ +pub mod circulating_supply; diff --git a/database/src/adapters/blocks.rs b/database/src/adapters/blocks.rs index 9558e76c..33862b0c 100644 --- a/database/src/adapters/blocks.rs +++ b/database/src/adapters/blocks.rs @@ -1,4 +1,5 @@ use actix_diesel::dsl::AsyncRunQueryDsl; +use anyhow::Context; use bigdecimal::{BigDecimal, ToPrimitive}; use diesel::{ExpressionMethods, PgConnection, QueryDsl}; @@ -37,3 +38,15 @@ pub async fn latest_block_height( .await? .and_then(|(block_height,)| block_height.to_u64())) } + +pub async fn get_latest_block_before_timestamp( + pool: &actix_diesel::Database, + timestamp: u64, +) -> anyhow::Result { + Ok(schema::blocks::table + .filter(schema::blocks::dsl::block_timestamp.le(BigDecimal::from(timestamp))) + .order(schema::blocks::dsl::block_timestamp.desc()) + .first_async::(pool) + .await + .context("DB Error")?) +} diff --git a/database/src/adapters/mod.rs b/database/src/adapters/mod.rs index cdb20a09..2a7b9a35 100644 --- a/database/src/adapters/mod.rs +++ b/database/src/adapters/mod.rs @@ -1,6 +1,7 @@ pub mod access_keys; pub mod account_changes; pub mod accounts; +pub mod aggregated; pub mod assets; pub mod blocks; pub mod chunks; diff --git a/database/src/models/aggregated/mod.rs b/database/src/models/aggregated/mod.rs index eecd50a6..1f0c3ff1 100644 --- a/database/src/models/aggregated/mod.rs +++ b/database/src/models/aggregated/mod.rs @@ -1 +1 @@ -pub(crate) mod circulating_supply; +pub mod circulating_supply; diff --git a/database/src/models/mod.rs b/database/src/models/mod.rs index ce71dd54..344cd09a 100644 --- a/database/src/models/mod.rs +++ b/database/src/models/mod.rs @@ -17,6 +17,7 @@ pub(crate) use serializers::extract_action_type_and_value_from_action_view; pub mod access_keys; pub mod account_changes; pub mod accounts; +pub mod aggregated; pub mod assets; pub mod blocks; pub mod chunks; diff --git a/indexer/src/main.rs b/indexer/src/main.rs index 47fa942f..b0d495fd 100644 --- a/indexer/src/main.rs +++ b/indexer/src/main.rs @@ -14,7 +14,6 @@ mod configs; // Categories for logging const INDEXER_FOR_EXPLORER: &str = "indexer_for_explorer"; -// const AGGREGATED: &str = "aggregated"; async fn handle_message( pool: &explorer_database::actix_diesel::Database, From 47d0210c40d443d19651c176ec7df99e5e3671c9 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Tue, 29 Nov 2022 07:14:52 +1300 Subject: [PATCH 19/42] chore: Add missing Changelog/Cargo version entries (#16) * chore: Update `indexer` to `10.2` * chore: Update `indexer` to `10.3` --- Cargo.lock | 4 ++-- indexer/CHANGELOG.md | 4 ++++ indexer/Cargo.toml | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 32a7353e..a9270c27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -834,7 +834,7 @@ dependencies = [ ] [[package]] -name = "circulating_supply" +name = "circulating-supply" version = "0.1.0" dependencies = [ "actix", @@ -1626,7 +1626,7 @@ dependencies = [ [[package]] name = "indexer-explorer-lake" -version = "0.10.1" +version = "0.10.3" dependencies = [ "actix", "actix-rt", diff --git a/indexer/CHANGELOG.md b/indexer/CHANGELOG.md index 5d888805..8578d4ec 100644 --- a/indexer/CHANGELOG.md +++ b/indexer/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.10.3 + +* Extract database logic to library crate + ## 0.10.2 * Avoid recreating access key on transfer to implicit account diff --git a/indexer/Cargo.toml b/indexer/Cargo.toml index edd0a9dd..3e569973 100644 --- a/indexer/Cargo.toml +++ b/indexer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "indexer-explorer-lake" -version = "0.10.1" +version = "0.10.3" authors = ["Near Inc "] edition = "2021" rust-version = "1.62.1" From 8db478099bf2ba88e54714ac7a67a427b8b94f69 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Wed, 30 Nov 2022 08:52:12 +1300 Subject: [PATCH 20/42] feat: Add missing diesel `migrations` (#12) * chore: Move `migrations` to same dir as `Diesel.toml` * chore: Remove unnecessary `.gitkeep` * feat: Add missing Diesel migrations * chore: Update `README.md` for running migrations --- README.md | 2 +- .../migrations/00000000000000_diesel_initial_setup/down.sql | 0 .../migrations/00000000000000_diesel_initial_setup/up.sql | 0 .../migrations/2020-12-07-153402_initial_schema/down.sql | 0 .../migrations/2020-12-07-153402_initial_schema/up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../2021-03-11-123839_index-action-kind-and-status/down.sql | 0 .../2021-03-11-123839_index-action-kind-and-status/up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../2021-04-28-154439_denormalize_action_receipt_actions/up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../2021-05-07-115559_convert_args_base64_to_args_json/down.sql | 0 .../2021-05-07-115559_convert_args_base64_to_args_json/up.sql | 0 .../migrations/2021-05-10-084700_indexer_on_args_field/down.sql | 0 .../migrations/2021-05-10-084700_indexer_on_args_field/up.sql | 0 .../2021-05-27-154211_account_changes_unique_idx/down.sql | 0 .../2021-05-27-154211_account_changes_unique_idx/up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../2021-08-02-183200_transactions_sorting_idx/down.sql | 0 .../2021-08-02-183200_transactions_sorting_idx/up.sql | 0 .../2021-08-04-151515_circulating_supply_table/down.sql | 0 .../2021-08-04-151515_circulating_supply_table/up.sql | 0 .../2021-08-06-123500_account_changes_ordering_column/down.sql | 0 .../2021-08-06-123500_account_changes_ordering_column/up.sql | 0 .../2021-08-11-163800_account_changes_ordering_idx/down.sql | 0 .../2021-08-11-163800_account_changes_ordering_idx/up.sql | 0 .../{src => }/migrations/2021-10-04-100000_assets_nft/down.sql | 0 .../{src => }/migrations/2021-10-04-100000_assets_nft/up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../{src => }/migrations/2022-01-12-100000_assets_ft/down.sql | 0 .../{src => }/migrations/2022-01-12-100000_assets_ft/up.sql | 0 .../down.sql | 0 .../up.sql | 0 .../2022-01-26-184201_index_action_receipt_actions/down.sql | 0 .../2022-01-26-184201_index_action_receipt_actions/up.sql | 0 .../down.sql | 2 ++ .../up.sql | 1 + database/migrations/2022-08-12-123800_ft_nft_indexes/down.sql | 2 ++ database/migrations/2022-08-12-123800_ft_nft_indexes/up.sql | 2 ++ .../down.sql | 1 + .../up.sql | 1 + database/src/migrations/.gitkeep | 0 50 files changed, 10 insertions(+), 1 deletion(-) rename database/{src => }/migrations/00000000000000_diesel_initial_setup/down.sql (100%) rename database/{src => }/migrations/00000000000000_diesel_initial_setup/up.sql (100%) rename database/{src => }/migrations/2020-12-07-153402_initial_schema/down.sql (100%) rename database/{src => }/migrations/2020-12-07-153402_initial_schema/up.sql (100%) rename database/{src => }/migrations/2021-01-14-170424_index-receipt-originated-from-transaction-hash/down.sql (100%) rename database/{src => }/migrations/2021-01-14-170424_index-receipt-originated-from-transaction-hash/up.sql (100%) rename database/{src => }/migrations/2021-01-20-152056_index-transactions-receiver-account-id/down.sql (100%) rename database/{src => }/migrations/2021-01-20-152056_index-transactions-receiver-account-id/up.sql (100%) rename database/{src => }/migrations/2021-03-11-123839_index-action-kind-and-status/down.sql (100%) rename database/{src => }/migrations/2021-03-11-123839_index-action-kind-and-status/up.sql (100%) rename database/{src => }/migrations/2021-04-22-090505_execution_outcomes_replace_chunk_relation_with_shard_id/down.sql (100%) rename database/{src => }/migrations/2021-04-22-090505_execution_outcomes_replace_chunk_relation_with_shard_id/up.sql (100%) rename database/{src => }/migrations/2021-04-28-154439_denormalize_action_receipt_actions/down.sql (100%) rename database/{src => }/migrations/2021-04-28-154439_denormalize_action_receipt_actions/up.sql (100%) rename database/{src => }/migrations/2021-05-06-093622_additional_indexes_for_action_receipt_actions/down.sql (100%) rename database/{src => }/migrations/2021-05-06-093622_additional_indexes_for_action_receipt_actions/up.sql (100%) rename database/{src => }/migrations/2021-05-07-115559_convert_args_base64_to_args_json/down.sql (100%) rename database/{src => }/migrations/2021-05-07-115559_convert_args_base64_to_args_json/up.sql (100%) rename database/{src => }/migrations/2021-05-10-084700_indexer_on_args_field/down.sql (100%) rename database/{src => }/migrations/2021-05-10-084700_indexer_on_args_field/up.sql (100%) rename database/{src => }/migrations/2021-05-27-154211_account_changes_unique_idx/down.sql (100%) rename database/{src => }/migrations/2021-05-27-154211_account_changes_unique_idx/up.sql (100%) rename database/{src => }/migrations/2021-06-02-173100_add_migration_state_change_reason_kind/down.sql (100%) rename database/{src => }/migrations/2021-06-02-173100_add_migration_state_change_reason_kind/up.sql (100%) rename database/{src => }/migrations/2021-08-02-183200_transactions_sorting_idx/down.sql (100%) rename database/{src => }/migrations/2021-08-02-183200_transactions_sorting_idx/up.sql (100%) rename database/{src => }/migrations/2021-08-04-151515_circulating_supply_table/down.sql (100%) rename database/{src => }/migrations/2021-08-04-151515_circulating_supply_table/up.sql (100%) rename database/{src => }/migrations/2021-08-06-123500_account_changes_ordering_column/down.sql (100%) rename database/{src => }/migrations/2021-08-06-123500_account_changes_ordering_column/up.sql (100%) rename database/{src => }/migrations/2021-08-11-163800_account_changes_ordering_idx/down.sql (100%) rename database/{src => }/migrations/2021-08-11-163800_account_changes_ordering_idx/up.sql (100%) rename database/{src => }/migrations/2021-10-04-100000_assets_nft/down.sql (100%) rename database/{src => }/migrations/2021-10-04-100000_assets_nft/up.sql (100%) rename database/{src => }/migrations/2021-10-14-180948_add_resharding_state_change_reason_kind/down.sql (100%) rename database/{src => }/migrations/2021-10-14-180948_add_resharding_state_change_reason_kind/up.sql (100%) rename database/{src => }/migrations/2022-01-12-100000_assets_ft/down.sql (100%) rename database/{src => }/migrations/2022-01-12-100000_assets_ft/up.sql (100%) rename database/{src => }/migrations/2022-01-26-184200_drop_duplicated_index_execution_outcome/down.sql (100%) rename database/{src => }/migrations/2022-01-26-184200_drop_duplicated_index_execution_outcome/up.sql (100%) rename database/{src => }/migrations/2022-01-26-184201_index_action_receipt_actions/down.sql (100%) rename database/{src => }/migrations/2022-01-26-184201_index_action_receipt_actions/up.sql (100%) create mode 100644 database/migrations/2022-06-07-000000_drop_action_receipt_actions_args_amount_index/down.sql create mode 100644 database/migrations/2022-06-07-000000_drop_action_receipt_actions_args_amount_index/up.sql create mode 100644 database/migrations/2022-08-12-123800_ft_nft_indexes/down.sql create mode 100644 database/migrations/2022-08-12-123800_ft_nft_indexes/up.sql create mode 100644 database/migrations/2022-09-23-100000_index_action_receipt_actions_on_timestamp_and_order_index/down.sql create mode 100644 database/migrations/2022-09-23-100000_index_action_receipt_actions_on_timestamp_and_order_index/up.sql delete mode 100644 database/src/migrations/.gitkeep diff --git a/README.md b/README.md index 028c37b7..3f4b5fce 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ $ cargo install diesel_cli --no-default-features --features "postgres" And apply migrations ```bash -$ diesel migration run +$ cd database && diesel migration run ``` ### Compile NEAR Indexer for Explorer diff --git a/database/src/migrations/00000000000000_diesel_initial_setup/down.sql b/database/migrations/00000000000000_diesel_initial_setup/down.sql similarity index 100% rename from database/src/migrations/00000000000000_diesel_initial_setup/down.sql rename to database/migrations/00000000000000_diesel_initial_setup/down.sql diff --git a/database/src/migrations/00000000000000_diesel_initial_setup/up.sql b/database/migrations/00000000000000_diesel_initial_setup/up.sql similarity index 100% rename from database/src/migrations/00000000000000_diesel_initial_setup/up.sql rename to database/migrations/00000000000000_diesel_initial_setup/up.sql diff --git a/database/src/migrations/2020-12-07-153402_initial_schema/down.sql b/database/migrations/2020-12-07-153402_initial_schema/down.sql similarity index 100% rename from database/src/migrations/2020-12-07-153402_initial_schema/down.sql rename to database/migrations/2020-12-07-153402_initial_schema/down.sql diff --git a/database/src/migrations/2020-12-07-153402_initial_schema/up.sql b/database/migrations/2020-12-07-153402_initial_schema/up.sql similarity index 100% rename from database/src/migrations/2020-12-07-153402_initial_schema/up.sql rename to database/migrations/2020-12-07-153402_initial_schema/up.sql diff --git a/database/src/migrations/2021-01-14-170424_index-receipt-originated-from-transaction-hash/down.sql b/database/migrations/2021-01-14-170424_index-receipt-originated-from-transaction-hash/down.sql similarity index 100% rename from database/src/migrations/2021-01-14-170424_index-receipt-originated-from-transaction-hash/down.sql rename to database/migrations/2021-01-14-170424_index-receipt-originated-from-transaction-hash/down.sql diff --git a/database/src/migrations/2021-01-14-170424_index-receipt-originated-from-transaction-hash/up.sql b/database/migrations/2021-01-14-170424_index-receipt-originated-from-transaction-hash/up.sql similarity index 100% rename from database/src/migrations/2021-01-14-170424_index-receipt-originated-from-transaction-hash/up.sql rename to database/migrations/2021-01-14-170424_index-receipt-originated-from-transaction-hash/up.sql diff --git a/database/src/migrations/2021-01-20-152056_index-transactions-receiver-account-id/down.sql b/database/migrations/2021-01-20-152056_index-transactions-receiver-account-id/down.sql similarity index 100% rename from database/src/migrations/2021-01-20-152056_index-transactions-receiver-account-id/down.sql rename to database/migrations/2021-01-20-152056_index-transactions-receiver-account-id/down.sql diff --git a/database/src/migrations/2021-01-20-152056_index-transactions-receiver-account-id/up.sql b/database/migrations/2021-01-20-152056_index-transactions-receiver-account-id/up.sql similarity index 100% rename from database/src/migrations/2021-01-20-152056_index-transactions-receiver-account-id/up.sql rename to database/migrations/2021-01-20-152056_index-transactions-receiver-account-id/up.sql diff --git a/database/src/migrations/2021-03-11-123839_index-action-kind-and-status/down.sql b/database/migrations/2021-03-11-123839_index-action-kind-and-status/down.sql similarity index 100% rename from database/src/migrations/2021-03-11-123839_index-action-kind-and-status/down.sql rename to database/migrations/2021-03-11-123839_index-action-kind-and-status/down.sql diff --git a/database/src/migrations/2021-03-11-123839_index-action-kind-and-status/up.sql b/database/migrations/2021-03-11-123839_index-action-kind-and-status/up.sql similarity index 100% rename from database/src/migrations/2021-03-11-123839_index-action-kind-and-status/up.sql rename to database/migrations/2021-03-11-123839_index-action-kind-and-status/up.sql diff --git a/database/src/migrations/2021-04-22-090505_execution_outcomes_replace_chunk_relation_with_shard_id/down.sql b/database/migrations/2021-04-22-090505_execution_outcomes_replace_chunk_relation_with_shard_id/down.sql similarity index 100% rename from database/src/migrations/2021-04-22-090505_execution_outcomes_replace_chunk_relation_with_shard_id/down.sql rename to database/migrations/2021-04-22-090505_execution_outcomes_replace_chunk_relation_with_shard_id/down.sql diff --git a/database/src/migrations/2021-04-22-090505_execution_outcomes_replace_chunk_relation_with_shard_id/up.sql b/database/migrations/2021-04-22-090505_execution_outcomes_replace_chunk_relation_with_shard_id/up.sql similarity index 100% rename from database/src/migrations/2021-04-22-090505_execution_outcomes_replace_chunk_relation_with_shard_id/up.sql rename to database/migrations/2021-04-22-090505_execution_outcomes_replace_chunk_relation_with_shard_id/up.sql diff --git a/database/src/migrations/2021-04-28-154439_denormalize_action_receipt_actions/down.sql b/database/migrations/2021-04-28-154439_denormalize_action_receipt_actions/down.sql similarity index 100% rename from database/src/migrations/2021-04-28-154439_denormalize_action_receipt_actions/down.sql rename to database/migrations/2021-04-28-154439_denormalize_action_receipt_actions/down.sql diff --git a/database/src/migrations/2021-04-28-154439_denormalize_action_receipt_actions/up.sql b/database/migrations/2021-04-28-154439_denormalize_action_receipt_actions/up.sql similarity index 100% rename from database/src/migrations/2021-04-28-154439_denormalize_action_receipt_actions/up.sql rename to database/migrations/2021-04-28-154439_denormalize_action_receipt_actions/up.sql diff --git a/database/src/migrations/2021-05-06-093622_additional_indexes_for_action_receipt_actions/down.sql b/database/migrations/2021-05-06-093622_additional_indexes_for_action_receipt_actions/down.sql similarity index 100% rename from database/src/migrations/2021-05-06-093622_additional_indexes_for_action_receipt_actions/down.sql rename to database/migrations/2021-05-06-093622_additional_indexes_for_action_receipt_actions/down.sql diff --git a/database/src/migrations/2021-05-06-093622_additional_indexes_for_action_receipt_actions/up.sql b/database/migrations/2021-05-06-093622_additional_indexes_for_action_receipt_actions/up.sql similarity index 100% rename from database/src/migrations/2021-05-06-093622_additional_indexes_for_action_receipt_actions/up.sql rename to database/migrations/2021-05-06-093622_additional_indexes_for_action_receipt_actions/up.sql diff --git a/database/src/migrations/2021-05-07-115559_convert_args_base64_to_args_json/down.sql b/database/migrations/2021-05-07-115559_convert_args_base64_to_args_json/down.sql similarity index 100% rename from database/src/migrations/2021-05-07-115559_convert_args_base64_to_args_json/down.sql rename to database/migrations/2021-05-07-115559_convert_args_base64_to_args_json/down.sql diff --git a/database/src/migrations/2021-05-07-115559_convert_args_base64_to_args_json/up.sql b/database/migrations/2021-05-07-115559_convert_args_base64_to_args_json/up.sql similarity index 100% rename from database/src/migrations/2021-05-07-115559_convert_args_base64_to_args_json/up.sql rename to database/migrations/2021-05-07-115559_convert_args_base64_to_args_json/up.sql diff --git a/database/src/migrations/2021-05-10-084700_indexer_on_args_field/down.sql b/database/migrations/2021-05-10-084700_indexer_on_args_field/down.sql similarity index 100% rename from database/src/migrations/2021-05-10-084700_indexer_on_args_field/down.sql rename to database/migrations/2021-05-10-084700_indexer_on_args_field/down.sql diff --git a/database/src/migrations/2021-05-10-084700_indexer_on_args_field/up.sql b/database/migrations/2021-05-10-084700_indexer_on_args_field/up.sql similarity index 100% rename from database/src/migrations/2021-05-10-084700_indexer_on_args_field/up.sql rename to database/migrations/2021-05-10-084700_indexer_on_args_field/up.sql diff --git a/database/src/migrations/2021-05-27-154211_account_changes_unique_idx/down.sql b/database/migrations/2021-05-27-154211_account_changes_unique_idx/down.sql similarity index 100% rename from database/src/migrations/2021-05-27-154211_account_changes_unique_idx/down.sql rename to database/migrations/2021-05-27-154211_account_changes_unique_idx/down.sql diff --git a/database/src/migrations/2021-05-27-154211_account_changes_unique_idx/up.sql b/database/migrations/2021-05-27-154211_account_changes_unique_idx/up.sql similarity index 100% rename from database/src/migrations/2021-05-27-154211_account_changes_unique_idx/up.sql rename to database/migrations/2021-05-27-154211_account_changes_unique_idx/up.sql diff --git a/database/src/migrations/2021-06-02-173100_add_migration_state_change_reason_kind/down.sql b/database/migrations/2021-06-02-173100_add_migration_state_change_reason_kind/down.sql similarity index 100% rename from database/src/migrations/2021-06-02-173100_add_migration_state_change_reason_kind/down.sql rename to database/migrations/2021-06-02-173100_add_migration_state_change_reason_kind/down.sql diff --git a/database/src/migrations/2021-06-02-173100_add_migration_state_change_reason_kind/up.sql b/database/migrations/2021-06-02-173100_add_migration_state_change_reason_kind/up.sql similarity index 100% rename from database/src/migrations/2021-06-02-173100_add_migration_state_change_reason_kind/up.sql rename to database/migrations/2021-06-02-173100_add_migration_state_change_reason_kind/up.sql diff --git a/database/src/migrations/2021-08-02-183200_transactions_sorting_idx/down.sql b/database/migrations/2021-08-02-183200_transactions_sorting_idx/down.sql similarity index 100% rename from database/src/migrations/2021-08-02-183200_transactions_sorting_idx/down.sql rename to database/migrations/2021-08-02-183200_transactions_sorting_idx/down.sql diff --git a/database/src/migrations/2021-08-02-183200_transactions_sorting_idx/up.sql b/database/migrations/2021-08-02-183200_transactions_sorting_idx/up.sql similarity index 100% rename from database/src/migrations/2021-08-02-183200_transactions_sorting_idx/up.sql rename to database/migrations/2021-08-02-183200_transactions_sorting_idx/up.sql diff --git a/database/src/migrations/2021-08-04-151515_circulating_supply_table/down.sql b/database/migrations/2021-08-04-151515_circulating_supply_table/down.sql similarity index 100% rename from database/src/migrations/2021-08-04-151515_circulating_supply_table/down.sql rename to database/migrations/2021-08-04-151515_circulating_supply_table/down.sql diff --git a/database/src/migrations/2021-08-04-151515_circulating_supply_table/up.sql b/database/migrations/2021-08-04-151515_circulating_supply_table/up.sql similarity index 100% rename from database/src/migrations/2021-08-04-151515_circulating_supply_table/up.sql rename to database/migrations/2021-08-04-151515_circulating_supply_table/up.sql diff --git a/database/src/migrations/2021-08-06-123500_account_changes_ordering_column/down.sql b/database/migrations/2021-08-06-123500_account_changes_ordering_column/down.sql similarity index 100% rename from database/src/migrations/2021-08-06-123500_account_changes_ordering_column/down.sql rename to database/migrations/2021-08-06-123500_account_changes_ordering_column/down.sql diff --git a/database/src/migrations/2021-08-06-123500_account_changes_ordering_column/up.sql b/database/migrations/2021-08-06-123500_account_changes_ordering_column/up.sql similarity index 100% rename from database/src/migrations/2021-08-06-123500_account_changes_ordering_column/up.sql rename to database/migrations/2021-08-06-123500_account_changes_ordering_column/up.sql diff --git a/database/src/migrations/2021-08-11-163800_account_changes_ordering_idx/down.sql b/database/migrations/2021-08-11-163800_account_changes_ordering_idx/down.sql similarity index 100% rename from database/src/migrations/2021-08-11-163800_account_changes_ordering_idx/down.sql rename to database/migrations/2021-08-11-163800_account_changes_ordering_idx/down.sql diff --git a/database/src/migrations/2021-08-11-163800_account_changes_ordering_idx/up.sql b/database/migrations/2021-08-11-163800_account_changes_ordering_idx/up.sql similarity index 100% rename from database/src/migrations/2021-08-11-163800_account_changes_ordering_idx/up.sql rename to database/migrations/2021-08-11-163800_account_changes_ordering_idx/up.sql diff --git a/database/src/migrations/2021-10-04-100000_assets_nft/down.sql b/database/migrations/2021-10-04-100000_assets_nft/down.sql similarity index 100% rename from database/src/migrations/2021-10-04-100000_assets_nft/down.sql rename to database/migrations/2021-10-04-100000_assets_nft/down.sql diff --git a/database/src/migrations/2021-10-04-100000_assets_nft/up.sql b/database/migrations/2021-10-04-100000_assets_nft/up.sql similarity index 100% rename from database/src/migrations/2021-10-04-100000_assets_nft/up.sql rename to database/migrations/2021-10-04-100000_assets_nft/up.sql diff --git a/database/src/migrations/2021-10-14-180948_add_resharding_state_change_reason_kind/down.sql b/database/migrations/2021-10-14-180948_add_resharding_state_change_reason_kind/down.sql similarity index 100% rename from database/src/migrations/2021-10-14-180948_add_resharding_state_change_reason_kind/down.sql rename to database/migrations/2021-10-14-180948_add_resharding_state_change_reason_kind/down.sql diff --git a/database/src/migrations/2021-10-14-180948_add_resharding_state_change_reason_kind/up.sql b/database/migrations/2021-10-14-180948_add_resharding_state_change_reason_kind/up.sql similarity index 100% rename from database/src/migrations/2021-10-14-180948_add_resharding_state_change_reason_kind/up.sql rename to database/migrations/2021-10-14-180948_add_resharding_state_change_reason_kind/up.sql diff --git a/database/src/migrations/2022-01-12-100000_assets_ft/down.sql b/database/migrations/2022-01-12-100000_assets_ft/down.sql similarity index 100% rename from database/src/migrations/2022-01-12-100000_assets_ft/down.sql rename to database/migrations/2022-01-12-100000_assets_ft/down.sql diff --git a/database/src/migrations/2022-01-12-100000_assets_ft/up.sql b/database/migrations/2022-01-12-100000_assets_ft/up.sql similarity index 100% rename from database/src/migrations/2022-01-12-100000_assets_ft/up.sql rename to database/migrations/2022-01-12-100000_assets_ft/up.sql diff --git a/database/src/migrations/2022-01-26-184200_drop_duplicated_index_execution_outcome/down.sql b/database/migrations/2022-01-26-184200_drop_duplicated_index_execution_outcome/down.sql similarity index 100% rename from database/src/migrations/2022-01-26-184200_drop_duplicated_index_execution_outcome/down.sql rename to database/migrations/2022-01-26-184200_drop_duplicated_index_execution_outcome/down.sql diff --git a/database/src/migrations/2022-01-26-184200_drop_duplicated_index_execution_outcome/up.sql b/database/migrations/2022-01-26-184200_drop_duplicated_index_execution_outcome/up.sql similarity index 100% rename from database/src/migrations/2022-01-26-184200_drop_duplicated_index_execution_outcome/up.sql rename to database/migrations/2022-01-26-184200_drop_duplicated_index_execution_outcome/up.sql diff --git a/database/src/migrations/2022-01-26-184201_index_action_receipt_actions/down.sql b/database/migrations/2022-01-26-184201_index_action_receipt_actions/down.sql similarity index 100% rename from database/src/migrations/2022-01-26-184201_index_action_receipt_actions/down.sql rename to database/migrations/2022-01-26-184201_index_action_receipt_actions/down.sql diff --git a/database/src/migrations/2022-01-26-184201_index_action_receipt_actions/up.sql b/database/migrations/2022-01-26-184201_index_action_receipt_actions/up.sql similarity index 100% rename from database/src/migrations/2022-01-26-184201_index_action_receipt_actions/up.sql rename to database/migrations/2022-01-26-184201_index_action_receipt_actions/up.sql diff --git a/database/migrations/2022-06-07-000000_drop_action_receipt_actions_args_amount_index/down.sql b/database/migrations/2022-06-07-000000_drop_action_receipt_actions_args_amount_index/down.sql new file mode 100644 index 00000000..9c3148dc --- /dev/null +++ b/database/migrations/2022-06-07-000000_drop_action_receipt_actions_args_amount_index/down.sql @@ -0,0 +1,2 @@ +CREATE INDEX action_receipt_actions_args_amount_idx ON action_receipt_actions((args->'args_json'->>'amount')) + WHERE action_receipt_actions.action_kind = 'FUNCTION_CALL' AND (action_receipt_actions.args->>'args_json') IS NOT NULL; diff --git a/database/migrations/2022-06-07-000000_drop_action_receipt_actions_args_amount_index/up.sql b/database/migrations/2022-06-07-000000_drop_action_receipt_actions_args_amount_index/up.sql new file mode 100644 index 00000000..34b5eb62 --- /dev/null +++ b/database/migrations/2022-06-07-000000_drop_action_receipt_actions_args_amount_index/up.sql @@ -0,0 +1 @@ +DROP INDEX action_receipt_actions_args_amount_idx; diff --git a/database/migrations/2022-08-12-123800_ft_nft_indexes/down.sql b/database/migrations/2022-08-12-123800_ft_nft_indexes/down.sql new file mode 100644 index 00000000..38044a6f --- /dev/null +++ b/database/migrations/2022-08-12-123800_ft_nft_indexes/down.sql @@ -0,0 +1,2 @@ +DROP INDEX assets__ft_contract_id_idx; +DROP INDEX assets__nft_contract_id_idx; diff --git a/database/migrations/2022-08-12-123800_ft_nft_indexes/up.sql b/database/migrations/2022-08-12-123800_ft_nft_indexes/up.sql new file mode 100644 index 00000000..779b539f --- /dev/null +++ b/database/migrations/2022-08-12-123800_ft_nft_indexes/up.sql @@ -0,0 +1,2 @@ +CREATE INDEX assets__ft_contract_id_idx ON assets__fungible_token_events (emitted_by_contract_account_id); +CREATE INDEX assets__nft_contract_id_idx ON assets__non_fungible_token_events (emitted_by_contract_account_id); diff --git a/database/migrations/2022-09-23-100000_index_action_receipt_actions_on_timestamp_and_order_index/down.sql b/database/migrations/2022-09-23-100000_index_action_receipt_actions_on_timestamp_and_order_index/down.sql new file mode 100644 index 00000000..1613c049 --- /dev/null +++ b/database/migrations/2022-09-23-100000_index_action_receipt_actions_on_timestamp_and_order_index/down.sql @@ -0,0 +1 @@ +DROP INDEX action_receipt_actions_receipt_included_in_block_timestamp_index_in_action_receipt_idx; diff --git a/database/migrations/2022-09-23-100000_index_action_receipt_actions_on_timestamp_and_order_index/up.sql b/database/migrations/2022-09-23-100000_index_action_receipt_actions_on_timestamp_and_order_index/up.sql new file mode 100644 index 00000000..131c0154 --- /dev/null +++ b/database/migrations/2022-09-23-100000_index_action_receipt_actions_on_timestamp_and_order_index/up.sql @@ -0,0 +1 @@ +CREATE INDEX action_receipt_actions_receipt_included_in_block_timestamp_index_in_action_receipt_idx ON action_receipt_actions(receipt_included_in_block_timestamp, index_in_action_receipt); diff --git a/database/src/migrations/.gitkeep b/database/src/migrations/.gitkeep deleted file mode 100644 index e69de29b..00000000 From faa882a3d98d679abbcfe0160c7ddf0a5ee75a13 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Wed, 30 Nov 2022 08:59:40 +1300 Subject: [PATCH 21/42] refactor: Use default AWS SDK credentials provider (#14) --- README.md | 4 ++-- indexer/CHANGELOG.md | 10 ++++++++++ indexer/src/configs.rs | 30 +----------------------------- 3 files changed, 13 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 3f4b5fce..af5abd0b 100644 --- a/README.md +++ b/README.md @@ -52,8 +52,8 @@ You need to provide credentials via `.env` file for: ``` - AWS S3 (permission to read from buckets): ```bash - $ echo "LAKE_AWS_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" >> .env - $ echo "LAKE_AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + $ echo "AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE" >> .env + $ echo "AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" >> .env ``` Then you need to apply migrations to create necessary database structure. For this you'll need `diesel-cli`, you can install it like so: diff --git a/indexer/CHANGELOG.md b/indexer/CHANGELOG.md index 8578d4ec..3dacbb6e 100644 --- a/indexer/CHANGELOG.md +++ b/indexer/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## 0.11.0 + +* `indexer-explorer-lake` now uses the default AWS credentials provider. Credentials can no longer be set via command line arguments and environment variables need to be updated as follows: +```diff +- "LAKE_AWS_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" +- "LAKE_AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" ++ "AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE" ++ "AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" +``` + ## 0.10.3 * Extract database logic to library crate diff --git a/indexer/src/configs.rs b/indexer/src/configs.rs index 35e5a666..489b90fe 100644 --- a/indexer/src/configs.rs +++ b/indexer/src/configs.rs @@ -22,12 +22,6 @@ pub(crate) struct Opts { /// Connection string to connect to the PostgreSQL Database to fetch AlertRules from #[clap(long, env)] pub database_url: String, - /// AWS Access Key with the rights to read from AWS S3 - #[clap(long, env)] - pub lake_aws_access_key: String, - /// AWS Secret Access Key with the rights to read from AWS S3 - #[clap(long, env)] - pub lake_aws_secret_access_key: String, /// Enabled Indexer for Explorer debug level of logs #[clap(long)] pub debug: bool, @@ -69,26 +63,6 @@ impl Opts { } } - // Creates AWS Credentials for NEAR Lake - fn lake_credentials(&self) -> aws_types::credentials::SharedCredentialsProvider { - let provider = aws_types::Credentials::new( - self.lake_aws_access_key.clone(), - self.lake_aws_secret_access_key.clone(), - None, - None, - "alertexer_lake", - ); - aws_types::credentials::SharedCredentialsProvider::new(provider) - } - - /// Creates AWS Shared Config for NEAR Lake - pub fn lake_aws_sdk_config(&self) -> aws_types::sdk_config::SdkConfig { - aws_types::sdk_config::SdkConfig::builder() - .credentials_provider(self.lake_credentials()) - .region(aws_types::region::Region::new("eu-central-1")) - .build() - } - pub fn rpc_url(&self) -> &str { match self.chain_id { ChainId::Mainnet(_) => "https://rpc.mainnet.near.org", @@ -99,9 +73,7 @@ impl Opts { impl Opts { pub async fn to_lake_config(&self) -> near_lake_framework::LakeConfig { - let s3_config = aws_sdk_s3::config::Builder::from(&self.lake_aws_sdk_config()).build(); - - let config_builder = near_lake_framework::LakeConfigBuilder::default().s3_config(s3_config); + let config_builder = near_lake_framework::LakeConfigBuilder::default(); match &self.chain_id { ChainId::Mainnet(_) => config_builder From 34a6112d9da799486a8113ce37fc2e97ac234446 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Wed, 30 Nov 2022 09:14:58 +1300 Subject: [PATCH 22/42] chore: Update `Cargo.toml` version to match previous code changes (#17) --- Cargo.lock | 2 +- indexer/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a9270c27..b1cb6b51 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1626,7 +1626,7 @@ dependencies = [ [[package]] name = "indexer-explorer-lake" -version = "0.10.3" +version = "0.11.0" dependencies = [ "actix", "actix-rt", diff --git a/indexer/Cargo.toml b/indexer/Cargo.toml index 3e569973..46ddc8d6 100644 --- a/indexer/Cargo.toml +++ b/indexer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "indexer-explorer-lake" -version = "0.10.3" +version = "0.11.0" authors = ["Near Inc "] edition = "2021" rust-version = "1.62.1" From 720be80ea6e9e4e8dc12a1d0a697e4f19aeac173 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Thu, 1 Dec 2022 08:04:02 +1300 Subject: [PATCH 23/42] feat: Enable JSON logging via environment variable (#13) * feat: Enable JSON logging via environment variable * chore: Update CHANGELOG & Cargo version --- Cargo.lock | 2 +- circulating-supply/src/main.rs | 11 ++++++++--- indexer/CHANGELOG.md | 4 ++++ indexer/Cargo.toml | 2 +- indexer/src/configs.rs | 11 ++++++++--- 5 files changed, 22 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b1cb6b51..8d0c5923 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1626,7 +1626,7 @@ dependencies = [ [[package]] name = "indexer-explorer-lake" -version = "0.11.0" +version = "0.11.1" dependencies = [ "actix", "actix-rt", diff --git a/circulating-supply/src/main.rs b/circulating-supply/src/main.rs index 7e7402b0..3fa68423 100644 --- a/circulating-supply/src/main.rs +++ b/circulating-supply/src/main.rs @@ -24,9 +24,14 @@ const CIRCULATING_SUPPLY: &str = "circulating_supply"; async fn main() { dotenv::dotenv().ok(); - tracing_subscriber::fmt::Subscriber::builder() - .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) - .init(); + let subscriber = tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()); + + if std::env::var("ENABLE_JSON_LOGS").is_ok() { + subscriber.json().init() + } else { + subscriber.compact().init() + } let pool = models::establish_connection( &std::env::var("DATABASE_URL") diff --git a/indexer/CHANGELOG.md b/indexer/CHANGELOG.md index 3dacbb6e..cc790e9a 100644 --- a/indexer/CHANGELOG.md +++ b/indexer/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.11.1 + +* Add capability to output logs in JSON format + ## 0.11.0 * `indexer-explorer-lake` now uses the default AWS credentials provider. Credentials can no longer be set via command line arguments and environment variables need to be updated as follows: diff --git a/indexer/Cargo.toml b/indexer/Cargo.toml index 46ddc8d6..82bb1d68 100644 --- a/indexer/Cargo.toml +++ b/indexer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "indexer-explorer-lake" -version = "0.11.0" +version = "0.11.1" authors = ["Near Inc "] edition = "2021" rust-version = "1.62.1" diff --git a/indexer/src/configs.rs b/indexer/src/configs.rs index 489b90fe..2f99184a 100644 --- a/indexer/src/configs.rs +++ b/indexer/src/configs.rs @@ -140,10 +140,15 @@ pub(crate) fn init_tracing(debug: bool) -> anyhow::Result<()> { } } - tracing_subscriber::fmt::Subscriber::builder() + let subscriber = tracing_subscriber::fmt::Subscriber::builder() .with_env_filter(env_filter) - .with_writer(std::io::stderr) - .init(); + .with_writer(std::io::stderr); + + if std::env::var("ENABLE_JSON_LOGS").is_ok() { + subscriber.json().init(); + } else { + subscriber.compact().init(); + } Ok(()) } From 4d8c8f7270da4f12ae185ff33745c6ba59488793 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Fri, 2 Dec 2022 08:35:11 +1300 Subject: [PATCH 24/42] feat: Expose block metrics from Indexer over http (#15) * refactor: Move indexer stream to separate thread * feat: Standup HTTP server * fix: Ensure lake `ReceiverStream` finishes when process exits * feat: Serve block metrics via http * refactor: Rename `HTTP_PORT` -> `PORT` * chore: Update `CHANGELOG.md` and Cargo version * refactor: Rename block metrics for clarity --- Cargo.lock | 359 ++++++++++++++++++++++++++++++++++++++++- indexer/CHANGELOG.md | 4 + indexer/Cargo.toml | 5 +- indexer/src/main.rs | 56 +++---- indexer/src/metrics.rs | 63 ++++++++ 5 files changed, 456 insertions(+), 31 deletions(-) create mode 100644 indexer/src/metrics.rs diff --git a/Cargo.lock b/Cargo.lock index 8d0c5923..fc633890 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -32,6 +32,23 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "actix-codec" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57a7559404a7f3573127aab53c08ce37a6c6a315c374a31070f3c91cd1b4a7fe" +dependencies = [ + "bitflags", + "bytes", + "futures-core", + "futures-sink", + "log", + "memchr", + "pin-project-lite", + "tokio", + "tokio-util", +] + [[package]] name = "actix-diesel" version = "0.3.1-alpha.0" @@ -46,6 +63,43 @@ dependencies = [ "r2d2", ] +[[package]] +name = "actix-http" +version = "3.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c83abf9903e1f0ad9973cc4f7b9767fd5a03a583f51a5b7a339e07987cd2724" +dependencies = [ + "actix-codec", + "actix-rt", + "actix-service", + "actix-utils", + "ahash 0.7.6", + "base64 0.13.0", + "bitflags", + "brotli", + "bytes", + "bytestring", + "derive_more", + "encoding_rs", + "flate2", + "futures-core", + "h2", + "http", + "httparse", + "httpdate", + "itoa", + "language-tags", + "local-channel", + "mime", + "percent-encoding", + "pin-project-lite", + "rand 0.8.5", + "sha1", + "smallvec", + "tracing", + "zstd", +] + [[package]] name = "actix-macros" version = "0.2.3" @@ -56,6 +110,19 @@ dependencies = [ "syn", ] +[[package]] +name = "actix-router" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66ff4d247d2b160861fa2866457e85706833527840e4133f8f49aa423a38799" +dependencies = [ + "bytestring", + "http", + "regex", + "serde", + "tracing", +] + [[package]] name = "actix-rt" version = "2.7.0" @@ -67,6 +134,97 @@ dependencies = [ "tokio", ] +[[package]] +name = "actix-server" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0da34f8e659ea1b077bb4637948b815cd3768ad5a188fdcd74ff4d84240cd824" +dependencies = [ + "actix-rt", + "actix-service", + "actix-utils", + "futures-core", + "futures-util", + "mio", + "num_cpus", + "socket2", + "tokio", + "tracing", +] + +[[package]] +name = "actix-service" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b894941f818cfdc7ccc4b9e60fa7e53b5042a2e8567270f9147d5591893373a" +dependencies = [ + "futures-core", + "paste", + "pin-project-lite", +] + +[[package]] +name = "actix-utils" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88a1dcdff1466e3c2488e1cb5c36a71822750ad43839937f85d2f4d9f8b705d8" +dependencies = [ + "local-waker", + "pin-project-lite", +] + +[[package]] +name = "actix-web" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4e5ebffd51d50df56a3ae0de0e59487340ca456f05dd0b90c0a7a6dd6a74d31" +dependencies = [ + "actix-codec", + "actix-http", + "actix-macros", + "actix-router", + "actix-rt", + "actix-server", + "actix-service", + "actix-utils", + "actix-web-codegen", + "ahash 0.7.6", + "bytes", + "bytestring", + "cfg-if 1.0.0", + "cookie", + "derive_more", + "encoding_rs", + "futures-core", + "futures-util", + "itoa", + "language-tags", + "log", + "mime", + "once_cell", + "pin-project-lite", + "regex", + "serde", + "serde_json", + "serde_urlencoded", + "smallvec", + "socket2", + "time 0.3.9", + "url", +] + +[[package]] +name = "actix-web-codegen" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa9362663c8643d67b2d5eafba49e4cb2c8a053a29ed00a0bea121f17c76b13" +dependencies = [ + "actix-router", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "actix_derive" version = "0.6.0" @@ -78,6 +236,12 @@ dependencies = [ "syn", ] +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "ahash" version = "0.4.7" @@ -104,6 +268,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "ansi_term" version = "0.12.1" @@ -703,6 +882,27 @@ dependencies = [ "syn", ] +[[package]] +name = "brotli" +version = "3.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ad2d4653bf5ca36ae797b1f4bb4dbddb60ce49ca4aed8a2ce4829f60425b80" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bs58" version = "0.4.0" @@ -729,9 +929,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.1.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" +checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" [[package]] name = "bytes-utils" @@ -749,6 +949,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c58ec36aac5066d5ca17df51b3e70279f5670a72102f5752cb7e7c856adfc70" +[[package]] +name = "bytestring" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7f83e57d9154148e355404702e2694463241880b939570d7c97c014da7a69a1" +dependencies = [ + "bytes", +] + [[package]] name = "c2-chacha" version = "0.3.3" @@ -797,6 +1006,9 @@ name = "cc" version = "1.0.73" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" +dependencies = [ + "jobserver", +] [[package]] name = "cfg-if" @@ -899,6 +1111,17 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +[[package]] +name = "cookie" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "344adc371239ef32293cb1c4fe519592fcf21206c79c02854320afcdf3ab4917" +dependencies = [ + "percent-encoding", + "time 0.3.9", + "version_check", +] + [[package]] name = "core-foundation" version = "0.9.3" @@ -1274,6 +1497,16 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "flate2" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1626,10 +1859,11 @@ dependencies = [ [[package]] name = "indexer-explorer-lake" -version = "0.11.1" +version = "0.11.2" dependencies = [ "actix", "actix-rt", + "actix-web", "anyhow", "aws-sdk-s3", "aws-types", @@ -1641,10 +1875,12 @@ dependencies = [ "explorer-database", "futures", "itertools", + "lazy_static", "near-jsonrpc-client", "near-lake-framework", "near-sdk", "openssl-probe", + "prometheus", "r2d2", "tokio", "tokio-stream", @@ -1693,6 +1929,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" +[[package]] +name = "jobserver" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.57" @@ -1708,6 +1953,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67c21572b4949434e4fc1e1978b99c5f77064153c59d998bf13ecd96fb5ecba7" +[[package]] +name = "language-tags" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388" + [[package]] name = "lazy_static" version = "1.4.0" @@ -1720,6 +1971,24 @@ version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5916d2ae698f6de9bfb891ad7a8d65c09d232dc58cc4ac433c7da3b2fd84bc2b" +[[package]] +name = "local-channel" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f303ec0e94c6c54447f84f3b0ef7af769858a9c4ef56ef2a986d3dcd4c3fc9c" +dependencies = [ + "futures-core", + "futures-sink", + "futures-util", + "local-waker", +] + +[[package]] +name = "local-waker" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e34f76eb3611940e0e7d53a9aaa4e6a3151f69541a282fd0dad5571420c53ff1" + [[package]] name = "lock_api" version = "0.4.7" @@ -1781,6 +2050,15 @@ version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +[[package]] +name = "miniz_oxide" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +dependencies = [ + "adler", +] + [[package]] name = "mio" version = "0.8.3" @@ -2391,6 +2669,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "paste" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1de2e551fb905ac83f73f7aedf2f0cb4a0da7e35efa24a202a936269f1f18e1" + [[package]] name = "percent-encoding" version = "2.1.0" @@ -2513,6 +2797,27 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "prometheus" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c" +dependencies = [ + "cfg-if 1.0.0", + "fnv", + "lazy_static", + "memchr", + "parking_lot", + "protobuf", + "thiserror", +] + +[[package]] +name = "protobuf" +version = "2.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" + [[package]] name = "quote" version = "1.0.18" @@ -2875,6 +3180,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "006769ba83e921b3085caa8334186b00cf92b4cb1a6cf4632fbccc8eff5c7549" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest 0.10.3", +] + [[package]] name = "sha2" version = "0.9.9" @@ -3118,10 +3434,18 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2702e08a7a860f005826c6815dcac101b19b5eb330c27fe4a5928fec1d20ddd" dependencies = [ + "itoa", "libc", "num_threads", + "time-macros", ] +[[package]] +name = "time-macros" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" + [[package]] name = "tinyvec" version = "1.6.0" @@ -3671,3 +3995,32 @@ dependencies = [ "syn", "synstructure", ] + +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.3+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44ccf97612ac95f3ccb89b2d7346b345e52f1c3019be4984f0455fb4ba991f8a" +dependencies = [ + "cc", + "libc", +] diff --git a/indexer/CHANGELOG.md b/indexer/CHANGELOG.md index cc790e9a..e26f0e75 100644 --- a/indexer/CHANGELOG.md +++ b/indexer/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.11.2 + +* Expose prometheus metrics via HTTP server + ## 0.11.1 * Add capability to output logs in JSON format diff --git a/indexer/Cargo.toml b/indexer/Cargo.toml index 82bb1d68..b6fdc2a5 100644 --- a/indexer/Cargo.toml +++ b/indexer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "indexer-explorer-lake" -version = "0.11.1" +version = "0.11.2" authors = ["Near Inc "] edition = "2021" rust-version = "1.62.1" @@ -8,6 +8,7 @@ rust-version = "1.62.1" [dependencies] actix = "0.13.0" actix-rt = "2.2.0" +actix-web = "=4.0.1" anyhow = "1.0.51" aws-types = "0.13.0" aws-sdk-s3 = "0.13.0" @@ -18,9 +19,11 @@ clap = { version = "3.1.6", features = ["color", "derive", "env"] } dotenv = "0.15.0" futures = "0.3.5" itertools = "0.10.3" +lazy_static = "^1.4" # syn version conflict, replace with crates.io version once released near-sdk = { git = "https://github.com/near/near-sdk-rs", rev="03487c184d37b0382dd9bd41c57466acad58fc1f" } openssl-probe = { version = "0.1.2" } +prometheus = "0.13.0" r2d2 = "0.8.8" tokio = { version = "1.1", features = ["sync", "time"] } tokio-stream = { version = "0.1" } diff --git a/indexer/src/main.rs b/indexer/src/main.rs index b0d495fd..0775978e 100644 --- a/indexer/src/main.rs +++ b/indexer/src/main.rs @@ -11,6 +11,7 @@ use explorer_database::{adapters, models, receipts_cache}; use crate::configs::Opts; mod configs; +mod metrics; // Categories for logging const INDEXER_FOR_EXPLORER: &str = "indexer_for_explorer"; @@ -21,6 +22,9 @@ async fn handle_message( strict_mode: bool, receipts_cache: receipts_cache::ReceiptsCache, ) -> anyhow::Result<()> { + metrics::BLOCK_COUNT.inc(); + metrics::LATEST_BLOCK_HEIGHT.set(streamer_message.block.header.height.try_into().unwrap()); + debug!( target: INDEXER_FOR_EXPLORER, "ReceiptsCache #{} \n {:#?}", streamer_message.block.header.height, &receipts_cache @@ -153,34 +157,32 @@ async fn main() -> anyhow::Result<()> { std::sync::Arc::new(Mutex::new(SizedCache::with_size(100_000))); let config: near_lake_framework::LakeConfig = opts.to_lake_config().await; - let (sender, stream) = near_lake_framework::streamer(config); + let (_, stream) = near_lake_framework::streamer(config); - tracing::info!( - target: INDEXER_FOR_EXPLORER, - "Starting Indexer for Explorer (lake)...", - ); - let mut handlers = tokio_stream::wrappers::ReceiverStream::new(stream) - .map(|streamer_message| { - info!( - target: crate::INDEXER_FOR_EXPLORER, - "Block height {}", &streamer_message.block.header.height - ); - handle_message( - &pool, - streamer_message, - strict_mode, - std::sync::Arc::clone(&receipts_cache), - ) - }) - .buffer_unordered(1usize); + tokio::spawn(async move { + tracing::info!( + target: INDEXER_FOR_EXPLORER, + "Starting Indexer for Explorer (lake)...", + ); + let mut handlers = tokio_stream::wrappers::ReceiverStream::new(stream) + .map(|streamer_message| { + info!( + target: crate::INDEXER_FOR_EXPLORER, + "Block height {}", &streamer_message.block.header.height + ); + handle_message( + &pool, + streamer_message, + strict_mode, + std::sync::Arc::clone(&receipts_cache), + ) + }) + .buffer_unordered(1usize); - while let Some(_handle_message) = handlers.next().await {} - drop(handlers); // close the channel so the sender will stop + while let Some(_handle_message) = handlers.next().await {} + }); - // propagate errors from the sender - match sender.await { - Ok(Ok(())) => Ok(()), - Ok(Err(e)) => Err(e), - Err(e) => Err(anyhow::Error::from(e)), // JoinError - } + metrics::init_server().await?; + + Ok(()) } diff --git a/indexer/src/metrics.rs b/indexer/src/metrics.rs new file mode 100644 index 00000000..d61eeeab --- /dev/null +++ b/indexer/src/metrics.rs @@ -0,0 +1,63 @@ +use actix_web::{get, App, HttpServer, Responder}; +use lazy_static::lazy_static; +use prometheus::{Encoder, IntCounter, IntGauge, Opts}; +use tracing::info; + +lazy_static! { + pub(crate) static ref LATEST_BLOCK_HEIGHT: IntGauge = try_create_int_gauge( + "indexer_explorer_lake_latest_block_height", + "Height of last processed block" + ) + .unwrap(); + pub(crate) static ref BLOCK_COUNT: IntCounter = try_create_int_counter( + "indexer_explorer_lake_block_count", + "Number of indexed blocks" + ) + .unwrap(); +} + +fn try_create_int_gauge(name: &str, help: &str) -> prometheus::Result { + let opts = Opts::new(name, help); + let gauge = IntGauge::with_opts(opts)?; + prometheus::register(Box::new(gauge.clone()))?; + Ok(gauge) +} + +fn try_create_int_counter(name: &str, help: &str) -> prometheus::Result { + let opts = Opts::new(name, help); + let counter = IntCounter::with_opts(opts)?; + prometheus::register(Box::new(counter.clone()))?; + Ok(counter) +} + +#[get("/metrics")] +async fn get_metrics() -> impl Responder { + let mut buffer = Vec::::new(); + let encoder = prometheus::TextEncoder::new(); + loop { + match encoder.encode(&prometheus::gather(), &mut buffer) { + Ok(_) => break, + Err(err) => { + eprintln!("{:?}", err); + } + } + } + String::from_utf8(buffer.clone()).unwrap() +} + +pub(crate) async fn init_server() -> Result<(), std::io::Error> { + let port: u16 = std::env::var("PORT") + .unwrap_or_else(|_| String::from("3030")) + .parse() + .expect("Unable to parse `PORT`"); + + info!( + target: crate::INDEXER_FOR_EXPLORER, + "Starting metrics server on http://0.0.0.0:{port}" + ); + + HttpServer::new(|| App::new().service(get_metrics)) + .bind(("0.0.0.0", port))? + .run() + .await +} From e454ea4410aa9846a885e9f97463f34f3019b090 Mon Sep 17 00:00:00 2001 From: ecp88 <109925246+ecp88@users.noreply.github.com> Date: Tue, 6 Dec 2022 22:43:01 -0600 Subject: [PATCH 25/42] DPLT-632: Add Dockerfile files for circulating-supply and indexer (#18) --- circulating-supply/Cargo.toml | 1 + circulating-supply/Dockerfile | 15 +++++++++++++++ indexer/Cargo.toml | 2 +- indexer/Dockerfile | 15 +++++++++++++++ 4 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 circulating-supply/Dockerfile create mode 100644 indexer/Dockerfile diff --git a/circulating-supply/Cargo.toml b/circulating-supply/Cargo.toml index 64009ae7..9648034d 100644 --- a/circulating-supply/Cargo.toml +++ b/circulating-supply/Cargo.toml @@ -1,5 +1,6 @@ [package] name = "circulating-supply" +rust-version = "1.64" version = "0.1.0" edition = "2021" diff --git a/circulating-supply/Dockerfile b/circulating-supply/Dockerfile new file mode 100644 index 00000000..9851ebbe --- /dev/null +++ b/circulating-supply/Dockerfile @@ -0,0 +1,15 @@ +FROM rust:1.64 AS builder +WORKDIR /tmp/ +COPY Cargo.toml Cargo.lock ./ +COPY database database +# Copy 'indexer' because is member of the workspace but is not compiled or added to the +# output image. +COPY indexer indexer +COPY circulating-supply circulating-supply +RUN cargo build -p circulating-supply --release + +FROM ubuntu:20.04 +RUN apt update && apt install -yy openssl ca-certificates +RUN apt-get install libpq5 -y +COPY --from=builder /tmp/target/release/circulating-supply . +ENTRYPOINT [ "./circulating-supply" ] diff --git a/indexer/Cargo.toml b/indexer/Cargo.toml index b6fdc2a5..5f8fae54 100644 --- a/indexer/Cargo.toml +++ b/indexer/Cargo.toml @@ -3,7 +3,7 @@ name = "indexer-explorer-lake" version = "0.11.2" authors = ["Near Inc "] edition = "2021" -rust-version = "1.62.1" +rust-version = "1.64" [dependencies] actix = "0.13.0" diff --git a/indexer/Dockerfile b/indexer/Dockerfile new file mode 100644 index 00000000..c1fef49e --- /dev/null +++ b/indexer/Dockerfile @@ -0,0 +1,15 @@ +FROM rust:1.64 AS builder +WORKDIR /tmp/ +COPY Cargo.toml Cargo.lock ./ +COPY database database +COPY indexer indexer +# Copy 'circulating-supply' because is member of the workspace but is not compiled or added to the +# output image. +COPY circulating-supply circulating-supply +RUN cargo build -p indexer-explorer-lake --release + +FROM ubuntu:20.04 +RUN apt update && apt install -yy openssl ca-certificates +RUN apt-get install libpq5 -y +COPY --from=builder /tmp/target/release/indexer-explorer-lake . +ENTRYPOINT [ "./indexer-explorer-lake" ] From 7161b77e90de16d76dc21dc21459f5b5f9ff9464 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Fri, 9 Dec 2022 08:25:36 +1300 Subject: [PATCH 26/42] refactor: Metrics server config/errors (#20) * refactor: Parse `port` via CLI and env rather than just env * refactor: Return `anyhow::Result` from `metrics::init_server` --- indexer/src/configs.rs | 3 +++ indexer/src/main.rs | 4 +--- indexer/src/metrics.rs | 8 ++------ 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/indexer/src/configs.rs b/indexer/src/configs.rs index 2f99184a..aa1d5ba5 100644 --- a/indexer/src/configs.rs +++ b/indexer/src/configs.rs @@ -31,6 +31,9 @@ pub(crate) struct Opts { /// Sets the concurrency for indexing. Note: concurrency (set to 2+) may lead to warnings due to tight constraints between transactions and receipts (those will get resolved eventually, but unless it is the second pass of indexing, concurrency won't help at the moment). #[clap(long, default_value = "1")] pub concurrency: std::num::NonZeroU16, + /// Port to enable metrics/health service + #[clap(long, short, env, default_value_t = 3030)] + pub port: u16, /// Chain ID: testnet or mainnet #[clap(subcommand)] pub chain_id: ChainId, diff --git a/indexer/src/main.rs b/indexer/src/main.rs index 0775978e..d0b13b28 100644 --- a/indexer/src/main.rs +++ b/indexer/src/main.rs @@ -182,7 +182,5 @@ async fn main() -> anyhow::Result<()> { while let Some(_handle_message) = handlers.next().await {} }); - metrics::init_server().await?; - - Ok(()) + metrics::init_server(opts.port).await } diff --git a/indexer/src/metrics.rs b/indexer/src/metrics.rs index d61eeeab..2a309419 100644 --- a/indexer/src/metrics.rs +++ b/indexer/src/metrics.rs @@ -45,12 +45,7 @@ async fn get_metrics() -> impl Responder { String::from_utf8(buffer.clone()).unwrap() } -pub(crate) async fn init_server() -> Result<(), std::io::Error> { - let port: u16 = std::env::var("PORT") - .unwrap_or_else(|_| String::from("3030")) - .parse() - .expect("Unable to parse `PORT`"); - +pub(crate) async fn init_server(port: u16) -> anyhow::Result<()> { info!( target: crate::INDEXER_FOR_EXPLORER, "Starting metrics server on http://0.0.0.0:{port}" @@ -60,4 +55,5 @@ pub(crate) async fn init_server() -> Result<(), std::io::Error> { .bind(("0.0.0.0", port))? .run() .await + .map_err(|e| anyhow::anyhow!("Error starting metrics server: {}", e)) } From 3c82a67ce1e15ad7469181cc33cbef3aec9249fa Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Mon, 12 Dec 2022 09:02:21 +1300 Subject: [PATCH 27/42] chore: Drop `-lake` suffix in Cargo name --- Cargo.lock | 2 +- README.md | 2 +- indexer/Cargo.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0bb96584..092adb31 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1858,7 +1858,7 @@ dependencies = [ ] [[package]] -name = "indexer-explorer-lake" +name = "indexer-explorer" version = "0.11.0" dependencies = [ "actix", diff --git a/README.md b/README.md index 46ecdfdd..1d04b13f 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ By default NEAR Indexer for Explorer processes only a single block at a time. Yo So final command to run NEAR Indexer for Explorer can look like: ```bash -$ ./target/release/indexer-explorer-lake \ +$ ./target/release/indexer-explorer \ --non-strict-mode \ --concurrency 1 \ from-latest diff --git a/indexer/Cargo.toml b/indexer/Cargo.toml index b3f90adf..37b58ed6 100644 --- a/indexer/Cargo.toml +++ b/indexer/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "indexer-explorer-lake" +name = "indexer-explorer" version = "0.11.0" authors = ["Near Inc "] edition = "2021" From 0477f96ae5ffa495ef1151fb0f9467b82e254ab1 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Mon, 12 Dec 2022 09:13:37 +1300 Subject: [PATCH 28/42] chore: Remove unused files --- .dockerignore | 15 - CHANGELOG.md | 268 ------------------ Dockerfile | 78 ----- .../down.sql | 2 - .../up.sql | 1 - .../2022-08-12-123800_ft_nft_indexes/down.sql | 2 - .../2022-08-12-123800_ft_nft_indexes/up.sql | 2 - .../down.sql | 1 - .../up.sql | 1 - run-in-docker.sh | 113 -------- 10 files changed, 483 deletions(-) delete mode 100644 .dockerignore delete mode 100644 CHANGELOG.md delete mode 100644 Dockerfile delete mode 100644 migrations/2022-06-07-000000_drop_action_receipt_actions_args_amount_index/down.sql delete mode 100644 migrations/2022-06-07-000000_drop_action_receipt_actions_args_amount_index/up.sql delete mode 100644 migrations/2022-08-12-123800_ft_nft_indexes/down.sql delete mode 100644 migrations/2022-08-12-123800_ft_nft_indexes/up.sql delete mode 100644 migrations/2022-09-23-100000_index_action_receipt_actions_on_timestamp_and_order_index/down.sql delete mode 100644 migrations/2022-09-23-100000_index_action_receipt_actions_on_timestamp_and_order_index/up.sql delete mode 100755 run-in-docker.sh diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 645fa63b..00000000 --- a/.dockerignore +++ /dev/null @@ -1,15 +0,0 @@ -.git -.gitignore -target -*.swp - -# Don't invalidate the Docker cache if this file or the Dockerfile change since -# the engine will be forced to read it anyways -.dockerignore -Dockerfile - -# Don't invalidate the Docker cache if the docs change -*.md -docs -LICENSE -CODEOWNERS diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index a703d6ff..00000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,268 +0,0 @@ -# Changelog - -## 0.10.32 - -* Avoid recreating access key on transfer to implicit account - -## 0.10.31 - patch - -* Upgrade Indexer Framework to be based on [`nearcore` version 1.30.0 patched](https://github.com/near/nearcore/commit/267e36e39fb5bb29c1df23c73afbcaa750ce96b1) - -## 0.10.30 - -* Upgrade Indexer Framework to be based on [`nearcore` version `1.30.0-rc.2`](https://github.com/near/nearcore/releases/tag/1.30.0-rc.2) - -## 0.10.29 - -* Fixed circulating supply computation given that the args are not base64-encoded anymore (was broken since 0.10.22 release) - -## 0.10.28 - -* Fixed function args extraction (`args->args_json`) for actions (`action_receipt_actions` and `transaction_actions` tables). This bug affected all the records since 0.10.22 release. Consider removing and re-indexing all the records from the time you deployed 0.10.22+ release. - -## 0.10.27 - -* Upgrade Indexer Framework to be based on [`nearcore` version `1.29.0`](https://github.com/near/nearcore/releases/tag/1.29.0) -* (MIGRATION REQUIRED) Add new complex index on `action_receipt_actions` (`block_timestamp` + `index_in_actions`) that is used for strict ordering - -## 0.10.26 - -* Upgrade `nearcore` to `1.29.0-rc.3` - -## 0.10.25 - -* Update `serializer` to encode `args_base64` since `nearcore` is no more encoding `args` in `FunctionCall` Actions (fix of #305) - -## 0.10.24 - -* Upgrade `nearcore` to [`1.29.0-rc.2`](https://github.com/near/nearcore/releases/tag/1.29.0-rc.2) (according to nearcore release notes, it will require 3GB more RAM than before) - -## 0.10.23 - -* Fix the way we get `sha256` of the deployed contract code according to the changes introduced in `nearcore 1.29.0-rc.1` - -## 0.10.22 - -* Upgrade `nearcore` to `1.29.0-rc.1` - -## 0.10.21 - -* Upgrade `nearcore` to `1.28.0` - -## 0.10.20 - -* Upgrade `nearcore` to `1.28.0-rc.1` - -## 0.10.19 - -* Upgrade `nearcore` to `1.27.0` - -## 0.10.18 - -* Upgrade `nearcore` to `1.27.0-rc.2` - -## 0.10.17 - -* Upgrade `nearcore` to `1.27.0-rc.1` - -## 0.10.16 - -* Upgrade `nearcore` to `1.26.0` - -## 0.10.15 - -* Upgrade `nearcore` to `1.26.0-rc.1` - -## 0.10.14 - -* Fix the bug in logic of finding parent transaction for a receipt - -## 0.10.13 - -* Upgrade `nearcore` to `1.25.0-rc.3` - -## 0.10.12 - -* Upgrade `nearcore` to `1.25.0-rc.1` - -## 0.10.11 - -* Upgrade `nearcore` to `1.24.0-rc.4` - -## 0.10.10 - -* Upgrade `nearcore` to `1.24.0-rc.3` - -## 0.10.9 - -* Add [FT events](https://nomicon.io/Standards/FungibleToken/Event.html) support: `assets__fungible_token_events` table stores the information about FT `mint`, `transfer`, `burn` events -* Add index on `action_receipt_actions` table. Applying migration could take some time -* Introduce `ReceiptsCache` to optimize the way we match `Receipts` with parent `Transactions` -* Upgrade nearcore to `1.24.0-rc.2` - -## 0.10.8 - -* Upgrade nearcore to `1.24.0-rc.1` - -## 0.10.7 - -* Upgrade `nearcore` to 1.23.1 -* Optimized the latency of putting the data into database - -## 0.10.6 - -* Upgrade `nearcore` to 1.23.1-rc.2 - - -## 0.10.5 - -* Upgrade `nearcore` to 1.23.1-rc.1 - -## 0.10.4 - -* Upgrade `nearcore` to 1.23.0 -* Handle transaction hash collisions (issue #84) - -## 0.10.3 - -* Upgrade `nearcore` to 1.22.0 -* Add [NFT events](https://nomicon.io/Standards/NonFungibleToken/Event.html) support: `assets__non_fungible_token_events` table stores the information about NFT `mint`, `transfer`, `burn` events - -## 0.10.2 - -- Change the retry logic. Make indexer fail with error if is has retried for 5 min -- Upgrade `nearcore` to 1.22.0 - -## 0.10.1 - -* Upgrade `nearcore` to 1.21.1 - -## 0.10.0 - -* Drop `--allow-missing-relations-in-first-blocks` flag -* Introduce `--non-strict-mode` which does the same as `--allow-missing-relations-in-first-blocks` flag did but endlessly -* Add `--stop-after-number-of-blocks ` flag to plan Indexer for Explorer to stop once it indexed the provided `` of blocks. May be useful for debug or maintenance purposes. - -## Breaking changes - -* The flag `--allow-missing-relations-in-first-blocks` is not available anymore in favor of `--non-strict-mode` flag - -## 0.9.3 - -* Escape `args_json` on the fly to avoid null-byte issues -* Upgrade to NEAR Indexer Framework `0.10.0` -* Refactor the storing Accounts and AccessKeys from genesis to optimize memory usage -* Improve logging to better understand what Indexer for Explorer is doing on the start - -## 0.9.2 (hotfix) - -* Change `receiver_id` field type to `String` to be compatible with `nearcore` `AccessKeyPermissionView` struct (it caused problems during AccessKey serialization) - -## 0.9.1 - -* Upgrade `nearcore` to 1.21.0 (rc1) - -## 0.9.0 (nearcore dependency contains bug) - -* Upgrade `nearcore` to 1.21.0 - -## Breaking changes - -* `init` command has changed according to changes in `nearcore`: - - `download` argument has been replaced with `download_config` and `download_genesis` - - `boot_nodes` argument was added - - `download_config_url` was added -* `AccountId` from `near-primitives` was replaced with separate crate `near-account-id` and it is no longer an alias for `String` - - All the fields related to an account id have type `near_account_id::AccountId` - -## 0.8.0 - -* Background calculation of circulating supply and storing it to DB -* Improvements on some tables (add indexes, simplify sorting etc.) - -## 0.7.1 - -* Update nearcore version to 1.20.0-rc.2 - -## 0.7.0 - -* Handle null-bytes in `AddKey` actions -* Update nearcore version to 1.20.0 - -## Breaking change - -`init_configs` function from nearcore has been extended with additional optional parameter `max_gas_burnt_view`. We've extended NEAR Indexer for Explorer `InitConfigArgs` - -## 0.6.9 - -* Add `--concurrency` parameter to adjust the number of simultaneously running asynchronous adapters - -## 0.6.8 - -* Update NEAR Indexer Framework version to 0.9.2 (with optimized delayed receipts tracking system) - -## 0.6.7 - -* Remove duplicates from `account_changes` table by fixing unique index ([see issue #74](https://github.com/near/near-indexer-for-explorer/issues/74)) - -## 0.6.6 (hotfix) - -* Upgrade `nearcore` to 1.19.1 (hotfix) - -## 0.6.5 - -* Update NEAR Indexer Framework version to 0.9.1 (previous contained a bug with processing delayed local receipts) - -## 0.6.4 (contains bug) - -* Fix the overwriting of `created_by_receipt_id` for implicit accounts that may confuse users ([see issue #68 for ref](https://github.com/near/near-indexer-for-explorer/issues/68)) - -## 0.6.3 - -* Denormalize table `action_receipt_actions` in order to speed up some queries by avoid - additional joins -* Extend `extract_action_type_and_value_from_action_view` function to try to parse base64 encoded args - as a JSON object to put them decoded in the function call args `action_receipt_actions.args` additionally - -## 0.6.2 - -* Upgrade `nearcore` dependency to exclude recent updates to runtime which caused a bug ([see for ref](https://github.com/near/nearcore/releases/tag/1.19.0-rc.2)) - -## 0.6.1 - -* Upgrade `nearcore` to support newer protocol version (45) - -## 0.6.0 - -* Upgrade `nearcore` to get NEAR Indexer Framework 0.9.0 -* Corresponding changes into adapters according to changes in `StreamerMessage` structure -* NEAR Indexer for Explorer now uses stable Rust (`rust-toolchain` has been updated accordingly) - -## 0.5.0 - -* Tweak `sync-from-interruption` mode to start syncing from N blocks earlier that actual interruption - -## 0.4.0 - -* Update `nearcore` dependency -* Update underlying dependencies to correspond `nearcore` - -**The way of starting `actix` runtime has changes** - -## 0.3.0 - -* Migrate from `tokio-diesel` to `actix-diesel` (patched by @frol) - -## 0.2.3 - -* Upgrade `nearcore` dependency -* Upgrade some external dependencies (`actix`, `tokio`) - -## 0.2.2 - -* Fill `deleted_by_receipt_id` if `access_key` on owner account deletion - -## 0.2.1 - -* Add `access_key` on transfer to implicit account action -* Upgrade `nearcore` dependency diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 0cfaab95..00000000 --- a/Dockerfile +++ /dev/null @@ -1,78 +0,0 @@ -# syntax=docker/dockerfile-upstream:experimental - -# ============================== BUILD ====================================== -FROM ubuntu:18.04 as build - -RUN apt-get update -qq && apt-get install -y \ - git \ - cmake \ - g++ \ - pkg-config \ - libssl-dev \ - curl \ - llvm \ - clang \ - libpq-dev \ - ca-certificates \ - && rm -rf /var/lib/apt/lists/* - -COPY ./rust-toolchain /tmp/rust-toolchain - -ENV RUSTUP_HOME=/usr/local/rustup \ - CARGO_HOME=/usr/local/cargo \ - PATH=/usr/local/cargo/bin:$PATH - -RUN curl https://sh.rustup.rs -sSf | \ - sh -s -- -y --no-modify-path --default-toolchain "$(cat /tmp/rust-toolchain)" - -RUN cargo install diesel_cli --no-default-features --features "postgres" --bin diesel - -WORKDIR /near - -# This is some clever stuff we do to build JUST the Cargo.toml dependencies first, so that Docker can cache them so long as Cargo.toml doesn't change -# We do this because building dependencies takes ~45 minutes -RUN cargo +"$(cat /tmp/rust-toolchain)" new --bin indexer-explorer -WORKDIR /near/indexer-explorer - -COPY ./Cargo.toml ./Cargo.toml -COPY ./Cargo.lock ./Cargo.lock - -ENV CARGO_TARGET_DIR=/tmp/target -ENV RUSTC_FLAGS='-C target-cpu=x86-64' -ENV PORTABLE=ON -RUN cargo +"$(cat /tmp/rust-toolchain)" build --release -RUN rm src/*.rs -RUN rm /tmp/target/release/indexer-explorer* - -# Now that the dependencies are built, copy just the rest of the Rust code in and build -# that too (so that a change in the run-in-docker.sh script doesn't invalidate the entire -# Docker cache for the Rust build) -COPY ./src ./src - -# This touch is necessary so that Rust doesn't skip the build (even though the source has completely changed... Rust cache is weird :P) -RUN touch src/main.rs - -RUN cargo +"$(cat /tmp/rust-toolchain)" build --release -p indexer-explorer - -# ============================== EXECUTION ====================================== -FROM ubuntu:18.04 as execution - -RUN apt-get update -qq && apt-get install -y \ - libssl-dev \ - libpq-dev \ - ca-certificates \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /near/indexer-explorer - -COPY --from=build /tmp/target/release/indexer-explorer . - -COPY --from=build /usr/local/cargo/bin/diesel . - -COPY ./migrations ./migrations - -COPY ./run-in-docker.sh . - -# Pass the BOOT_NODES environment variable to set boot nodes in the initialized genesis -# An arbitrary number of extra parameters to the indexer that runs can be set via CMD -ENTRYPOINT bash run-in-docker.sh ./diesel "${DATABASE_URL}" ./indexer-explorer diff --git a/migrations/2022-06-07-000000_drop_action_receipt_actions_args_amount_index/down.sql b/migrations/2022-06-07-000000_drop_action_receipt_actions_args_amount_index/down.sql deleted file mode 100644 index 9c3148dc..00000000 --- a/migrations/2022-06-07-000000_drop_action_receipt_actions_args_amount_index/down.sql +++ /dev/null @@ -1,2 +0,0 @@ -CREATE INDEX action_receipt_actions_args_amount_idx ON action_receipt_actions((args->'args_json'->>'amount')) - WHERE action_receipt_actions.action_kind = 'FUNCTION_CALL' AND (action_receipt_actions.args->>'args_json') IS NOT NULL; diff --git a/migrations/2022-06-07-000000_drop_action_receipt_actions_args_amount_index/up.sql b/migrations/2022-06-07-000000_drop_action_receipt_actions_args_amount_index/up.sql deleted file mode 100644 index 34b5eb62..00000000 --- a/migrations/2022-06-07-000000_drop_action_receipt_actions_args_amount_index/up.sql +++ /dev/null @@ -1 +0,0 @@ -DROP INDEX action_receipt_actions_args_amount_idx; diff --git a/migrations/2022-08-12-123800_ft_nft_indexes/down.sql b/migrations/2022-08-12-123800_ft_nft_indexes/down.sql deleted file mode 100644 index 38044a6f..00000000 --- a/migrations/2022-08-12-123800_ft_nft_indexes/down.sql +++ /dev/null @@ -1,2 +0,0 @@ -DROP INDEX assets__ft_contract_id_idx; -DROP INDEX assets__nft_contract_id_idx; diff --git a/migrations/2022-08-12-123800_ft_nft_indexes/up.sql b/migrations/2022-08-12-123800_ft_nft_indexes/up.sql deleted file mode 100644 index 779b539f..00000000 --- a/migrations/2022-08-12-123800_ft_nft_indexes/up.sql +++ /dev/null @@ -1,2 +0,0 @@ -CREATE INDEX assets__ft_contract_id_idx ON assets__fungible_token_events (emitted_by_contract_account_id); -CREATE INDEX assets__nft_contract_id_idx ON assets__non_fungible_token_events (emitted_by_contract_account_id); diff --git a/migrations/2022-09-23-100000_index_action_receipt_actions_on_timestamp_and_order_index/down.sql b/migrations/2022-09-23-100000_index_action_receipt_actions_on_timestamp_and_order_index/down.sql deleted file mode 100644 index 1613c049..00000000 --- a/migrations/2022-09-23-100000_index_action_receipt_actions_on_timestamp_and_order_index/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP INDEX action_receipt_actions_receipt_included_in_block_timestamp_index_in_action_receipt_idx; diff --git a/migrations/2022-09-23-100000_index_action_receipt_actions_on_timestamp_and_order_index/up.sql b/migrations/2022-09-23-100000_index_action_receipt_actions_on_timestamp_and_order_index/up.sql deleted file mode 100644 index 131c0154..00000000 --- a/migrations/2022-09-23-100000_index_action_receipt_actions_on_timestamp_and_order_index/up.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE INDEX action_receipt_actions_receipt_included_in_block_timestamp_index_in_action_receipt_idx ON action_receipt_actions(receipt_included_in_block_timestamp, index_in_action_receipt); diff --git a/run-in-docker.sh b/run-in-docker.sh deleted file mode 100755 index 71d6bcd6..00000000 --- a/run-in-docker.sh +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail # Bash "strict mode" -script_dirpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - - -# ================================================================================================== -# Constants -# ================================================================================================== -# The path where the localnet NEAR config dir will be initialized -LOCALNET_NEAR_DIRPATH="/root/.near/localnet" -CONFIG_JSON_FILEPATH="${LOCALNET_NEAR_DIRPATH}/config.json" - -# Config properties that will be set as part of startup -TRACKED_SHARD_CONFIG_PROPERTY="tracked_shards" -ARCHIVE_CONFIG_PROPERTY="archive" - - -# ================================================================================================== -# Arg Parsing & Validation -# ================================================================================================== -show_helptext_and_exit() { - echo "Usage: $(basename "${0}") diesel_binary_filepath database_url indexer_binary_filepath [extra_indexer_param]..." - echo "" - echo " diesel_binary_filepath The filepath to the Diesel binary that will be used to run the database migration" - echo " database_url The URL of the database against which the Diesel migration should be run, and the " - echo " indexer should connect to (e.g. \"postgres://near:near@contract-helper-db:5432/indexer\")" - echo " indexer_binary_filepath The filepath to the binary that will run the indexer node" - echo " extra_indexer_param... An arbitrary number of extra parameters that will be passed as-is to the indexer" - echo "" - exit 1 # Exit with an error so that if this is accidentally called by CI, the script will fail -} - -diesel_binary_filepath="${1:-}" -database_url="${2:-}" -indexer_binary_filepath="${3:-}" - -if [ -z "${diesel_binary_filepath}" ]; then - echo "Error: no Diesel binary filepath provided" >&2 - show_helptext_and_exit -fi -if ! [ -f "${diesel_binary_filepath}" ]; then - echo "Error: provided Diesel binary filepath '${some_filepath_arg}' isn't a valid file" >&2 - show_helptext_and_exit -fi -if [ -z "${database_url}" ]; then - echo "Error: no database URL provided" >&2 - show_helptext_and_exit -fi -if [ -z "${indexer_binary_filepath}" ]; then - echo "Error: no indexer binary filepath provided" >&2 - show_helptext_and_exit -fi -if ! [ -f "${indexer_binary_filepath}" ]; then - echo "Error: provided indexer binary filepath '${some_filepath_arg}' isn't a valid file" >&2 - show_helptext_and_exit -fi - -shift 3 # Prep for consuming the extra indexer params below - - -# ================================================================================================== -# Main Logic -# ================================================================================================== -# We add this check to see if the localnet directory already exists so that we can restart the -# indexer-for-explorer container: if the directory doesn't exist, the container is starting for the -# first time; if it already exists, the container is restarting so there's no need to do the migration -# or genesis setup -if ! [ -d "${LOCALNET_NEAR_DIRPATH}" ]; then - if ! DATABASE_URL="${database_url}" "${diesel_binary_filepath}" migration run; then - echo "Error: The Diesel migration failed" >&2 - exit 1 - fi - - if ! DATABASE_URL="${database_url}" "${indexer_binary_filepath}" --home-dir "${LOCALNET_NEAR_DIRPATH}" init ${BOOT_NODES:+--boot-nodes=${BOOT_NODES}} --chain-id localnet; then - echo "Error: An error occurred generating the genesis information" >&2 - exit 1 - fi - - # Required due to https://github.com/near/near-indexer-for-explorer#configure-near-indexer-for-explorer - if ! num_tracked_shard_instances="$(grep -c "\"${TRACKED_SHARD_CONFIG_PROPERTY}\":" "${CONFIG_JSON_FILEPATH}" || true)"; then - echo "Error: An error occurred getting the number of instances of the '${TRACKED_SHARD_CONFIG_PROPERTY}' config property to verify there's only one" >&2 - exit 1 - fi - if [ "${num_tracked_shard_instances}" -ne 1 ]; then - echo "Error: Expected exactly one line to match property '${TRACKED_SHARD_CONFIG_PROPERTY}' in config file '${CONFIG_JSON_FILEPATH}' but got ${num_tracked_shard_instances}" >&2 - exit 1 - fi - if ! sed -i 's/"'${TRACKED_SHARD_CONFIG_PROPERTY}'": \[\]/"'${TRACKED_SHARD_CONFIG_PROPERTY}'": \[0\]/' "${CONFIG_JSON_FILEPATH}"; then - echo "Error: An error occurred setting the tracked shards in the config" >&2 - exit 1 - fi - - # Required to keep more than 5 blocks in memory - if ! num_archive_instances="$(grep -c "\"${ARCHIVE_CONFIG_PROPERTY}\":" "${CONFIG_JSON_FILEPATH}" || true)"; then - echo "Error: An error occurred getting the number of instances of the '${ARCHIVE_CONFIG_PROPERTY}' config property to verify there's only one" >&2 - exit 1 - fi - if [ "${num_archive_instances}" -ne 1 ]; then - echo "Error: Expected exactly one line to match property '${ARCHIVE_CONFIG_PROPERTY}' in config file '${CONFIG_JSON_FILEPATH}' but got ${num_archive_instances}" >&2 - exit 1 - fi - if ! sed -i 's/"'${ARCHIVE_CONFIG_PROPERTY}'": false/"'${ARCHIVE_CONFIG_PROPERTY}'": true/' "${CONFIG_JSON_FILEPATH}"; then - echo "Error: An error occurred setting the archive mode to true" >&2 - exit 1 - fi -fi - -# NOTE1: If the --store-genesis flag isn't set, the accounts in genesis won't get created in the DB which will lead to foreign key constraint violations -# See https://github.com/near/near-indexer-for-explorer/issues/167 -# NOTE2: The funky ${1+"${@}"} incantation is how you you feed arguments exactly as-is to a child script in Bash -# ${*} loses quoting and ${@} trips set -e if no arguments are passed, so this incantation says, "if and only if -# ${1} exists, evaluate ${@}" -DATABASE_URL="${database_url}" "${indexer_binary_filepath}" --home-dir "${LOCALNET_NEAR_DIRPATH}" run --store-genesis sync-from-latest ${1+"${@}"} From cb8649ee1a27532308b054620340d58f5718241e Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Mon, 12 Dec 2022 09:34:24 +1300 Subject: [PATCH 29/42] chore: Delete `release.yml` as we now build in GCP --- .github/workflows/release.yml | 117 ---------------------------------- 1 file changed, 117 deletions(-) delete mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 183435eb..00000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,117 +0,0 @@ -name: release -on: - push: - # Enable when testing release infrastructure on a branch. - # branches: - # - ci/release-check - tags: - - "[0-9]+.[0-9]+.[0-9]+" - - "[0-9]+.[0-9]+.[0-9]+-*" -jobs: - create-release: - name: create-release - runs-on: ubuntu-latest - # env: - # Set to force version number, e.g., when no tag exists. - # NEAR_LAKE_VERSION: TEST-0.0.0 - outputs: - upload_url: ${{ steps.release.outputs.upload_url }} - indexer_version: ${{ env.INDEXER_VERSION }} - steps: - - name: Get the release version from the tag - shell: bash - if: env.INDEXER_VERSION == '' - run: | - # Apparently, this is the right way to get a tag name. Really? - # - # See: https://github.community/t5/GitHub-Actions/How-to-get-just-the-tag-name/m-p/32167/highlight/true#M1027 - echo "INDEXER_VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV - echo "version is: ${{ env.INDEXER_VERSION }}" - - name: Create GitHub release - id: release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ env.INDEXER_VERSION }} - release_name: ${{ env.INDEXER_VERSION }} - - build-release: - name: build-release - needs: ['create-release'] - runs-on: ${{ matrix.os }} - env: - # For some builds, we use cross to test on 32-bit and big-endian - # systems. - CARGO: cargo - # When CARGO is set to CROSS, this is set to `--target matrix.target`. - TARGET_FLAGS: "" - # When CARGO is set to CROSS, TARGET_DIR includes matrix.target. - TARGET_DIR: ./target - # For some builds, we disable ledger support - FEATURES_FLAGS: - # Emit backtraces on panics. - RUST_BACKTRACE: 1 - # Build static releases with PCRE2. - PCRE2_SYS_STATIC: 1 - strategy: - fail-fast: false - matrix: - include: - - build: linux-x86_64 - os: ubuntu-latest - rust: stable - target: x86_64-unknown-linux-gnu - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - with: - fetch-depth: 1 - - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: ${{ matrix.rust }} - profile: minimal - override: true - target: ${{ matrix.target }} - - - name: Use Cross - run: | - cargo install cross - echo "CARGO=cross" >> $GITHUB_ENV - echo "TARGET_FLAGS=--target ${{ matrix.target }}" >> $GITHUB_ENV - echo "TARGET_DIR=./target/${{ matrix.target }}" >> $GITHUB_ENV - - - name: Build cross environment - shell: bash - run: | - if [ -d "./cross/${{ matrix.target }}" ]; then - docker build --tag "cross:${{ matrix.target }}" "./cross/${{ matrix.target }}" - fi - - - name: Show command used for Cargo - run: | - echo "cargo command is: ${{ env.CARGO }}" - echo "target flag is: ${{ env.TARGET_FLAGS }}" - echo "target dir is: ${{ env.TARGET_DIR }}" - - - name: Build release binary - run: ${{ env.CARGO }} build --verbose --release ${{ env.TARGET_FLAGS }} ${{ env.FEATURES_FLAGS }} - - - name: Strip release binary (linux and macos) - run: | - strip "${{ env.TARGET_DIR }}/release/indexer-explorer-lake" - cp "${{ env.TARGET_DIR }}/release/indexer-explorer-lake" indexer-explorer-lake - echo "ASSET=indexer-explorer-lake" >> $GITHUB_ENV - - - name: Upload release archive - uses: actions/upload-release-asset@v1.0.1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ needs.create-release.outputs.upload_url }} - asset_path: ${{ env.ASSET }} - asset_name: ${{ env.ASSET }} - asset_content_type: application/octet-stream From 826e84af6740d4991ebf5458818da315e6073e06 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Mon, 12 Dec 2022 09:34:40 +1300 Subject: [PATCH 30/42] chore: Remove `Cross` --- .github/workflows/test.yml | 42 +++-------------------- Cross.toml | 6 ---- cross/x86_64-unknown-linux-gnu/Dockerfile | 15 -------- 3 files changed, 4 insertions(+), 59 deletions(-) delete mode 100644 Cross.toml delete mode 100644 cross/x86_64-unknown-linux-gnu/Dockerfile diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 955177a2..6939923c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,27 +9,12 @@ on: jobs: test: name: test + runs-on: ubuntu-latest env: - # For some builds, we use cross to test on 32-bit and big-endian - # systems. - CARGO: cargo - # When CARGO is set to CROSS, this is set to `--target matrix.target`. - TARGET_FLAGS: "" - # When CARGO is set to CROSS, TARGET_DIR includes matrix.target. - TARGET_DIR: ./target # For some builds, we disable ledger support FEATURES_FLAGS: # Emit backtraces on panics. RUST_BACKTRACE: 1 - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - include: - - build: linux-x86_64 - os: ubuntu-latest - rust: stable - target: x86_64-unknown-linux-gnu steps: - name: Checkout repository @@ -38,34 +23,15 @@ jobs: - name: Install Rust uses: actions-rs/toolchain@v1 with: - toolchain: ${{ matrix.rust }} + toolchain: stable profile: minimal override: true - - name: Use Cross - run: | - cargo install cross - echo "CARGO=cross" >> $GITHUB_ENV - echo "TARGET_FLAGS=--target ${{ matrix.target }}" >> $GITHUB_ENV - echo "TARGET_DIR=./target/${{ matrix.target }}" >> $GITHUB_ENV - - - name: Build cross environment - shell: bash - run: | - if [ -d "./cross/${{ matrix.target }}" ]; then - docker build --tag "cross:${{ matrix.target }}" "./cross/${{ matrix.target }}" - fi - - - name: Show command used for Cargo - run: | - echo "cargo command is: ${{ env.CARGO }}" - echo "target flag is: ${{ env.TARGET_FLAGS }}" - - name: Build indexer-explorer-lake and all crates - run: ${{ env.CARGO }} build --verbose --all ${{ env.TARGET_FLAGS }} ${{ env.FEATURES_FLAGS }} + run: cargo build --verbose --all ${{ env.FEATURES_FLAGS }} - name: Run tests - run: ${{ env.CARGO }} test --verbose --all ${{ env.TARGET_FLAGS }} ${{ env.FEATURES_FLAGS }} + run: cargo test --verbose --all ${{ env.FEATURES_FLAGS }} rustfmt: name: rustfmt diff --git a/Cross.toml b/Cross.toml deleted file mode 100644 index 36c74329..00000000 --- a/Cross.toml +++ /dev/null @@ -1,6 +0,0 @@ -[target.x86_64-unknown-linux-gnu] -# Run the following command to get the build environment: -# ``` -# docker build --tag cross:x86_64-unknown-linux-gnu ./cross/x86_64-unknown-linux-gnu/ -# ``` -image = "cross:x86_64-unknown-linux-gnu" diff --git a/cross/x86_64-unknown-linux-gnu/Dockerfile b/cross/x86_64-unknown-linux-gnu/Dockerfile deleted file mode 100644 index 089e4a06..00000000 --- a/cross/x86_64-unknown-linux-gnu/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM ubuntu:18.04 - -# Copy from nearcore: -# https://github.com/near/nearcore/blob/master/Dockerfile -RUN apt-get update -qq && \ - apt-get install -y \ - git \ - cmake \ - g++ \ - pkg-config \ - libssl-dev \ - libpq-dev \ - curl \ - llvm \ - clang From 4f68f8710a8aaaaf9c6f6c0e7f882facc6e91aa5 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Mon, 12 Dec 2022 09:40:11 +1300 Subject: [PATCH 31/42] chore: Point to `docs.near.org` instead of `near-indexers.io` Co-authored-by: Bohdan Khorolets --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1d04b13f..cdc328f6 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ NOTE: Please, keep in mind that the access to the database is shared across ever The final setup consists of the following components: * PostgreSQL database (you can run it locally or in the cloud), which can hold the whole history of the blockchain (as of August 2022, mainnet takes 3TB of data in PostgreSQL storage, and testnet takes 1TB) -* NEAR Indexer for Explorer binary that operates as a NEAR Lake Framework based indexer, it requires [AWS S3 credentials](https://near-indexers.io/tutorials/lake/credentials) +* NEAR Indexer for Explorer binary that operates as a NEAR Lake Framework based indexer, it requires [AWS S3 credentials](https://docs.near.org/tutorials/indexer/credentials) ### Prepare Development Environment From 9361c23592ab0a23fbf7f0b4d2648480d4e1d906 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Mon, 12 Dec 2022 09:53:17 +1300 Subject: [PATCH 32/42] chore: Add link to "store genesis" issue --- README.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/README.md b/README.md index cdc328f6..422a342f 100644 --- a/README.md +++ b/README.md @@ -84,11 +84,7 @@ You can choose NEAR Indexer for Explorer start options: - `from-interruption` - start indexing blocks from the block NEAR Indexer was interrupted last time but earlier for `` if provided - `from-block --height ` - start indexing blocks from the specific block height -Unlike the original [NEAR Indexer for Explorer](https://github.com/near/near-indexer-for-explorer) you **can't** tell Indexer to store data from genesis (Accounts and Access Keys) by adding key `--store-genesis` to the `run` command. So please, ensure you took care about the genesis data in your database in order this indexer to work properly. - -``` -TODO: Implement a tool to fill the database with the genesis data -``` +Unlike the original NEAR Indexer for Explorer you **can't** tell Indexer to store data from genesis (Accounts and Access Keys) by adding key `--store-genesis` to the `run` command. So please, ensure you took care about the genesis data in your database in order this indexer to work properly. This capability will be implemented eventually, it's progress can be tracked here: #327. NEAR Indexer for Explorer works in strict mode by default, but you can disable it. The strict mode means that every piece of data will be retried to store to database in case of error. Errors may occur when the parent piece of data is still processed but the child piece is already trying to be stored. So Indexer keeps retrying to store the data until success. However, if you're running Indexer not from the genesis it is possible that you really miss some of parent data and it'll be impossible to store a child one, so you can disable strict mode to ensure you've passed the strong relation data area and you're running Indexer where it is impossible to loose any piece of data. From 2713da1374b7cfe185524ee46b492bcf5d347fa6 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Mon, 12 Dec 2022 09:54:31 +1300 Subject: [PATCH 33/42] chore: Add missing `chain-id` in example command --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 422a342f..67105436 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,7 @@ So final command to run NEAR Indexer for Explorer can look like: $ ./target/release/indexer-explorer \ --non-strict-mode \ --concurrency 1 \ + mainnet \ from-latest ``` From 800a64e420f5358205c6dda34cf0ed7a6841aee7 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Mon, 12 Dec 2022 10:20:22 +1300 Subject: [PATCH 34/42] chore: Add breaking changes to `CHANGELOG.md` --- indexer/CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/indexer/CHANGELOG.md b/indexer/CHANGELOG.md index 72209c3d..5dedd5c4 100644 --- a/indexer/CHANGELOG.md +++ b/indexer/CHANGELOG.md @@ -2,7 +2,13 @@ ## 0.11.0 -* Migrate to NEAR Lake Framework +* Migrate to NEAR Lake Framework, `indexer-explorer` no longer runs a NEAR node and instead consumes block from [near-lake-indexer](https://github.com/near/near-lake-indexer). + +### Breaking Changes + +* Storing genesis file is no longer possible (#327) +* Command line arguments for starting the indexer have changed slightly, see [Readme](../README.md) for updated reference +* AWS credentials are now required for reading data from S3 ## 0.10.32 From 9433d06e2b6ba9983479c927b46d4c4f43bb7879 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Mon, 12 Dec 2022 10:47:37 +1300 Subject: [PATCH 35/42] chore: Rephrase paragraph about `non-strict-mode` --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 67105436..a33b6507 100644 --- a/README.md +++ b/README.md @@ -86,10 +86,9 @@ You can choose NEAR Indexer for Explorer start options: Unlike the original NEAR Indexer for Explorer you **can't** tell Indexer to store data from genesis (Accounts and Access Keys) by adding key `--store-genesis` to the `run` command. So please, ensure you took care about the genesis data in your database in order this indexer to work properly. This capability will be implemented eventually, it's progress can be tracked here: #327. -NEAR Indexer for Explorer works in strict mode by default, but you can disable it. The strict mode means that every piece of data -will be retried to store to database in case of error. Errors may occur when the parent piece of data is still processed but the child piece is already trying to be stored. So Indexer keeps retrying to store the data until success. However, if you're running Indexer not from the genesis it is possible that you really miss some of parent data and it'll be impossible to store a child one, so you can disable strict mode to ensure you've passed the strong relation data area and you're running Indexer where it is impossible to loose any piece of data. +NEAR Indexer for Explorer works in strict mode by default. In strict mode, the Indexer will ensure parent data exists before storing children, infinitely retrying until this condition is met. This is necessary as a parent (i.e. `block`) may still be processing while a child (i.e. `receipt`) is ready to be stored. This scenario will likely occur if you have not stored the genesis file or do not have all data prior to the block you start indexing from. In this case, you can disable strict mode to store data prior to the block you are concerned about, and then re-enable it once you have passed this block. -To disable strict mode you need to provide: +To disable strict mode provide the following command arugment: ``` --non-strict-mode From 758c271bdf3a9bb5f0286807ba0a6694931c0df9 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Mon, 12 Dec 2022 10:51:12 +1300 Subject: [PATCH 36/42] chore: Add subheadings for running indexer --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index a33b6507..a1b76baa 100644 --- a/README.md +++ b/README.md @@ -84,8 +84,10 @@ You can choose NEAR Indexer for Explorer start options: - `from-interruption` - start indexing blocks from the block NEAR Indexer was interrupted last time but earlier for `` if provided - `from-block --height ` - start indexing blocks from the specific block height +#### Storing genesis file Unlike the original NEAR Indexer for Explorer you **can't** tell Indexer to store data from genesis (Accounts and Access Keys) by adding key `--store-genesis` to the `run` command. So please, ensure you took care about the genesis data in your database in order this indexer to work properly. This capability will be implemented eventually, it's progress can be tracked here: #327. +#### Strict mode NEAR Indexer for Explorer works in strict mode by default. In strict mode, the Indexer will ensure parent data exists before storing children, infinitely retrying until this condition is met. This is necessary as a parent (i.e. `block`) may still be processing while a child (i.e. `receipt`) is ready to be stored. This scenario will likely occur if you have not stored the genesis file or do not have all data prior to the block you start indexing from. In this case, you can disable strict mode to store data prior to the block you are concerned about, and then re-enable it once you have passed this block. To disable strict mode provide the following command arugment: @@ -94,8 +96,10 @@ To disable strict mode provide the following command arugment: --non-strict-mode ``` +#### Concurrency By default NEAR Indexer for Explorer processes only a single block at a time. You can adjust this with the `--concurrency` argument (when the blocks are mostly empty, it is fine to go with as many as 100 blocks of concurrency). +#### Starting So final command to run NEAR Indexer for Explorer can look like: ```bash From 5e79896b65580aca70bcd3675d4a9885708d3dd5 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Mon, 12 Dec 2022 16:31:55 +1300 Subject: [PATCH 37/42] docs: Add notes about docker deployments in Readme --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index a1b76baa..e7f58cc5 100644 --- a/README.md +++ b/README.md @@ -149,3 +149,11 @@ GRANT readonly TO explorer; ```bash $ PGPASSWORD="password" psql -h 127.0.0.1 -U explorer databasename ``` + +## Deployments +Both `indexer-explorer` and `circulating-supply` binaries are run within Docker, their `Dockerfile`s can be found within their respective directoires/workspaces. Docker images are built using Google Cloud Build and then deployed to Google Cloud Run. The following commands can be used to build the Docker images: + +```bash +$ docker build -f ./indexer/Dockerfile . +$ docker build -f ./circulating-supply/Dockerfile . +``` From 4581db25a9e1eb8e56eb8dcc883e3d04b1f8e699 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Wed, 14 Dec 2022 07:36:31 +1300 Subject: [PATCH 38/42] refactor: Rename `retriable.rs` -> `retryable.rs` --- database/src/lib.rs | 2 +- database/src/{retriable.rs => retryable.rs} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename database/src/{retriable.rs => retryable.rs} (100%) diff --git a/database/src/lib.rs b/database/src/lib.rs index 142241c2..d40ff571 100644 --- a/database/src/lib.rs +++ b/database/src/lib.rs @@ -9,7 +9,7 @@ pub mod receipts_cache; mod schema; #[macro_use] -mod retriable; +mod retryable; const INTERVAL: std::time::Duration = std::time::Duration::from_millis(100); const EXPLORER_DATABASE: &str = "explorer_database"; diff --git a/database/src/retriable.rs b/database/src/retryable.rs similarity index 100% rename from database/src/retriable.rs rename to database/src/retryable.rs From 58e7b489e341162774566d3a9d26e6908ddfba57 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Wed, 14 Dec 2022 07:37:38 +1300 Subject: [PATCH 39/42] refactor: List duplicate expr out of match clause --- indexer/src/configs.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/indexer/src/configs.rs b/indexer/src/configs.rs index aa1d5ba5..8bed25f6 100644 --- a/indexer/src/configs.rs +++ b/indexer/src/configs.rs @@ -79,13 +79,10 @@ impl Opts { let config_builder = near_lake_framework::LakeConfigBuilder::default(); match &self.chain_id { - ChainId::Mainnet(_) => config_builder - .mainnet() - .start_block_height(get_start_block_height(self).await), - ChainId::Testnet(_) => config_builder - .testnet() - .start_block_height(get_start_block_height(self).await), + ChainId::Mainnet(_) => config_builder.mainnet(), + ChainId::Testnet(_) => config_builder.testnet(), } + .start_block_height(get_start_block_height(self).await) .build() .expect("Failed to build LakeConfig") } From 37825da5696fab73410cc7fbb630c1595a33ae93 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Wed, 14 Dec 2022 09:34:05 +1300 Subject: [PATCH 40/42] refactor: Swap indexing/metrics threads --- indexer/src/main.rs | 46 ++++++++++++++++++++++-------------------- indexer/src/metrics.rs | 9 ++++----- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/indexer/src/main.rs b/indexer/src/main.rs index d0b13b28..45de6024 100644 --- a/indexer/src/main.rs +++ b/indexer/src/main.rs @@ -159,28 +159,30 @@ async fn main() -> anyhow::Result<()> { let config: near_lake_framework::LakeConfig = opts.to_lake_config().await; let (_, stream) = near_lake_framework::streamer(config); - tokio::spawn(async move { - tracing::info!( - target: INDEXER_FOR_EXPLORER, - "Starting Indexer for Explorer (lake)...", - ); - let mut handlers = tokio_stream::wrappers::ReceiverStream::new(stream) - .map(|streamer_message| { - info!( - target: crate::INDEXER_FOR_EXPLORER, - "Block height {}", &streamer_message.block.header.height - ); - handle_message( - &pool, - streamer_message, - strict_mode, - std::sync::Arc::clone(&receipts_cache), - ) - }) - .buffer_unordered(1usize); + tokio::spawn(metrics::init_server(opts.port).expect("Failed to start metrics server")); + + tracing::info!( + target: INDEXER_FOR_EXPLORER, + "Starting Indexer for Explorer (lake)...", + ); + + let mut handlers = tokio_stream::wrappers::ReceiverStream::new(stream) + .map(|streamer_message| { + info!( + target: crate::INDEXER_FOR_EXPLORER, + "Block height {}", &streamer_message.block.header.height + ); + handle_message( + &pool, + streamer_message, + strict_mode, + std::sync::Arc::clone(&receipts_cache), + ) + }) + .buffer_unordered(1usize); - while let Some(_handle_message) = handlers.next().await {} - }); + while let Some(_handle_message) = handlers.next().await {} - metrics::init_server(opts.port).await + // unreachable statement, loop above is endless + Ok(()) } diff --git a/indexer/src/metrics.rs b/indexer/src/metrics.rs index 2a309419..559a7127 100644 --- a/indexer/src/metrics.rs +++ b/indexer/src/metrics.rs @@ -45,15 +45,14 @@ async fn get_metrics() -> impl Responder { String::from_utf8(buffer.clone()).unwrap() } -pub(crate) async fn init_server(port: u16) -> anyhow::Result<()> { +pub(crate) fn init_server(port: u16) -> anyhow::Result { info!( target: crate::INDEXER_FOR_EXPLORER, "Starting metrics server on http://0.0.0.0:{port}" ); - HttpServer::new(|| App::new().service(get_metrics)) + Ok(HttpServer::new(|| App::new().service(get_metrics)) .bind(("0.0.0.0", port))? - .run() - .await - .map_err(|e| anyhow::anyhow!("Error starting metrics server: {}", e)) + .disable_signals() + .run()) } From 49424624f975c47b77a9a7d86b97e42e44169057 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Wed, 14 Dec 2022 09:34:56 +1300 Subject: [PATCH 41/42] feat: Handle errors thrown during indexing --- indexer/src/main.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/indexer/src/main.rs b/indexer/src/main.rs index 45de6024..dc28c041 100644 --- a/indexer/src/main.rs +++ b/indexer/src/main.rs @@ -181,7 +181,16 @@ async fn main() -> anyhow::Result<()> { }) .buffer_unordered(1usize); - while let Some(_handle_message) = handlers.next().await {} + while let Some(handle_message) = handlers.next().await { + if let Err(e) = handle_message { + tracing::error!( + target: crate::INDEXER_FOR_EXPLORER, + "Encountered error while indexing: {}", + e + ); + anyhow::bail!(e) + } + } // unreachable statement, loop above is endless Ok(()) From 4519bb48420066044fe79a72cdd9e06d693095c3 Mon Sep 17 00:00:00 2001 From: Morgan McCauley Date: Wed, 14 Dec 2022 13:57:21 +1300 Subject: [PATCH 42/42] refactor: Drop `-lake` prefix in metrics --- indexer/src/metrics.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/indexer/src/metrics.rs b/indexer/src/metrics.rs index 559a7127..81c47aba 100644 --- a/indexer/src/metrics.rs +++ b/indexer/src/metrics.rs @@ -5,15 +5,12 @@ use tracing::info; lazy_static! { pub(crate) static ref LATEST_BLOCK_HEIGHT: IntGauge = try_create_int_gauge( - "indexer_explorer_lake_latest_block_height", + "indexer_explorer_latest_block_height", "Height of last processed block" ) .unwrap(); - pub(crate) static ref BLOCK_COUNT: IntCounter = try_create_int_counter( - "indexer_explorer_lake_block_count", - "Number of indexed blocks" - ) - .unwrap(); + pub(crate) static ref BLOCK_COUNT: IntCounter = + try_create_int_counter("indexer_explorer_block_count", "Number of indexed blocks").unwrap(); } fn try_create_int_gauge(name: &str, help: &str) -> prometheus::Result {