diff --git a/.cargo/makefiles/docker.toml b/.cargo/makefiles/docker.toml index 4335f61..f4af52e 100644 --- a/.cargo/makefiles/docker.toml +++ b/.cargo/makefiles/docker.toml @@ -18,3 +18,6 @@ args = ["build", "-t", "ghcr.io/0xintuition/substreams-sink:latest", "-f", "subs command = "docker" args = ["build", "-t", "ghcr.io/0xintuition/cli:latest", "-f", "cli/Dockerfile", "."] +[tasks.build-envio-indexer] +command = "docker" +args = ["build", "-t", "ghcr.io/0xintuition/envio-indexer:latest", "-f", "envio-indexer/Dockerfile", "."] diff --git a/.env.sample b/.env.sample index 188552f..f132919 100644 --- a/.env.sample +++ b/.env.sample @@ -34,6 +34,7 @@ FLAG_LOCAL_WITH_CLASSIFICATION=true HASURA_GRAPHQL_ADMIN_SECRET=myadminsecretkey HASURA_GRAPHQL_ENDPOINT=http://graphql-engine:8080 HF_TOKEN=optional +HYPERSYNC_TOKEN=your-token IMAGE_GUARD_URL=http://api:3000 INDEXING_SOURCE=substreams INTUITION_CONTRACT_ADDRESS=430BbF52503Bd4801E51182f4cB9f8F534225DE5 diff --git a/Cargo.lock b/Cargo.lock index 20b5b9e..25100d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -61,6 +61,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -69,9 +84,9 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "alloy" -version = "0.8.3" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59febb24956a41c29bb5f450978fbe825bd6456b3f80586c8bd558dc882e7b6a" +checksum = "bbcc41e8a11a4975b18ec6afba2cc48d591fa63336a4c526dacb50479a8d6b35" dependencies = [ "alloy-consensus", "alloy-contract", @@ -105,9 +120,9 @@ dependencies = [ [[package]] name = "alloy-consensus" -version = "0.8.3" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e88e1edea70787c33e11197d3f32ae380f3db19e6e061e539a5bcf8184a6b326" +checksum = "f4138dc275554afa6f18c4217262ac9388790b2fc393c2dfe03c51d357abf013" dependencies = [ "alloy-eips", "alloy-primitives", @@ -123,9 +138,9 @@ dependencies = [ [[package]] name = "alloy-consensus-any" -version = "0.8.3" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57b1bb53f40c0273cd1975573cd457b39213e68584e36d1401d25fd0398a1d65" +checksum = "0fa04e1882c31288ce1028fdf31b6ea94cfa9eafa2e497f903ded631c8c6a42c" dependencies = [ "alloy-consensus", "alloy-eips", @@ -137,9 +152,9 @@ dependencies = [ [[package]] name = "alloy-contract" -version = "0.8.3" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b668c78c4b1f12f474ede5a85e8ce550d0aa1ef7d49fd1d22855a43b960e725" +checksum = "5f21886c1fea0626f755a49b2ac653b396fb345233f6170db2da3d0ada31560c" dependencies = [ "alloy-dyn-abi", "alloy-json-abi", @@ -199,9 +214,9 @@ dependencies = [ [[package]] name = "alloy-eip7702" -version = "0.4.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c986539255fb839d1533c128e190e557e52ff652c9ef62939e233a81dd93f7e" +checksum = "cabf647eb4650c91a9d38cb6f972bb320009e7e9d61765fb688a86f1563b33e8" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -212,9 +227,9 @@ dependencies = [ [[package]] name = "alloy-eips" -version = "0.8.3" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9fadfe089e9ccc0650473f2d4ef0a28bc015bbca5631d9f0f09e49b557fdb3" +checksum = "52dd5869ed09e399003e0e0ec6903d981b2a92e74c5d37e6b40890bad2517526" dependencies = [ "alloy-eip2930", "alloy-eip7702", @@ -230,10 +245,11 @@ dependencies = [ [[package]] name = "alloy-genesis" -version = "0.8.3" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b2a4cf7b70f3495788e74ce1c765260ffe38820a2a774ff4aacb62e31ea73f9" +checksum = "e7d2a7fe5c1a9bd6793829ea21a636f30fc2b3f5d2e7418ba86d96e41dd1f460" dependencies = [ + "alloy-eips", "alloy-primitives", "alloy-serde", "alloy-trie", @@ -254,9 +270,9 @@ dependencies = [ [[package]] name = "alloy-json-rpc" -version = "0.8.3" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e29040b9d5fe2fb70415531882685b64f8efd08dfbd6cc907120650504821105" +checksum = "2008bedb8159a255b46b7c8614516eda06679ea82f620913679afbd8031fea72" dependencies = [ "alloy-primitives", "alloy-sol-types", @@ -268,9 +284,9 @@ dependencies = [ [[package]] name = "alloy-network" -version = "0.8.3" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "510cc00b318db0dfccfdd2d032411cfae64fc144aef9679409e014145d3dacc4" +checksum = "4556f01fe41d0677495df10a648ddcf7ce118b0e8aa9642a0e2b6dd1fb7259de" dependencies = [ "alloy-consensus", "alloy-consensus-any", @@ -293,9 +309,9 @@ dependencies = [ [[package]] name = "alloy-network-primitives" -version = "0.8.3" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9081c099e798b8a2bba2145eb82a9a146f01fc7a35e9ab6e7b43305051f97550" +checksum = "f31c3c6b71340a1d076831823f09cb6e02de01de5c6630a9631bdb36f947ff80" dependencies = [ "alloy-consensus", "alloy-eips", @@ -334,9 +350,9 @@ dependencies = [ [[package]] name = "alloy-provider" -version = "0.8.3" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc2dfaddd9a30aa870a78a4e1316e3e115ec1e12e552cbc881310456b85c1f24" +checksum = "5a22c4441b3ebe2d77fa9cf629ba68c3f713eb91779cff84275393db97eddd82" dependencies = [ "alloy-chains", "alloy-consensus", @@ -374,9 +390,9 @@ dependencies = [ [[package]] name = "alloy-pubsub" -version = "0.8.3" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "695809e743628d54510c294ad17a4645bd9f465aeb0d20ee9ce9877c9712dc9c" +checksum = "2269fd635f7b505f27c63a3cb293148cd02301efce4c8bdd9ff54fbfc4a20e23" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -415,9 +431,9 @@ dependencies = [ [[package]] name = "alloy-rpc-client" -version = "0.8.3" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "531137b283547d5b9a5cafc96b006c64ef76810c681d606f28be9781955293b6" +checksum = "d06a292b37e182e514903ede6e623b9de96420e8109ce300da288a96d88b7e4b" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -441,9 +457,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types" -version = "0.8.3" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3410a472ce26c457e9780f708ee6bd540b30f88f1f31fdab7a11d00bd6aa1aee" +checksum = "9383845dd924939e7ab0298bbfe231505e20928907d7905aa3bf112287305e06" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -454,9 +470,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-any" -version = "0.8.3" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed98e1af55a7d856bfa385f30f63d8d56be2513593655c904a8f4a7ec963aa3e" +checksum = "ca445cef0eb6c2cf51cfb4e214fbf1ebd00893ae2e6f3b944c8101b07990f988" dependencies = [ "alloy-consensus-any", "alloy-rpc-types-eth", @@ -465,9 +481,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-engine" -version = "0.8.3" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03bd16fa4959255ebf4a7702df08f325e5631df5cdca07c8a8e58bdc10fe02e3" +checksum = "4a5f821f30344862a0b6eb9a1c2eb91dfb2ff44c7489f37152a526cdcab79264" dependencies = [ "alloy-consensus", "alloy-eips", @@ -481,9 +497,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-eth" -version = "0.8.3" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8737d7a6e37ca7bba9c23e9495c6534caec6760eb24abc9d5ffbaaba147818e1" +checksum = "0938bc615c02421bd86c1733ca7205cc3d99a122d9f9bff05726bd604b76a5c2" dependencies = [ "alloy-consensus", "alloy-consensus-any", @@ -493,17 +509,17 @@ dependencies = [ "alloy-rlp", "alloy-serde", "alloy-sol-types", - "derive_more", "itertools 0.13.0", "serde", "serde_json", + "thiserror 2.0.6", ] [[package]] name = "alloy-serde" -version = "0.8.3" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5851bf8d5ad33014bd0c45153c603303e730acc8a209450a7ae6b4a12c2789e2" +checksum = "ae0465c71d4dced7525f408d84873aeebb71faf807d22d74c4a426430ccd9b55" dependencies = [ "alloy-primitives", "serde", @@ -512,9 +528,9 @@ dependencies = [ [[package]] name = "alloy-signer" -version = "0.8.3" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e10ca565da6500cca015ba35ee424d59798f2e1b85bc0dd8f81dafd401f029a" +checksum = "9bfa395ad5cc952c82358d31e4c68b27bf4a89a5456d9b27e226e77dac50e4ff" dependencies = [ "alloy-primitives", "async-trait", @@ -526,9 +542,9 @@ dependencies = [ [[package]] name = "alloy-signer-local" -version = "0.8.3" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47fababf5a745133490cde927d48e50267f97d3d1209b9fc9f1d1d666964d172" +checksum = "fbdc63ce9eda1283fcbaca66ba4a414b841c0e3edbeef9c86a71242fc9e84ccc" dependencies = [ "alloy-consensus", "alloy-network", @@ -615,9 +631,9 @@ dependencies = [ [[package]] name = "alloy-transport" -version = "0.8.3" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "538a04a37221469cac0ce231b737fd174de2fdfcdd843bdd068cb39ed3e066ad" +checksum = "d17722a198f33bbd25337660787aea8b8f57814febb7c746bc30407bdfc39448" dependencies = [ "alloy-json-rpc", "base64 0.22.1", @@ -635,9 +651,9 @@ dependencies = [ [[package]] name = "alloy-transport-http" -version = "0.8.3" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ed40eb1e1265b2911512f6aa1dcece9702d078f5a646730c45e39e2be00ac1c" +checksum = "6e1509599021330a31c4a6816b655e34bf67acb1cc03c564e09fd8754ff6c5de" dependencies = [ "alloy-json-rpc", "alloy-transport", @@ -650,9 +666,9 @@ dependencies = [ [[package]] name = "alloy-transport-ipc" -version = "0.8.3" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a172a59d24706b26a79a837f86d51745cb26ca6f8524712acd0208a14cff95" +checksum = "fa4da44bc9a5155ab599666d26decafcf12204b72a80eeaba7c5e234ee8ac205" dependencies = [ "alloy-json-rpc", "alloy-pubsub", @@ -669,9 +685,9 @@ dependencies = [ [[package]] name = "alloy-transport-ws" -version = "0.8.3" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fba0e39d181d13c266dbb8ca54ed584a2c66d6e9279afca89c7a6b1825e98abb" +checksum = "58011745b2f17b334db40df9077d75b181f78360a5bc5c35519e15d4bfce15e2" dependencies = [ "alloy-pubsub", "alloy-transport", @@ -904,6 +920,12 @@ dependencies = [ "rand", ] +[[package]] +name = "array-init-cursor" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf7d0a018de4f6aa429b9d33d69edf69072b1c5b1cb8d3e4a5f7ef898fc3eb76" + [[package]] name = "arrayvec" version = "0.7.6" @@ -966,6 +988,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atoi_simd" +version = "0.15.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae037714f313c1353189ead58ef9eec30a8e8dc101b2622d461418fd59e28a9" + [[package]] name = "atomic-waker" version = "1.1.2" @@ -1554,6 +1582,15 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bit-set" version = "0.5.3" @@ -1640,6 +1677,27 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "brotli" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -1674,6 +1732,26 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "bytemuck" +version = "1.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef657dfab802224e671f5818e9a4935f9b1957ed18e58292690cc39e7a4092a3" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fa76293b4f7bb636ab88fd78228235b5248b4d05cc589aed610f954af5d7c7a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "byteorder" version = "1.5.0" @@ -1735,6 +1813,24 @@ dependencies = [ "serde", ] +[[package]] +name = "capnp" +version = "0.19.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e985a566bdaae9a428a957d12b10c318d41b2afddb54cfbb764878059df636e" +dependencies = [ + "embedded-io", +] + +[[package]] +name = "capnpc" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c75ba30e0f08582d53c2f3710cf4bb65ff562614b1ba86906d7391adffe189ec" +dependencies = [ + "capnp", +] + [[package]] name = "cassowary" version = "0.3.0" @@ -1954,6 +2050,15 @@ dependencies = [ "warp", ] +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -2031,6 +2136,16 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + [[package]] name = "crossbeam-epoch" version = "0.9.18" @@ -2248,6 +2363,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" dependencies = [ + "convert_case", "proc-macro2", "quote", "syn 2.0.90", @@ -2310,6 +2426,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "dyn-clone" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" + [[package]] name = "ecdsa" version = "0.14.8" @@ -2384,6 +2506,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + [[package]] name = "encoding_rs" version = "0.8.35" @@ -2416,6 +2544,29 @@ dependencies = [ "log", ] +[[package]] +name = "envio-indexer" +version = "0.1.0" +dependencies = [ + "alloy", + "anyhow", + "aws-config", + "aws-sdk-sqs", + "aws-smithy-runtime-api", + "clap", + "dotenvy", + "env_logger", + "envy", + "hypersync-client", + "log", + "models", + "serde", + "serde_json", + "thiserror 2.0.6", + "tokio", + "url", +] + [[package]] name = "envy" version = "0.4.2" @@ -2452,6 +2603,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "ethnum" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b90ca2580b73ab6a1f724b76ca11ab632df820fd6040c336200d2c1df7b3c82c" + [[package]] name = "event-listener" version = "2.5.3" @@ -2469,12 +2626,39 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "fast-float" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95765f67b4b18863968b4a1bd5bb576f732b29a4a28c7cd84c09fa3e2875f33c" + +[[package]] +name = "faster-hex" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2a2b11eda1d40935b26cf18f6833c526845ae8c41e58d09af6adeb6f0269183" +dependencies = [ + "serde", +] + [[package]] name = "fastrand" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "fastrange-rs" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e90a1392cd6ec5ebe42ccaf251f2b7ba6be654c377f05c913f3898bfb2172512" + [[package]] name = "fastrlp" version = "0.3.1" @@ -2712,8 +2896,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -2870,6 +3056,8 @@ checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash 0.8.11", "allocator-api2", + "rayon", + "serde", ] [[package]] @@ -3158,6 +3346,7 @@ dependencies = [ "tokio", "tokio-rustls 0.26.1", "tower-service", + "webpki-roots 0.26.7", ] [[package]] @@ -3221,6 +3410,82 @@ dependencies = [ "tracing", ] +[[package]] +name = "hypersync-client" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6012e4303c710dd1889d031daac23b4f9e1b726c06d9f03fc192677ba285563" +dependencies = [ + "alloy-dyn-abi", + "alloy-json-abi", + "alloy-primitives", + "anyhow", + "arrayvec", + "bincode", + "capnp", + "faster-hex", + "fastrange-rs", + "futures 0.3.31", + "hypersync-format", + "hypersync-net-types", + "hypersync-schema", + "log", + "nohash-hasher", + "num_cpus", + "polars-arrow", + "polars-parquet", + "rand", + "rayon", + "reqwest 0.12.9", + "ruint", + "serde", + "serde_json", + "tokio", + "tokio-util", + "url", + "xxhash-rust", +] + +[[package]] +name = "hypersync-format" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eceee85251a8fb645ebfadf4af21768d257e5d92de567466df010a6069dbff8e" +dependencies = [ + "alloy-primitives", + "arrayvec", + "derive_more", + "faster-hex", + "nohash-hasher", + "sbbf-rs-safe", + "serde", + "thiserror 1.0.69", + "xxhash-rust", +] + +[[package]] +name = "hypersync-net-types" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2b51d8ac12eacb7fe1c63cf212435eaa360661da636386e2fe230bc56634c81" +dependencies = [ + "arrayvec", + "capnp", + "capnpc", + "hypersync-format", + "serde", +] + +[[package]] +name = "hypersync-schema" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5358157eac4bbc9fd6bc5bd26d74dc7fb8a7d85f0e5f48266d8d67231d9c24d5" +dependencies = [ + "anyhow", + "polars-arrow", +] + [[package]] name = "iana-time-zone" version = "0.1.61" @@ -3662,6 +3927,25 @@ dependencies = [ "hashbrown 0.15.2", ] +[[package]] +name = "lz4" +version = "1.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d1febb2b4a79ddd1980eede06a8f7902197960aa0383ffcfdd62fe723036725" +dependencies = [ + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "lzma-rs" version = "0.3.0" @@ -3730,6 +4014,15 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "memmap2" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f49388d20533534cd19360ad3d6a7dadc885944aa802ba3995040c5ec11288c6" +dependencies = [ + "libc", +] + [[package]] name = "metrics" version = "0.23.0" @@ -3825,6 +4118,8 @@ dependencies = [ "alloy", "async-trait", "chrono", + "hex", + "hypersync-client", "macon", "rand", "serde", @@ -3872,6 +4167,28 @@ dependencies = [ "version_check", ] +[[package]] +name = "multiversion" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4851161a11d3ad0bf9402d90ffc3967bf231768bfd7aeb61755ad06dbf1a142" +dependencies = [ + "multiversion-macros", + "target-features", +] + +[[package]] +name = "multiversion-macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79a74ddee9e0c27d2578323c13905793e91622148f138ba29738f9dddb835e90" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "target-features", +] + [[package]] name = "native-tls" version = "0.2.12" @@ -3889,6 +4206,12 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nohash-hasher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" + [[package]] name = "nom" version = "7.1.3" @@ -4152,6 +4475,16 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "parquet-format-safe" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1131c54b167dd4e4799ce762e1ab01549ebb94d5bdd13e6ec1b467491c378e1f" +dependencies = [ + "async-trait", + "futures 0.3.31", +] + [[package]] name = "paste" version = "1.0.15" @@ -4273,6 +4606,133 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" +[[package]] +name = "planus" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1691dd09e82f428ce8d6310bd6d5da2557c82ff17694d2a32cad7242aea89f" +dependencies = [ + "array-init-cursor", +] + +[[package]] +name = "polars-arrow" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32d19c6db79cb6a3c55af3b5a3976276edaab64cbf7f69b392617c2af30d7742" +dependencies = [ + "ahash 0.8.11", + "atoi_simd", + "bytemuck", + "chrono", + "dyn-clone", + "either", + "ethnum", + "fast-float", + "getrandom", + "hashbrown 0.14.5", + "itoa", + "lz4", + "multiversion", + "num-traits", + "parking_lot", + "polars-arrow-format", + "polars-error", + "polars-utils", + "ryu", + "simdutf8", + "streaming-iterator", + "strength_reduce", + "version_check", + "zstd", +] + +[[package]] +name = "polars-arrow-format" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b0ef2474af9396b19025b189d96e992311e6a47f90c53cd998b36c4c64b84c" +dependencies = [ + "planus", + "serde", +] + +[[package]] +name = "polars-compute" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30194a5ff325f61d6fcb62dc215c9210f308fc4fc85a493ef777dbcd938cba24" +dependencies = [ + "bytemuck", + "either", + "num-traits", + "polars-arrow", + "polars-error", + "polars-utils", + "strength_reduce", + "version_check", +] + +[[package]] +name = "polars-error" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07101d1803ca2046cdb3a8adb1523ddcc879229860f0ac56a853034269dec1e1" +dependencies = [ + "polars-arrow-format", + "simdutf8", + "thiserror 1.0.69", +] + +[[package]] +name = "polars-parquet" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb2993265079ffa07dd16277189444424f8d787b00b01c6f6e001f58bab543ce" +dependencies = [ + "ahash 0.8.11", + "async-stream", + "base64 0.22.1", + "brotli", + "bytemuck", + "ethnum", + "flate2", + "futures 0.3.31", + "lz4", + "num-traits", + "parquet-format-safe", + "polars-arrow", + "polars-compute", + "polars-error", + "polars-utils", + "simdutf8", + "snap", + "streaming-decompression", + "zstd", +] + +[[package]] +name = "polars-utils" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19dd73207bd15efb0ae5c9c3ece3227927ed6a16ad63578acec342378e6bdcb4" +dependencies = [ + "ahash 0.8.11", + "bytemuck", + "bytes", + "hashbrown 0.14.5", + "indexmap 2.7.0", + "memmap2", + "num-traits", + "once_cell", + "polars-error", + "raw-cpuid", + "rayon", + "smartstring", + "stacker", + "version_check", +] + [[package]] name = "portable-atomic" version = "1.10.0" @@ -4428,6 +4888,15 @@ version = "2.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" +[[package]] +name = "psm" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "200b9ff220857e53e184257720a14553b2f4aa02577d2ed9842d45d4b9654810" +dependencies = [ + "cc", +] + [[package]] name = "ptr_meta" version = "0.1.4" @@ -4469,6 +4938,58 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quinn" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef" +dependencies = [ + "bytes", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls 0.23.19", + "socket2", + "thiserror 2.0.6", + "tokio", + "tracing", +] + +[[package]] +name = "quinn-proto" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" +dependencies = [ + "bytes", + "getrandom", + "rand", + "ring", + "rustc-hash", + "rustls 0.23.19", + "rustls-pki-types", + "slab", + "thiserror 2.0.6", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c40286217b4ba3a71d644d752e6a0b71f13f1b6a2c5311acfcbe0c2418ed904" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.59.0", +] + [[package]] name = "quote" version = "1.0.37" @@ -4554,6 +5075,26 @@ dependencies = [ "bitflags 2.6.0", ] +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "recvmsg" version = "1.0.0" @@ -4697,7 +5238,10 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", + "quinn", + "rustls 0.23.19", "rustls-pemfile 2.2.0", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", @@ -4705,11 +5249,13 @@ dependencies = [ "system-configuration 0.6.1", "tokio", "tokio-native-tls", + "tokio-rustls 0.26.1", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", + "webpki-roots 0.26.7", "windows-registry", ] @@ -5011,6 +5557,9 @@ name = "rustls-pki-types" version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +dependencies = [ + "web-time", +] [[package]] name = "rustls-webpki" @@ -5066,6 +5615,25 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "sbbf-rs" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5525db49c7719816ac719ea8ffd0d0b4586db1a3f5d3e7751593230dacc642fd" +dependencies = [ + "cpufeatures", + "fastrange-rs", +] + +[[package]] +name = "sbbf-rs-safe" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9902ffeb2cff3f8c072c60c7d526ac9560fc9a66fe1dfc3c240eba5e2151ba3c" +dependencies = [ + "sbbf-rs", +] + [[package]] name = "schannel" version = "0.1.27" @@ -5423,6 +5991,23 @@ dependencies = [ "serde", ] +[[package]] +name = "smartstring" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29" +dependencies = [ + "autocfg", + "static_assertions", + "version_check", +] + +[[package]] +name = "snap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" + [[package]] name = "socket2" version = "0.5.8" @@ -5723,12 +6308,46 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "stacker" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799c883d55abdb5e98af1a7b3f23b9b6de8ecada0ecac058672d7635eb48ca7b" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.59.0", +] + [[package]] name = "static_assertions" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "streaming-decompression" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf6cc3b19bfb128a8ad11026086e31d3ce9ad23f8ea37354b31383a187c44cf3" +dependencies = [ + "fallible-streaming-iterator", +] + +[[package]] +name = "streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" + +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "stringprep" version = "0.1.5" @@ -5915,6 +6534,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "target-features" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1bbb9f3c5c463a01705937a24fdabc5047929ac764b2d5b9cf681c1f5041ed5" + [[package]] name = "tempfile" version = "3.14.0" @@ -6170,6 +6795,7 @@ checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" dependencies = [ "bytes", "futures-core", + "futures-io", "futures-sink", "pin-project-lite", "tokio", @@ -6555,6 +7181,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] @@ -6818,6 +7445,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-roots" version = "0.25.4" @@ -7132,6 +7769,12 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" +[[package]] +name = "xxhash-rust" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" + [[package]] name = "yansi" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index 9ad4125..cc66d3d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,18 +1,21 @@ [workspace] resolver = "2" -members = ["cli", "consumer", "histoflux", "image-guard", "models", "shared-utils", "substreams-sink"] +members = ["cli", "consumer", "envio-indexer", "histoflux", "image-guard", "models", "shared-utils", "substreams-sink"] [workspace.dependencies] -alloy = { version = "0.8.3", features = ["full"] } +alloy = { version = "0.9.2", features = ["full"] } +anyhow = "1.0.82" async-trait = "0.1.83" aws-config = { version = "1.0.1", features = ["behavior-version-latest"] } aws-sdk-sqs = { version = "1.3.0" } aws-smithy-runtime-api = "1.7.0" chrono = { version = "0.4.35", features = ["serde"] } +clap = { version = "4.5.20", features = ["derive"] } dotenvy = "0.15.7" envy = "0.4.2" env_logger = "0.11.5" hex = { version = "0.4.3", features = ["serde"] } +hypersync-client = "0.17.2" log = "0.4.20" macon = "1.2.0" rand = "0.8.5" @@ -33,3 +36,4 @@ strum_macros = "0.26.4" thiserror = "2.0.3" tokio = { version = "1.20.1", features = ["full"] } utoipa = { version = "5.2.0", features = ["macros", "axum_extras", "chrono"] } +url = "2.4.1" diff --git a/cli/Dockerfile b/cli/Dockerfile index b632741..6f3ca97 100644 --- a/cli/Dockerfile +++ b/cli/Dockerfile @@ -8,6 +8,7 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends \ pkg-config \ libssl-dev \ + capnproto \ && rm -rf /var/lib/apt/lists/* # Copy the entire workspace @@ -24,6 +25,7 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends \ ca-certificates \ libssl1.1 \ + capnproto \ && rm -rf /var/lib/apt/lists/* # Copy the binary from builder diff --git a/consumer/Dockerfile b/consumer/Dockerfile index 1d3e98b..98c23b6 100644 --- a/consumer/Dockerfile +++ b/consumer/Dockerfile @@ -18,6 +18,7 @@ RUN apt-get update && \ libssl-dev \ pkg-config \ curl \ + capnproto \ && rm -rf /var/lib/apt/lists/* RUN cargo chef cook --release --recipe-path recipe.json diff --git a/consumer/src/mode/decoded/redeemed.rs b/consumer/src/mode/decoded/redeemed.rs index 939d970..4356c28 100644 --- a/consumer/src/mode/decoded/redeemed.rs +++ b/consumer/src/mode/decoded/redeemed.rs @@ -34,7 +34,6 @@ impl Redeemed { &self, pg_pool: &PgPool, event: &DecodedMessage, - redemption_id: String, vault: &Vault, ) -> Result<(), ConsumerError> { if let Some(triple_id) = vault.triple_id.clone() { @@ -44,7 +43,7 @@ impl Redeemed { .block_number(U256Wrapper::try_from(event.block_number)?) .block_timestamp(event.block_timestamp) .transaction_hash(event.transaction_hash.clone()) - .redemption_id(redemption_id) + .redemption_id(DecodedMessage::event_id(event)) .triple_id(triple_id) .build() .upsert(pg_pool) @@ -56,7 +55,7 @@ impl Redeemed { .block_number(U256Wrapper::try_from(event.block_number)?) .block_timestamp(event.block_timestamp) .transaction_hash(event.transaction_hash.clone()) - .redemption_id(redemption_id) + .redemption_id(DecodedMessage::event_id(event)) .atom_id( vault .atom_id @@ -79,7 +78,7 @@ impl Redeemed { event: &DecodedMessage, ) -> Result { Redemption::builder() - .id(event.transaction_hash.clone()) + .id(DecodedMessage::event_id(event)) .sender_id(sender_account.id.clone()) .receiver_id(receiver_account.id.clone()) .sender_total_shares_in_vault(self.senderTotalSharesInVault) @@ -112,7 +111,7 @@ impl Redeemed { U256::ZERO.saturating_sub(self.assetsForReceiver), )) .triple_id(triple_id) - .redemption_id(event.log_index.to_string()) + .redemption_id(DecodedMessage::event_id(event)) .block_number(U256Wrapper::try_from(event.block_number)?) .block_timestamp(event.block_timestamp) .transaction_hash(event.transaction_hash.clone()) @@ -133,7 +132,7 @@ impl Redeemed { .clone() .ok_or(ConsumerError::VaultAtomNotFound)?, ) - .redemption_id(event.log_index.to_string()) + .redemption_id(DecodedMessage::event_id(event)) .block_number(U256Wrapper::try_from(event.block_number)?) .block_timestamp(event.block_timestamp) .transaction_hash(event.transaction_hash.clone()) @@ -171,14 +170,13 @@ impl Redeemed { get_or_create_account(self.receiver.to_string(), decoded_consumer_context).await?; // 2. Create redemption record - let redemption = self - .create_redemption_record( - &decoded_consumer_context.pg_pool, - &sender_account, - &receiver_account, - event, - ) - .await?; + self.create_redemption_record( + &decoded_consumer_context.pg_pool, + &sender_account, + &receiver_account, + event, + ) + .await?; // 3. Get vault and current share price let current_share_price = self @@ -209,13 +207,8 @@ impl Redeemed { .await?; // 6. Create event record - self.create_event( - &decoded_consumer_context.pg_pool, - event, - redemption.id, - &vault, - ) - .await?; + self.create_event(&decoded_consumer_context.pg_pool, event, &vault) + .await?; // 7. Create signal record self.create_signal(&decoded_consumer_context.pg_pool, event, &vault) diff --git a/docker-compose.yml b/docker-compose.yml index 57ab068..e175c0f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -248,31 +248,46 @@ services: exec ipfs daemon --migrate=true --agent-version-suffix=docker " - substreams-sink: - container_name: substreams-sink - image: ghcr.io/0xintuition/substreams-sink:latest - restart: always - entrypoint: ["./substreams-sink"] + # substreams-sink: + # container_name: substreams-sink + # image: ghcr.io/0xintuition/substreams-sink:latest + # restart: always + # entrypoint: ["./substreams-sink"] + # environment: + # LOCALSTACK_URL: $LOCALSTACK_URL + # AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID + # AWS_REGION: $AWS_REGION + # AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY + # DATABASE_URL: $DATABASE_URL + # RAW_CONSUMER_QUEUE_URL: $RAW_CONSUMER_QUEUE_URL + # SUBSTREAMS_ENDPOINT: $SUBSTREAMS_ENDPOINT + # SUBSTREAMS_PACKAGE: $SUBSTREAMS_PACKAGE + # SUBSTREAMS_MODULE: $SUBSTREAMS_MODULE + # SUBSTREAMS_START_BLOCK: $SUBSTREAMS_START_BLOCK + # SUBSTREAMS_API_TOKEN: $SUBSTREAMS_API_TOKEN + # command: ["$SUBSTREAMS_ENDPOINT", "$SUBSTREAMS_PACKAGE", "$SUBSTREAMS_MODULE", "$SUBSTREAMS_START_BLOCK:"] + # depends_on: + # sqs: + # condition: service_healthy + # hasura-migrations: + # condition: service_started + # volumes: + # - substreams-sink-data:/data + + envio-indexer: + container_name: envio-indexer + image: ghcr.io/0xintuition/envio-indexer:latest environment: LOCALSTACK_URL: $LOCALSTACK_URL AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID AWS_REGION: $AWS_REGION AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY - DATABASE_URL: $DATABASE_URL RAW_CONSUMER_QUEUE_URL: $RAW_CONSUMER_QUEUE_URL - SUBSTREAMS_ENDPOINT: $SUBSTREAMS_ENDPOINT - SUBSTREAMS_PACKAGE: $SUBSTREAMS_PACKAGE - SUBSTREAMS_MODULE: $SUBSTREAMS_MODULE - SUBSTREAMS_START_BLOCK: $SUBSTREAMS_START_BLOCK - SUBSTREAMS_API_TOKEN: $SUBSTREAMS_API_TOKEN - command: ["$SUBSTREAMS_ENDPOINT", "$SUBSTREAMS_PACKAGE", "$SUBSTREAMS_MODULE", "$SUBSTREAMS_START_BLOCK:"] + HYPERSYNC_TOKEN: $HYPERSYNC_TOKEN depends_on: sqs: condition: service_healthy - hasura-migrations: - condition: service_started - volumes: - - substreams-sink-data:/data + command: ["--network", "base-sepolia"] hasura-migrations: container_name: hasura-migrations diff --git a/envio-indexer/Cargo.toml b/envio-indexer/Cargo.toml new file mode 100644 index 0000000..ec1758f --- /dev/null +++ b/envio-indexer/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "envio-indexer" +version = "0.1.0" +edition = "2021" + +[dependencies] +alloy.workspace = true +anyhow.workspace = true +aws-config.workspace = true +aws-sdk-sqs.workspace = true +aws-smithy-runtime-api.workspace = true +clap.workspace = true +dotenvy.workspace = true +env_logger.workspace = true +envy.workspace = true +hypersync-client.workspace = true +log.workspace = true +models = { path = "../models" } +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +tokio.workspace = true +url.workspace = true \ No newline at end of file diff --git a/envio-indexer/Dockerfile b/envio-indexer/Dockerfile new file mode 100644 index 0000000..29aaa4d --- /dev/null +++ b/envio-indexer/Dockerfile @@ -0,0 +1,41 @@ +# Stage 1: Build +FROM rust:latest AS builder + +# Set the working directory +WORKDIR /usr/src/app + +# Copy the entire workspace +COPY . . +ENV SQLX_OFFLINE=true +# Install runtime dependencies only +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + libssl3 \ + capnproto \ + && rm -rf /var/lib/apt/lists/* + +# Build the release version of the crate +RUN cargo build --release --package envio-indexer + +# Stage 2: Runtime +FROM debian:bookworm-slim + +# Install runtime dependencies +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + libssl3 \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Set the working directory +WORKDIR /usr/src/app + +# Copy the compiled binary from the builder stage +COPY --from=builder /usr/src/app/target/release/envio-indexer . + +# Set runtime configs +ENV RUST_LOG=info + +# Run the binary +ENTRYPOINT ["/usr/src/app/envio-indexer"] diff --git a/envio-indexer/README.md b/envio-indexer/README.md new file mode 100644 index 0000000..7966e62 --- /dev/null +++ b/envio-indexer/README.md @@ -0,0 +1,12 @@ +You need to export your hypersync token in the `HYPERSYNC_TOKEN` environment variable. +You can create one here: https://envio.dev/app/api-tokens + +```bash +export HYPERSYNC_TOKEN=your_token +``` + +Run with + +```bash +cargo run -- --network base-sepolia +``` \ No newline at end of file diff --git a/envio-indexer/src/app.rs b/envio-indexer/src/app.rs new file mode 100644 index 0000000..c95c2f0 --- /dev/null +++ b/envio-indexer/src/app.rs @@ -0,0 +1,145 @@ +use crate::{error::IndexerError, Args, Network}; +use aws_sdk_sqs::Client as AWSClient; +use clap::Parser; +use hypersync_client::{net_types::Query, simple_types::Event, Client, ClientConfig}; +use log::info; +use models::raw_logs::RawLog; +use serde::Deserialize; +use tokio::time::{sleep, Duration}; +use url::Url; + +/// The environment variables +#[derive(Clone, Deserialize, Debug)] +pub struct Env { + pub hypersync_token: String, + pub localstack_url: Option, + pub raw_consumer_queue_url: String, +} + +/// The application +pub struct App { + pub client: Client, + pub args: Args, + pub aws_sqs_client: AWSClient, + pub raw_consumer_queue_url: String, +} + +impl App { + /// Initialize the application + pub async fn init() -> Result { + // Initialize the logger + env_logger::init(); + // Read the .env file from the current directory or parents + dotenvy::dotenv().ok(); + // Parse the .env file + let env = envy::from_env::()?; + // Parse the CLI arguments + let args = Args::parse(); + // Create the client for the given network + let client = Self::create_client(&env, &args)?; + // Get the current height of the server + let height = client.get_height().await?; + info!("Server height is {}", height); + // Create the SQS client + let aws_sqs_client = Self::get_aws_client(env.localstack_url.clone()).await; + // Get the raw consumer queue url + let raw_consumer_queue_url = env.raw_consumer_queue_url; + + Ok(Self { + client, + args, + aws_sqs_client, + raw_consumer_queue_url, + }) + } + + /// Create a client for the given network + pub fn create_client(env: &Env, args: &Args) -> Result { + Ok(match args.network { + Network::BaseSepolia => Client::new(ClientConfig { + url: Some(Url::parse("https://84532.hypersync.xyz")?), + bearer_token: Some(env.hypersync_token.clone()), + ..Default::default() + })?, + }) + } + + /// Create a query for the given network + pub fn query(&self) -> Result { + if self.args.network == Network::BaseSepolia { + Ok(serde_json::from_str(include_str!( + "queries/base_sepolia_query.json" + ))?) + } else { + Err(IndexerError::AnyhowError(anyhow::anyhow!( + "Invalid network" + ))) + } + } + + /// This function returns an [`aws_sdk_sqs::Client`] based on the + /// environment variables + pub async fn get_aws_client(localstack_url: Option) -> AWSClient { + let shared_config = if let Some(localstack_url) = localstack_url { + info!("Running SQS locally {:?}", localstack_url); + + aws_config::from_env() + .endpoint_url(localstack_url) + .load() + .await + } else { + aws_config::from_env().load().await + }; + + AWSClient::new(&shared_config) + } + + /// Process a batch of events + async fn process_events(&self, events: Vec) -> Result<(), IndexerError> { + for event in events { + let raw_log = RawLog::try_from(event)?; + let message = serde_json::to_string(&raw_log)?; + info!("{:#?}", message); + + self.aws_sqs_client + .send_message() + .queue_url(&self.raw_consumer_queue_url) + .message_group_id("raw") + .message_body(message) + .send() + .await?; + } + Ok(()) + } + + /// Start the indexer + pub async fn start_indexer(&self) -> Result<(), IndexerError> { + // Get the query for the given network + let mut query = self.query()?; + + loop { + let res = self.client.get_events(query.clone()).await?; + + // We can optimize this by processing the events in batches + for batch in res.data { + self.process_events(batch).await?; + } + + info!("scanned up to block {}", res.next_block); + + if let Some(archive_height) = res.archive_height { + if archive_height < res.next_block { + // wait if we are at the head + // notice we use explicit get_height in order to not waste data requests. + // get_height is lighter compared to spamming data requests at the tip. + while self.client.get_height().await? < res.next_block { + sleep(Duration::from_secs(1)).await; + } + } + } + + // continue query from next_block + query.from_block = res.next_block; + } + } +} diff --git a/envio-indexer/src/error.rs b/envio-indexer/src/error.rs new file mode 100644 index 0000000..1cd2e46 --- /dev/null +++ b/envio-indexer/src/error.rs @@ -0,0 +1,23 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum IndexerError { + #[error(transparent)] + AnyhowError(#[from] anyhow::Error), + #[error(transparent)] + AWSSendMessage( + #[from] + aws_smithy_runtime_api::client::result::SdkError< + aws_sdk_sqs::operation::send_message::SendMessageError, + aws_smithy_runtime_api::http::Response, + >, + ), + #[error(transparent)] + EnvError(#[from] envy::Error), + #[error(transparent)] + ModelError(#[from] models::error::ModelError), + #[error(transparent)] + ParseError(#[from] url::ParseError), + #[error(transparent)] + SerdeJsonError(#[from] serde_json::Error), +} diff --git a/envio-indexer/src/main.rs b/envio-indexer/src/main.rs new file mode 100644 index 0000000..79258a9 --- /dev/null +++ b/envio-indexer/src/main.rs @@ -0,0 +1,28 @@ +use app::App; +use clap::{Parser, ValueEnum}; +use error::IndexerError; +use log::info; + +mod app; +mod error; + +/// The network to index. Currently only Base Sepolia is supported. +#[derive(ValueEnum, Clone, Debug, PartialEq, Eq)] +enum Network { + BaseSepolia, +} + +/// The CLI arguments +#[derive(Parser)] +pub struct Args { + /// The network to index + #[arg(short, long)] + network: Network, +} + +#[tokio::main] +async fn main() -> Result<(), IndexerError> { + let app = App::init().await?; + info!("Starting indexer"); + app.start_indexer().await +} diff --git a/envio-indexer/src/queries/base_sepolia_query.json b/envio-indexer/src/queries/base_sepolia_query.json new file mode 100644 index 0000000..9c2ecf4 --- /dev/null +++ b/envio-indexer/src/queries/base_sepolia_query.json @@ -0,0 +1,31 @@ +{ + "from_block": 0, + "logs": [ + { + "address": ["0x1A6950807E33d5bC9975067e6D6b5Ea4cD661665"] + } + ], + "field_selection": { + "block": [ + "number", + "timestamp", + "hash" + ], + "log": [ + "block_number", + "log_index", + "transaction_index", + "data", + "address", + "topic0", + "topic1", + "topic2", + "topic3" + ], + "transaction": [ + "block_number", + "transaction_index", + "hash" + ] + } + } \ No newline at end of file diff --git a/hasura/migrations/intuition/1729947331632_basic_schema/up.sql b/hasura/migrations/intuition/1729947331632_basic_schema/up.sql index 2e44770..7a107b5 100644 --- a/hasura/migrations/intuition/1729947331632_basic_schema/up.sql +++ b/hasura/migrations/intuition/1729947331632_basic_schema/up.sql @@ -211,7 +211,7 @@ CREATE TABLE signal ( block_number NUMERIC(78, 0) NOT NULL, block_timestamp BIGINT NOT NULL, transaction_hash BYTEA NOT NULL, - -- Ensure that exactly one of atom_id, triple_id, deposit_id, or redemption_id is set + -- Ensure that exactly one of atom_id or triple_id is set CONSTRAINT check_signal_constraints CHECK ( ((atom_id IS NOT NULL AND triple_id IS NULL) OR diff --git a/histoflux/Cargo.toml b/histoflux/Cargo.toml index ae4484e..854168c 100644 --- a/histoflux/Cargo.toml +++ b/histoflux/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2021" [dependencies] -clap = { version = "4.5.20", features = ["derive"] } +clap.workspace = true zip = "2.2.0" anyhow = "1.0" serde = { version = "1.0", features = ["derive"] } diff --git a/image-guard/Dockerfile b/image-guard/Dockerfile index b8dd2b1..e3820c1 100644 --- a/image-guard/Dockerfile +++ b/image-guard/Dockerfile @@ -8,6 +8,7 @@ RUN apt-get update && apt-get install -y \ pkg-config \ libssl-dev \ curl \ + capnproto \ && rm -rf /var/lib/apt/lists/* # Copy the entire workspace diff --git a/models/Cargo.toml b/models/Cargo.toml index c20079d..225dddd 100644 --- a/models/Cargo.toml +++ b/models/Cargo.toml @@ -7,6 +7,8 @@ edition = "2021" alloy.workspace = true async-trait.workspace = true chrono.workspace = true +hex.workspace = true +hypersync-client.workspace = true macon.workspace = true rand.workspace = true serde.workspace = true diff --git a/models/src/error.rs b/models/src/error.rs index 60fac90..957a267 100644 --- a/models/src/error.rs +++ b/models/src/error.rs @@ -12,8 +12,14 @@ pub enum ModelError { InsertError(String), #[error("Invalid atom type: {0}")] InvalidAtomType(String), + #[error("Missing field: {0}")] + MissingField(String), #[error("Failed to query data: {0}")] QueryError(String), + #[error("Failed to parse data: {0}")] + ParseError(String), + #[error("Failed to serialize data: {0}")] + SerializeError(String), #[error(transparent)] SqlError(#[from] sqlx::Error), #[error("Unexpected null value: {0}")] diff --git a/models/src/raw_logs.rs b/models/src/raw_logs.rs index 3398acd..95eacaf 100644 --- a/models/src/raw_logs.rs +++ b/models/src/raw_logs.rs @@ -1,6 +1,8 @@ use chrono::{DateTime, Utc}; +use hypersync_client::simple_types::Event; use serde::{Deserialize, Deserializer, Serialize}; use sqlx::PgPool; +use std::convert::TryFrom; use crate::error::ModelError; @@ -78,3 +80,100 @@ impl RawLog { .map_err(|error| ModelError::InsertError(error.to_string())) } } + +/// We use this to convert an event from the hypersync client to a raw log. +/// This is a try from because we want to handle errors gracefully. Currently +/// we are using this in the envio-indexer to convert events to raw logs. +impl TryFrom for RawLog { + type Error = ModelError; + + /// This is the implementation of the try from trait. + fn try_from(event: Event) -> Result { + /// This is a helper function to parse a hex string to an i64. + fn parse_hex_to_i64(hex_str: &str, field_name: &str) -> Result { + i64::from_str_radix(hex_str.trim_start_matches("0x"), 16) + .map_err(|e| ModelError::ParseError(format!("Error parsing {}: {}", field_name, e))) + } + + /// This is a helper function to serialize a value to a string. + fn serialize_to_string(value: &T) -> Result { + serde_json::to_string(value) + .map(|s| s.trim_matches('"').to_string()) + .map_err(|e| ModelError::SerializeError(e.to_string())) + } + + let block_number = event + .block + .as_ref() + .ok_or(ModelError::MissingField("block number".to_string()))? + .number + .ok_or(ModelError::MissingField("block number".to_string()))?; + + let transaction_index = parse_hex_to_i64( + &event + .log + .transaction_index + .ok_or(ModelError::MissingField("transaction index".to_string()))? + .to_string(), + "transaction index", + )?; + + let log_index = parse_hex_to_i64( + &event + .log + .log_index + .ok_or(ModelError::MissingField("log index".to_string()))? + .to_string(), + "log index", + )?; + + let block_timestamp = parse_hex_to_i64( + &hex::encode( + event + .block + .as_ref() + .ok_or(ModelError::MissingField("block".to_string()))? + .timestamp + .clone() + .ok_or(ModelError::MissingField("timestamp".to_string()))? + .as_ref(), + ), + "block timestamp", + )?; + + let block_hash = serialize_to_string( + &event + .block + .ok_or(ModelError::MissingField("block".to_string()))? + .hash, + )?; + let transaction_hash = serialize_to_string(&event.log.transaction_hash)?; + let address = serialize_to_string(&event.log.address)?; + let data = hex::encode( + event + .log + .data + .ok_or(ModelError::MissingField("data".to_string()))? + .as_ref(), + ); + let topics = event + .log + .topics + .iter() + .map(|t| hex::encode(t.as_ref().map(|d| d.as_ref()).unwrap_or_default())) + .filter(|s| s != "null" && !s.is_empty()) + .collect::>(); + + Ok(RawLog::builder() + .block_number(block_number as i64) + .block_hash(block_hash) + .block_timestamp(block_timestamp) + .transaction_hash(transaction_hash) + .transaction_index(transaction_index) + .log_index(log_index) + .address(address) + .data(data) + .topics(topics) + .build()) + } +} diff --git a/substreams-sink/Dockerfile b/substreams-sink/Dockerfile index 69ff4ca..712a2ca 100644 --- a/substreams-sink/Dockerfile +++ b/substreams-sink/Dockerfile @@ -18,6 +18,7 @@ RUN apt-get update && \ libssl-dev \ pkg-config \ curl \ + capnproto \ && rm -rf /var/lib/apt/lists/* RUN cargo chef cook --release --recipe-path recipe.json @@ -37,6 +38,7 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends \ ca-certificates \ libssl3 \ + capnproto \ && rm -rf /var/lib/apt/lists/* # Copy binary from builder