diff --git a/.env.sample b/.env.sample index 0f43a43..0c8f824 100644 --- a/.env.sample +++ b/.env.sample @@ -1,11 +1,19 @@ +# Pocket RPC Configuration POKT_RPC_FULL_HOST= -HTTP_SERVER_PORT=8080 POKT_RPC_TIMEOUT=5s + +# Gateway Deployment Configuration +HTTP_SERVER_PORT=8080 ENVIRONMENT_STAGE=development -POKT_APPLICATIONS_ENCRYPTION_KEY= +EMIT_SERVICE_URL_PROM_METRICS=false + +# Pocket Business Logic +CHAIN_NETWORK=morse_mainnet SESSION_CACHE_TTL=75m -DB_CONNECTION_URL=postgres://myuser:mypassword@postgres:5432/postgres?sslmode=disable -API_KEY= ALTRUIST_REQUEST_TIMEOUT=10s -EMIT_SERVICE_URL_PROM_METRICS=false -CHAIN_NETWORK=morse_mainnet \ No newline at end of file +API_KEY= + +# App Stake Management +DB_CONNECTION_URL=postgres://myuser:mypassword@postgres:5433/postgres?sslmode=disable +POKT_APPLICATIONS_ENCRYPTION_KEY= +POKT_APPLICATION_PRIVATE_KEY= diff --git a/.gitignore b/.gitignore index 4b1be86..d62a6c3 100644 --- a/.gitignore +++ b/.gitignore @@ -53,4 +53,10 @@ docker-compose.yml ## MAC -.DS_Store \ No newline at end of file +.DS_Store + +## VSCode +.vscode + +## Copmiled binary +main diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d8d096a --- /dev/null +++ b/Makefile @@ -0,0 +1,40 @@ +include .env + +######################## +### Makefile Helpers ### +######################## + +.PHONY: prompt_user +# Internal helper target - prompt the user before continuing +prompt_user: + @echo "Are you sure? [y/N] " && read ans && [ $${ans:-N} = y ] + +.PHONY: list +list: ## List all make targets + @${MAKE} -pRrn : -f $(MAKEFILE_LIST) 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | sort + +.PHONY: help +.DEFAULT_GOAL := help +help: ## Prints all the targets in all the Makefiles + @grep -h -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-60s\033[0m %s\n", $$1, $$2}' + +######################## +### Database Helpers ### +######################## + +.PHONY: db_migrate +db_migrate: ## Run database migrations + @echo "Running database migrations..." + ./scripts/db_migrate.sh -u + +.PHONY: db_migrate +db_migrate: ## Run database migrations + @echo "Running database migrations..." + ./scripts/db_migrate.sh -u\ + + +PG_CMD := INSERT INTO pokt_applications (encrypted_private_key) VALUES (pgp_sym_encrypt('$(POKT_APPLICATION_PRIVATE_KEY)', '$(POKT_APPLICATIONS_ENCRYPTION_KEY)')); +db_insert_app_private_key: ## Insert application private key into database + @echo "Running SQL command..." + @echo "$(PG_CMD)" + @psql "$(DB_CONNECTION_URL)" -c "$(PG_CMD)" diff --git a/README.md b/README.md index 20f1a45..30c71eb 100644 --- a/README.md +++ b/README.md @@ -2,114 +2,180 @@ POKT Gateway Server -## What is POKT Gateway Server? +# What is POKT Gateway Server? -The POKT Gateway Server is a comprehensive solution designed to simplify the integration of applications with the POKT Network. Our goal is to reduce the complexities associated with directly interfacing with the protocol, making it accessible to a wide range of users, including application developers, existing centralized RPC platforms, and future gateway operators. +_tl;dr Streamline access to POKT Network's decentralized supply network._ -Learn more about the vision and overall architecture [overview](docs%2Foverview.md) +The POKT Gateway Server is a comprehensive solution designed to simplify the integration of applications with POKT Network. Its goal is to reduce the complexities associated with directly interfacing with the protocol, making it accessible to a wide range of users, including application developers, existing centralized RPC platforms, and future gateway operators. -## Gateway Operator Quick Getting Started -To onboard the gateway server without having to dig deep, you can follow the [Quick Onboarding Guide](docs%2Fquick-onboarding-guide.md) +Learn more about the vision and overall architecture [overview](./docs/overview.md). -#### Interested in learning more? We have an abdundance of information in [docs](docs) -1. [Gateway Server Overview](docs%2Foverview.md) -2. [Gateway Server API Endpoints](docs%2Fapi-endpoints.md) -3. [Gateway Server System Architecture](docs%2Fsystem-architecture.md) -4. [Gateway Server Node Selection](docs%2Fnode-selection.md) -5. [POKT Primer](docs%2Fpokt-primer.md) -6. [POKT's Relay Specification](docs%2Fpokt-relay-specification.md) +- [Gateway Operator Quickstart Guide](#gateway-operator-quickstart-guide) + - [Interested in learning more?](#interested-in-learning-more) +- [Docker Image Releases](#docker-image-releases) +- [Docker Compose](#docker-compose) +- [Minimum Hardware Requirements](#minimum-hardware-requirements) +- [Database Migrations](#database-migrations) + - [Creating a DB Migration](#creating-a-db-migration) + - [Applying a DB Migration](#applying-a-db-migration) + - [DB Migration helpers](#db-migration-helpers) + - [Applying Migrations](#applying-migrations) + - [Migrations Rollbacks](#migrations-rollbacks) +- [Unit Testing](#unit-testing) + - [Generating Mocks](#generating-mocks) + - [Running Tests](#running-tests) +- [Generating DB Queries](#generating-db-queries) +- [Contributing Guidelines](#contributing-guidelines) +- [Project Structure](#project-structure) + +## Gateway Operator Quickstart Guide + +To onboard the gateway server without having to dig deep, you can follow the [Quick Onboarding Guide](docs/quick-onboarding-guide.md). + +### Interested in learning more? + +We have an abundance of information in the [docs](docs) section: + +1. [Gateway Server Overview](docs/overview.md) +2. [Gateway Server API Endpoints](docs/api-endpoints.md) +3. [Gateway Server System Architecture](docs/system-architecture.md) +4. [Gateway Server Node Selection](docs/node-selection.md) +5. [POKT Primer](docs/pokt-primer.md) +6. [POKT's Relay Specification](docs/pokt-relay-specification.md) ## Docker Image Releases -Every release candidate is published to https://github.com/pokt-network/gateway-server/pkgs/container/pocket-gateway-server + +Every release candidate is published to [gateway-server/pkgs/container/pocket-gateway-server](https://github.com/pokt-network/gateway-server/pkgs/container/pocket-gateway-server). ## Docker Compose + There is an all-inclusive docker-compose file available for development [docker-compose.yml](docker-compose.yml.sample) -## Minimum Hardware Requirements to run +## Minimum Hardware Requirements + +To run a Gateway Server, we recommend the following minimum hardware requirements: + - 1GB of RAM - 1GB of storage - 4 vCPUs+ In production, we have observed memory usage increase to 4GB+. The memory footprint will be dependent on the number of app stakes/chains staked and total traffic throughput. -## Creating a DB Migration +## Database Migrations + + + +### Creating a DB Migration + Migrations are like version control for your database, allowing your team to define and share the application's database schema definition. -Before running a migration make sure to install the go lang migration cli on your machine. -https://github.com/golang-migrate/migrate/tree/master/cmd/migrate + +Before running a migration make sure to install the go lang migration cli on your machine. See [golang-migrate/migrate/tree/master/cmd/migrate](https://github.com/golang-migrate/migrate/tree/master/cmd/migrate) for reference. + ```sh ./scripts/migration.sh -n {migration_name} ``` + This command will generate a up and down migration in `db_migrations` -## Applying a DB Migration +### Applying a DB Migration + DB Migrations are applied upon server start, but as well, it can be applied manually through: + ```sh -./scripts/migration.sh {--down or --up} {number_of_times} +./scripts/migration.sh {--down or --up} {number_of_times} ``` -### Usage +### DB Migration helpers -#### Apply Migrations +#### Applying Migrations - To apply all migrations: + ```sh ./scripts/migration.sh --up ``` - To apply a specific number of migrations: + ```sh ./scripts/migration.sh --up 2 ``` -#### Rollback Migrations +#### Migrations Rollbacks + Make sure to provide either the number of migrations to rollback or the `--all` flag to rollback all migrations. - To roll back a specific number of migrations: + ```sh ./scripts/migration.sh --down 2 ``` - To roll back all migrations: + ```sh ./scripts/migration.sh --down --all ``` -## Running Tests +## Unit Testing + +### Generating Mocks + Install Mockery with -``` + +```bash go install github.com/vektra/mockery/v2@v2.40.1 ``` + You can generate the mock files through: + ```sh ./scripts/mockgen.sh ``` + By running this command, it will generate the mock files in `./mocks` folder. -Reference for mocks can be found here https://vektra.github.io/mockery/latest/ + +Reference for mocks can be found [here](https://vektra.github.io/mockery/latest). + +### Running Tests Run this command to run tests: + ```sh -go test ./... +go test -v -count=1 ./... ``` ## Generating DB Queries -Gateway server uses [PGGen]('https://github.com/jschaf/pggen) to create autogenerated type-safe queries. -Queries are added inside [queries.sql](internal%2Fdb_query%2Fqueries.sql) and then regenerated via `./scripts/querygen.sh`. + +Gateway server uses [PGGen](https://github.com/jschaf/pggen) to create autogenerated type-safe queries. +Queries are added inside [queries.sql](./internal/Fdb_query/queries.sql) and re-generated via `./scripts/querygen.sh`. ## Contributing Guidelines + 1. Create a Github Issue on the feature/issue you're working on. 2. Fork the project 3. Create new branch with `git checkout -b "branch_name"` where branch name describes the feature. - - All branches should be based off `main` -3. Write your code -4. Make sure your code lints with `go fmt ./...` (This will Lint and Prettify) -5. Commit code to your branch and issue a pull request and wait for at least one review. - - Always ensure changes are rebased on top of main branch. + - All branches should be based off `main` +4. Write your code +5. Make sure your code lints with `go fmt ./...` (This will Lint and Prettify) +6. Commit code to your branch and issue a pull request and wait for at least one review. + - Always ensure changes are rebased on top of main branch. ---- ## Project Structure -- **cmd:** Contains the entry point of the binaries - - **gateway_server:** HTTP Server for serving requests -- **internal:** Shared internal folder for all binaries -- **pkg:** Distributable dependencies -- **docs:** Project documentation and specifications +A partial high-level view of the code structure (generated) + +```bash +. +├── cmd # Contains the entry point of the binaries +│   └── gateway_server # HTTP Server for serving requests +├── internal # Shared internal folder for all binaries +├── pkg # Distributable dependencies +└── scripts # Contains scripts for development +``` + +_Generate via `tree -L 2`_ + +--- + +Olshansky's stream of thought here: https://www.notion.so/buildwithgrove/Olshansky-PATH-Journal-dfc92813ca5945ac88035cca97ccbff5?pvs=4 diff --git a/cmd/gateway_server/internal/transform/qos_node.go b/cmd/gateway_server/internal/transform/qos_node.go index 0492604..9c7633e 100644 --- a/cmd/gateway_server/internal/transform/qos_node.go +++ b/cmd/gateway_server/internal/transform/qos_node.go @@ -1,9 +1,10 @@ package transform import ( + "math" + "github.com/pokt-network/gateway-server/cmd/gateway_server/internal/models" internal_model "github.com/pokt-network/gateway-server/internal/node_selector_service/models" - "math" ) func ToPublicQosNode(node *internal_model.QosNode) *models.PublicQosNode { diff --git a/docs/altruist-chain-configuration.md b/docs/altruist-chain-configuration.md index b3e43ac..d8c12bf 100644 --- a/docs/altruist-chain-configuration.md +++ b/docs/altruist-chain-configuration.md @@ -1,20 +1,31 @@ -# Altruist (Failover) Request +# Chain & Altruist Configurations + +## Altruist (Failover) Request + +- [Altruist (Failover) Request](#altruist-failover-request) +- [Chain Configuration](#chain-configuration) +- [Inserting a custom chain configuration](#inserting-a-custom-chain-configuration) + In rare situations, a relay cannot be served from POKT Network. Some sample scenarios for when this can happen: + 1. Dispatcher Outages - Gateway server cannot retrieve the necessary information to send a relay 2. Bad overall QoS - Majority of Node operators may have their nodes misconfigured improperly or there is a lack of node operators supporting the chain with respect to load. 3. Chain halts - In extreme conditions, if the chain halts, node operators may stop responding to relay requests So the Gateway Server will attempt to route the traffic to a backup chain node. This could be any source, for example: + 1. Other gateway operators chain urls 2. Centralized Nodes -# Chain Configuration +## Chain Configuration + Given that every chain has differences and have different sources for fail over, the gateway server allows for optional customization for request timeouts, failover relay, and QoS checks. -The data is stored inside the `chain_configuration` table and is accessed via the [chain_configurations_registry_service.go](..%2Finternal%2Fchain_configurations_registry%2Fchain_configurations_registry_service.go). +The data is stored inside the `chain_configuration` table and is accessed via the [chain_configurations_registry_service.go](../internal/chain_configurations_registry/chain_configurations_registry_service.go). + +_While it is **recommended** that you provide a chain configuration, the gateway server will assume defaults provided from the specified [config_provider.go](../internal/global_config/config_provider.go) and the provided QoS [checks](../internal/node_selector_service/checks)_ if not provided. -_While it is **recommended** that you provide a chain configuration, the gateway server will assume defaults provided from the specified [config_provider.go](..%2Finternal%2Fglobal_config%2Fconfig_provider.go) and the provided QoS [checks](..%2Finternal%2Fnode_selector_service%2Fchecks)_ if not provided. +## Inserting a custom chain configuration -# Inserting a custom chain configuration ```sql -- Insert an example configuration for Ethereum -- INSERT INTO chain_configurations (chain_id, pocket_request_timeout_duration, altruist_url, altruist_request_timeout_duration, top_bucket_p90latency_duration, height_check_block_tolerance, data_integrity_check_lookback_height) VALUES ('0000', '15s', 'https://example.com', '30s', '150ms', 100, 25); @@ -22,8 +33,8 @@ INSERT INTO chain_configurations (chain_id, pocket_request_timeout_duration, alt - `chain_id` - id of the Pocket Network Chain - `pocket_request_time` - duration of the maximum amount of time for a network relay to respond -- `altruist_url` - source of the relay in the event that a network request fails +- `altruist_url` - source of the relay in the event that a network request fails - `altruist_request_timeout_duration` - duration of the maximum amount of time for a backup request to respond -- `top_bucket_p90latency_duration` - maximum amount of latency for nodes to be favored 0 <= x <= `top_bucket_p90latency_duration` +- `top_bucket_p90latency_duration` - maximum amount of latency for nodes to be favored 0 <= x <= `top_bucket_p90latency_duration` - `height_check_block_tolerance` - number of blocks a node is allowed to be behind (some chains may have node operators moving faster than others) - `data_integrity_check_lookback_height` - number of blocks data integrity will look behind for source of truth block for other node operators to attest too diff --git a/docs/api-endpoints.md b/docs/api-endpoints.md index a9b965b..5d582f7 100644 --- a/docs/api-endpoints.md +++ b/docs/api-endpoints.md @@ -1,16 +1,77 @@ -# POKT Gateway Server API Endpoints +# POKT Gateway Server API Endpoints -The Gateway Server currently exposes all its API endpoints in form of HTTP endpoints. Note, we can move this to Swagger in the future if our API endpoints become more complex. +The Gateway Server currently exposes all its API endpoints in form of HTTP endpoints. Postman collection can be found [here](https://www.postman.com/dark-shadow-851601/workspace/os-gateway/collection/27302708-537f3ba3-3193-4290-98d0-0d5836988a2f) -`x-api-key` is an api key set by the gateway operator to transmit internal private data +`x-api-key` is an api key set by the gateway operator to transmit internal private data -| Endpoint | HTTP METHOD | Description | HEADERS | Request Parameters | -|----------------------|-------------|----------------------------------------------------------------------------------------------------------------------------------------------------|-------------|------------------------------------------| -| `/relay/{chain_id}` | ANY | The main endpoint for your reverse proxy to send requests too | ANY | `{chain_id}` - Network identifier | -| `/metrics` | GET | Metadata on the gateway server performance for observability purposes | N/A | N/A | -| `/poktapps` | GET | A list of all the available app stakes | `x-api-key` | N/A | -| `/poktapps` | POST | Adds an existing app stake to the appstake database (not recommended due to security) | `x-api-key` | `private_key` - private key of app stake | -| `/poktapps/{app_id}` | DELETE | Removes an existing app stake from the appstake database (not recommended due to security) | `x-api-key` | `app_id` - id of the appstake | -| `/qosnodes` | GET | A list of nodes and public QoS state such as healthiness and last known error. This can be used to expose to node operators to improve visibility. | `x-api-key` | N/A | \ No newline at end of file +_TODO_IMPROVE: Move this to Swagger in the future if our API endpoints become more complex._ +_TODO_IMPROVE: Add an OpenAPI spec if the admin endpoints are kept and/or expanded on.._ + +- [API Endpoints](#api-endpoints) +- [Examples](#examples) + - [Relay](#relay) + - [Metrics](#metrics) + - [PoktApps](#poktapps) + - [List](#list) + - [Add](#add) + - [Delete](#delete) + - [QoS Noes](#qos-noes) + +## API Endpoints + +| Endpoint | HTTP METHOD | Description | HEADERS | Request Parameters | +| -------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | ----------- | ---------------------------------------- | +| `/relay/{chain_id}` | ANY | The main endpoint to send relays to | ANY | `{chain_id}` - Network identifier | +| `/metrics` | GET | Gateway metadata related to server performance and observability | N/A | N/A | +| `/poktapps` | GET | List all the available app stakes | `x-api-key` | N/A | +| `/poktapps` | POST | Add an existing app stake to the appstake database (not recommended due to security) | `x-api-key` | `private_key` - private key of app stake | +| `/poktapps/{app_id}` | DELETE | Remove an existing app stake from the appstake database (not recommended due to security) | `x-api-key` | `app_id` - id of the appstake | +| `/qosnodes` | GET | List of nodes and public QoS state such as healthiness and last known error. This can be used to expose to node operators to improve visibility. | `x-api-key` | N/A | + +## Examples + +These examples assume gateway server is running locally. + +### Relay + +```bash +curl -X POST -H "Content-Type: application/json" \ + --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \ + http://localhost:8080/relay/0021 +``` + +### Metrics + +```bash +curl -X GET http://localhost:8080/metrics +``` + +### PoktApps + +Make sure that the gateway server starts up with the `API_KEY` environment variable set. + +#### List + +```bash +curl -X GET http://localhost:8080/poktapps +``` + +#### Add + +```bash +curl -X POST -H "x-api-key: $API_KEY" https://localhost:8080/poktapps/{private_key} +``` + +#### Delete + +```bash +curl -X DELETE -H "x-api-key: $API_KEY" https://localhost:8080/poktapps/{app_id} +``` + +#### QoS Noes + +```bash +curl -X GET -H "x-api-key: $API_KEY" http://localhost:8080/qosnodes +``` diff --git a/docs/benchmarks/03_2024/03-2024-benchmark.md b/docs/benchmarks/03_2024/03-2024-benchmark.md index 6a4646b..d5ca2c7 100644 --- a/docs/benchmarks/03_2024/03-2024-benchmark.md +++ b/docs/benchmarks/03_2024/03-2024-benchmark.md @@ -1,9 +1,11 @@ # March 2024 Benchmark (RC 0.2.0) ## Benchmark Purpose + The purpose of this benchmark is to comprehensively assess the performance metrics, particularly CPU and Memory behaviors, incurred while serving requests through the gateway server. Specifically, this evaluation aims to gauge the efficiency of various operations involved with sending a relay such as JSON serialization, IO handling, cryptographic signing, and asynchronous background processes (QoS checks). ## Benchmark Environment + - **POKT Testnet**: The benchmark is conducted within the environment of the POKT Testnet to simulate real-world conditions accurately. - **RPC Method web3_clientVersion**: The benchmark uses a consistent time RPC method, web3_clientVersion, chosen deliberately to isolate the impact of the gateway server overhead. It's noteworthy that the computational overhead of sending a request to the POKT Network remains independent of the specific RPC Method employed. - **Gateway Server Hardware**: The gateway server is deployed on a dedicated DigitalOcean droplet instance (16 GB Memory / 8 Prem. Intel vCPUs / 100 GB Disk / FRA1), ensuring controlled conditions for performance evaluation. @@ -14,11 +16,13 @@ The purpose of this benchmark is to comprehensively assess the performance metri ## Scripts Load Testing Command + ```sh vegeta attack -duration=180s -rate=100/1s -targets=gateway_server.config | tee results.bin | vegeta report ``` Vegeta Target + ```sh POST http://{endpoint} Content-Type: application/json @@ -26,12 +30,15 @@ Content-Type: application/json ``` Payload.json + ```json -{"jsonrpc":"2.0","method":"web3_clientVersion","params":[],"id":1} +{ "jsonrpc": "2.0", "method": "web3_clientVersion", "params": [], "id": 1 } ``` ## Load Test Results + 100 RPS + ```text Requests [total, rate, throughput] 18000, 100.01, 99.87 Duration [total, attack, wait] 3m0s, 3m0s, 239.445ms @@ -39,11 +46,12 @@ Latencies [min, mean, 50, 90, 95, 99, max] 170.168ms, 191.573ms, 176.331ms, Bytes In [total, mean] 1260000, 70.00 Bytes Out [total, mean] 1206000, 67.00 Success [ratio] 100.00% -Status Codes [code:count] 200:18000 +Status Codes [code:count] 200:18000 Error Set: ``` 500 RPS + ```text Requests [total, rate, throughput] 90000, 500.01, 499.51 Duration [total, attack, wait] 3m0s, 3m0s, 176.636ms @@ -51,11 +59,12 @@ Latencies [min, mean, 50, 90, 95, 99, max] 169.036ms, 182.464ms, 176.267ms, Bytes In [total, mean] 6300000, 70.00 Bytes Out [total, mean] 6030000, 67.00 Success [ratio] 100.00% -Status Codes [code:count] 200:90000 +Status Codes [code:count] 200:90000 Error Set: ``` 1000 RPS + ```text Requests [total, rate, throughput] 180000, 1000.00, 998.87 Duration [total, attack, wait] 3m0s, 3m0s, 204.308ms @@ -63,32 +72,36 @@ Latencies [min, mean, 50, 90, 95, 99, max] 168.406ms, 183.103ms, 176.947ms, Bytes In [total, mean] 12600000, 70.00 Bytes Out [total, mean] 12060000, 67.00 Success [ratio] 100.00% -Status Codes [code:count] 200:180000 +Status Codes [code:count] 200:180000 Error Set: ``` ## Analysis ### CPU Metrics -![cpu-03-2024.png](resources%2Fcpu-03-2024.png) + +![cpu-03-2024.png](resources/cpu-03-2024.png) CPU metrics exhibit a slight uptick from approximately 10% to around 125% at peak load of 1,000 RPS. However, it's noteworthy that the gateway server did not reach full CPU utilization, with the maximum observed at 800%. ### Ram Metrics -![memory-03-2024.png](resources%2Fmemory-03-2024.png) + +![memory-03-2024.png](resources/memory-03-2024.png) RAM metrics show a similar pattern, with a slight increase from around 120MiB to approximately 280MiB at peak load. This increase is expected due to the opening of more network connections while serving traffic. ### Latency Analysis + Upon closer inspection, despite the tenfold increase in load from 100 RPS to 1,000 RPS, the benchmark latency remained relatively consistent at ~150MS. Nodies, with the gateway server in production, has seen multiple node operators achieve lower latencies, typically ranging from 50ms to 70ms P90 latency at similar or higher request rates. Therefore, a consistent baseline latency of ~150ms at even 100 RPS in our benchmarking environment warranted further investigation to determine root cause. By analyzing Prometheus metrics emitted by the gateway server, specifically `pocket_relay_latency` (which measures the latency of creating a relay, including hashing, signing, and sending it to POKT Nodes) and `relay_latency` (providing an end-to-end latency metric including node selection), it was possible to identify the source of additional latency overhead. -![node-selection-overhead-03-2024.png](resources%2Fnode-selection-overhead-03-2024.png) +![node-selection-overhead-03-2024.png](resources/node-selection-overhead-03-2024.png) This deep dive revealed that the bottleneck does not lie in QoS/Node selection, as the intersection of the two metrics indicated that node selection completes within fractions of a second, ruling out that the gateway server code has a bottleneck. The baseline latency overhead therefore is attributed to protocol requirements (hashing/signing a relay) and hardware specifications. To mitigate this latency, upgrading to more powerful CPUs or dedicated machines should decrease this latency. Nodies currently uses the AMD 5950X CPU for their gateway servers. ### Summary + This benchmark provides a comprehensive quantitative assessment of the gateway server's performance under varying loads within the POKT Testnet environment. Analysis of CPU metrics reveals a slight uptick in CPU utilization from approximately 10% to around 125% at peak load of 1,000 RPS (max capacity of 800%). Memory metrics also show a similar pattern, with memory utilization increasing from around 120MiB to approximately 280MiB at peak load. Despite the increase in load, the gateway server demonstrates resilience, maintaining consistent latency across different request rates. diff --git a/docs/docker-compose.md b/docs/docker-compose.md new file mode 100644 index 0000000..671ca7f --- /dev/null +++ b/docs/docker-compose.md @@ -0,0 +1,49 @@ +# [WIP] Docker Compose Example + +This is an example of the instructions to run the gateway server on a remote debian +server using docker-compose from scratch. + +Locally + +```bash + +# SSH into your server +ssh user@your-remote-server + +# Install the necessary dependencies +sudo apt update +sudo apt install -y postgresql postgresql-contrib git docker.io +sudo systemctl start docker +sudo systemctl enable docker +sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + +# Make a workspace directory +mkdir workspace +cd workspace + +# Add the following to ~/.profile +export PATH=$PATH:/usr/local/go/bin +export GOPATH=$HOME/go +export PATH=$PATH:$GOPATH/bin + +# Install go +wget https://go.dev/dl/go1.21.12.linux-amd64.tar.gz +sudo tar -C /usr/local -xzf go1.21.12.linux-amd64.tar.gz + +# Clone the gateway server repository +git clone https://github.com/pokt-network/gateway-server.git +cd gateway-server/ + + +# Uncomment the postgres related lines in the docker-compose.yml file +cp docker-compose.yml.sample docker-compose.yml + +# Update the .env accordingly +cp .env.sample .env + +# Start the gateway server +docker-compose up -d + +# OPTIONAL: Connect to the postgres container directly +PGPASSWORD=mypassword psql -h localhost -U myuser -d postgres -p 5433 +``` diff --git a/docs/json-optimizations.md b/docs/json-optimizations.md deleted file mode 100644 index 16a40e8..0000000 --- a/docs/json-optimizations.md +++ /dev/null @@ -1,11 +0,0 @@ -## Optimization Dependencies -- [FastHTTP](https://github.com/valyala/fasthttp) for both HTTP Client/Server -- [FastJSON](https://github.com/pquerna/ffjson) for performant JSON Serialization and Deserialization -- Lightweight Pocket Client - -We have implemented our own lightweight Pocket client to enhance speed and efficiency. Leveraging the power of [FastHTTP](https://github.com/valyala/fasthttp) and [FastJSON](https://github.com/pquerna/ffjson), our custom client achieves remarkable performance gains. Additionally, it has the capability to properly parse node runner's POKT errors properly given that the network runs diverse POKT clients (geomesh, leanpokt, their own custom client). - -### Why It's More Efficient/Faster -1. **FastHTTP:** This library is designed for high-performance scenarios, providing a faster alternative to standard HTTP clients. Its concurrency-focused design allows our Pocket client to handle multiple requests concurrently, improving overall responsiveness. -2. **FastJSON:** The use of FastJSON ensures swift and efficient JSON serialization and deserialization. This directly contributes to reduced processing times, making our Pocket client an excellent choice for high-scale web traffic. - diff --git a/docs/node-selection.md b/docs/node-selection.md index 13afe27..36d972b 100644 --- a/docs/node-selection.md +++ b/docs/node-selection.md @@ -1,7 +1,19 @@ -# POKT Gateway Node Selection +# POKT Gateway Node Selection + +- [Node Selection System Architecture](#node-selection-system-architecture) +- [QoS Controls](#qos-controls) +- [Node Selector](#node-selector) + - [Checks Framework](#checks-framework) + - [Existing QoS checks](#existing-qos-checks) + - [Height Check (i.e. sync checks)](#height-check-ie-sync-checks) + - [Data Integrity Check (quorum checks)](#data-integrity-check-quorum-checks) + - [Adding custom QoS checks](#adding-custom-qos-checks) +- [Future Improvements](#future-improvements) ## Node Selection System Architecture -![gateway-server-node-selection-system.png](resources%2Fgateway-server-node-selection-system.png) + +![gateway-server-node-selection-system.png](resources/gateway-server-node-selection-system.png) + - `Session Registry` - responsible for "priming" sessions asynchronously, providing session metadata, and feeding the node to the `NodeSelectorService` - `Pocket Relayer` - responsible for sending a relay to the network - `NodeSelectorService` - responsible for running QoS checks and identifying healthy nodes by chain. @@ -12,68 +24,81 @@ The gateway kit server determines if a set of nodes are healthy based off a simple weight formula with the following heuristics: -- Latency -- Success Responses -- Correctness in regard to other node operators. -- Liveliness (Synchronization) +1. Latency (round-trip-tim) +2. Success Responses (uptime & availability) +3. Correctness in regard to other node operators (quorum checks) +4. Liveliness (syn checks) ## Node Selector + After the sessions are primed, the nodes are fed to the `NodeSelectorService` which is responsible for: + 1. Running various QoS checks (Height and Data Integrity Checks) 2. Exposing functions for the main process to select a healthy node `FindNode(chainId string) string` ### Checks Framework + The gateway server provides a simple interface called a `CheckJob`. This interface consists of three simple functions + ```go type CheckJob interface { Perform() Name() string ShouldRun() bool - } +} ``` + Under the hood, the NodeSelectorService is responsible for asynchronously executing all the initialized `CheckJobs`. -### Existing QoS checks: -- **Height Check:** The general flow would be: - 1) Query all node operators height, - 2) compares heights with other node operators within a specific threshold - 3) filters out node operators that exceed the configurable block height tolerance. +### Existing QoS checks + +#### Height Check (i.e. sync checks) + +**The general flow would be:** -- **Data Integrity Check:** The general flow would be: - 1) Retrieve a unique block identifier (i.e block hash or total block tx count, etc) with a configurable block offset for randomness, - 2) Query other node operators for the same block identifier - 3) filter out other node operators that return a different identifier. +1. Query all node operators height, +2. compares heights with other node operators within a specific threshold +3. filters out node operators that exceed the configurable block height tolerance. + +#### Data Integrity Check (quorum checks) + +The general flow would be: + +1. Retrieve a unique block identifier (i.e block hash or total block tx count, etc) with a configurable block offset for randomness, +2. Query other node operators for the same block identifier +3. Filter out other node operators that return a different identifier. Some existing implementations of Checks can be found in: -1. [evm_data_integrity_check.go](..%2Finternal%2Fnode_selector_service%2Fchecks%2Fevm_data_integrity_check%2Fevm_data_integrity_check.go) -2. [evm_height_check.go](..%2Finternal%2Fnode_selector_service%2Fchecks%2Fevm_height_check%2Fevm_height_check.go) -3. [pokt_height_check.go](..%2Finternal%2Fnode_selector_service%2Fchecks%2Fpokt_height_check%2Fpokt_height_check.go) -4. [pokt_data_integrity_check.go](..%2Finternal%2Fnode_selector_service%2Fchecks%2Fpokt_data_integrity_check%2Fpokt_data_integrity_check.go) -5. [solana_height_check.go](..%2Finternal%2Fnode_selector_service%2Fchecks%2Fsolana_height_check%2Fsolana_height_check.go) -6. [solana_data_integrity_check.go](..%2Finternal%2Fnode_selector_service%2Fchecks%2Fsolana_data_integrity_check%2Fsolana_data_integrity_check.go) + +1. [evm_data_integrity_check.go](../internal/node_selector_service/checks/evm_data_integrity_check/evm_data_integrity_check.go) +2. [evm_height_check.go](../internal/node_selector_service/checks/evm_height_check/evm_height_check.go) +3. [pokt_height_check.go](../internal/node_selector_service/checks/pokt_height_check/pokt_height_check.go) +4. [pokt_data_integrity_check.go](../internal/node_selector_service/checks/pokt_data_integrity_check/pokt_data_integrity_check.go) +5. [solana_height_check.go](../internal/node_selector_service/checks/solana_height_check/solana_height_check.go) +6. [solana_data_integrity_check.go](../internal/node_selector_service/checks/solana_data_integrity_check/solana_data_integrity_check.go) + ### Adding custom QoS checks Every custom check must conform to the `CheckJob` interface. The gateway server provides a base check: + ```go type Check struct { -NodeList []*qos_models.QosNode -PocketRelayer pokt_v0.PocketRelayer -ChainConfiguration chain_configurations_registry.ChainConfigurationsService + NodeList []*qos_models.QosNode + PocketRelayer pokt_v0.PocketRelayer + ChainConfiguration chain_configurations_registry.ChainConfigurationsService } ``` + that developers should inherit. This base check provides a list of nodes to check and a `PocketRelayer` that allows the developer to send requests to the nodes in the network, and `ChainConfiguration` service that allows for per-chain specific check configurations. -Checks are designed to be opinionated and there are numerous ways to implement whether a node is healthy or not by definition. Therefore, implementing custom QoS checks will be dependent on the chain or data source the developer is looking to support. For example, the developer may want to send a request to a custom blockchain node with a custom JSON-RPC method to see if the node is synced by using the provided `PocketRelayer` to send a request to the node through Pocket network. -If the node is not synced, the developer can set a custom punishment through the various functions exposed in [qos_node.go](..%2Finternal%2Fnode_selector_service%2Fmodels%2Fqos_node.go), such as `SetTimeoutUntil` to punish the node. +Checks are designed to be opinionated and there are numerous ways to implement whether a node is healthy or not by definition. Therefore, implementing custom QoS checks will be dependent on the chain or data source the developer is looking to support. For example, the developer may want to send a request to a custom blockchain node with a custom JSON-RPC method to see if the node is synced by using the provided `PocketRelayer` to send a request to the node through Pocket network. +If the node is not synced, the developer can set a custom punishment through the various functions exposed in [qos_node.go](../internal/node_selector_service/models/qos_node.go), such as `SetTimeoutUntil` to punish the node. -Once the developer is finished implementing the CheckJob, they can enable the QoS check by initializing the newly created check into the `enabledChecks` variable inside [node_selector_service.go](..%2Finternal%2Fnode_selector_service%2Fnode_selector_service.go) and are encouraged to open up a PR for inclusion in the official repository. +Once the developer is finished implementing the CheckJob, they can enable the QoS check by initializing the newly created check into the `enabledChecks` variable inside [node_selector_service.go](../internal/node_selector_service/node_selector_service.go) and are encouraged to open up a PR for inclusion in the official repository. ## Future Improvements - Long term persistent results - - Pros: More data to work with on determining if a node is healthy - - Cons: Expensive, more complex logic (due to geographic regions) and can be punishing to new node operators + - Pros: More data to work with on determining if a node is healthy + - Cons: Expensive, more complex logic (due to geographic regions) and can be punishing to new node operators - Rolling up the results for long term storage & historical look back - - - diff --git a/docs/overview.md b/docs/overview.md index 963e061..72424a1 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -1,8 +1,9 @@ # POKT Gateway Server Overview -The Gateway server's goal is to reduce the complexity associated with directly interfacing with the protocol with an library that any developer can contribute to. The Gateway server kickstarts off as a light-weight process that enables developers of all kinds to be able to interact with the protocol and engage with 50+ blockchains without the need to store terabytes of data, require heavy computational power, or understand the POKT protocol specifications with a simple docker-compose file. The rhetorical question that we pose to future actors who want to maintain a blockchain node is: Why spin up an Ethereum node and maintain it yourself whenever you can just leverage POKT natively using the Gateway server? After all, using POKT would require a fraction of the required resources and technical staffing. +The Gateway server's goal is to reduce the complexity associated with directly interfacing with the protocol with an library that any developer can contribute to. The Gateway server kickstarts off as a light-weight process that enables developers of all kinds to be able to interact with the protocol and engage with 50+ blockchains without the need to store terabytes of data, require heavy computational power, or understand the POKT protocol specifications with a simple docker-compose file. The rhetorical question that we pose to future actors who want to maintain a blockchain node is: Why spin up an Ethereum node and maintain it yourself whenever you can just leverage POKT natively using the Gateway server? After all, using POKT would require a fraction of the required resources and technical staffing. ## Features + - Simple docker-compose file with minimal dependencies to spin up - a single tenancy HTTP endpoint for each blockchain that POKT supports, abstracting POKT's Relay Specification. This endpoint's throughput should scale based on the number of app stakes the developer provides. - QoS checks to allow for optimized latency and success rates @@ -14,13 +15,14 @@ The Gateway server's goal is to reduce the complexity associated with directly i - Functionality improvement such as allowing for proper decoding of POKT Node error messages such as max evidence sealed errors. ## What's not included in the Gateway Server + - Authentication - Rate Limiting & Multi-tenancy endpoints - SaaS-based UX - Reverse Proxy / Load Balancing Mechanisms - Any other Opinionated SaaS-like design decision. -The decision to exclude certain features such as Authentication, Rate Limiting and multi-tenancy endpoints, SaaS-based UX, and Reverse Proxy/Load Balancing Mechanisms is rooted in the project's philosophy. These aspects are often regarded as opinionated web2 functionalities, and there are already numerous resources available on how to build SaaS products with various authentication mechanisms, rate-limiting strategies, and user experience design patterns. +The decision to exclude certain features such as Authentication, Rate Limiting and multi-tenancy endpoints, SaaS-based UX, and Reverse Proxy/Load Balancing Mechanisms is rooted in the project's philosophy. These aspects are often regarded as opinionated web2 functionalities, and there are already numerous resources available on how to build SaaS products with various authentication mechanisms, rate-limiting strategies, and user experience design patterns. The Gateway server aims to simplify the POKT protocol, not reinventing the wheel. Each Gateway, being a distinct entity with its unique requirements and team dynamics, is better suited to decide on these aspects independently. For instance, the choice of authentication mechanisms can vary widely between teams, ranging from widely-used services like Auth0 and Amazon Cognito to in-house authentication solutions tailored to the specific language and skill set of the development team. @@ -29,10 +31,12 @@ By not including these opinionated web2 functionalities, the Gateway server ackn As the project evolves, we anticipate that individual Gateways will incorporate their implementations of these features based on their unique requirements and preferences. This decentralized approach empowers developers to make decisions that align with their specific use cases, promoting a more customized and efficient integration with the Gateway server. ## Future + We envision that the server will be used as a foundation for the entirety of the ecosystem to continue to build on top of such as: + - Building their frontends and extending their backend to include POKT by using the gateway server for their own SaaS business - Create Demo SaaS gateways that use the gateway server as the underlying foundation. - Using POKT as a hyper scaler whenever they need more computational power or access to more blockchains (sticking the process into their LB rotation) - Using POKT as a backend as a failover whenever their centralized nodes go down (sticking the process into their LB rotation) -Over time, as more gateways enter the network, there will be re-occurring patterns on what is needed on the foundational level and developers can create RFPs to have them included. For example, while rate limiting and multi-tenancy endpoints feel too opinionated right now, there is a future where we can create a service that distributes these endpoints natively in the GW server. The use cases are limitless and we expect that over time, community contributions into the gateway server will enable some of the aforementioned use cases natively. \ No newline at end of file +Over time, as more gateways enter the network, there will be re-occurring patterns on what is needed on the foundational level and developers can create RFPs to have them included. For example, while rate limiting and multi-tenancy endpoints feel too opinionated right now, there is a future where we can create a service that distributes these endpoints natively in the GW server. The use cases are limitless and we expect that over time, community contributions into the gateway server will enable some of the aforementioned use cases natively. diff --git a/docs/performance-optimizations.md b/docs/performance-optimizations.md new file mode 100644 index 0000000..a8221ef --- /dev/null +++ b/docs/performance-optimizations.md @@ -0,0 +1,18 @@ +# Performance Optimizations + +1. [FastHTTP](https://github.com/valyala/fasthttp) for both HTTP Client/Server +2. [FastJSON](https://github.com/pquerna/ffjson) for performant JSON Serialization and Deserialization +3. Lightweight Pocket Client + +## Pocket Client Optimizations + +We have implemented our own lightweight Pocket client to enhance speed and efficiency. + +Leveraging the power of [FastHTTP](https://github.com/valyala/fasthttp) and [FastJSON](https://github.com/pquerna/ffjson), our custom client achieves remarkable performance gains. + +Additionally, it has the capability to properly parse node runner's POKT errors properly given that the network runs diverse POKT clients (geomesh, leanpokt, their own custom client). + +### Why It's More Efficient/Faster + +1. **FastHTTP:** This library is designed for high-performance scenarios, providing a faster alternative to standard HTTP clients. Its concurrency-focused design allows our Pocket client to handle multiple requests concurrently, improving overall responsiveness. +2. **FastJSON:** The use of FastJSON ensures swift and efficient JSON serialization and deserialization. This directly contributes to reduced processing times, making our Pocket client an excellent choice for high-scale web traffic. diff --git a/docs/pokt-primer.md b/docs/pokt-primer.md index 33c6b63..0a92860 100644 --- a/docs/pokt-primer.md +++ b/docs/pokt-primer.md @@ -1,31 +1,43 @@ -# POKT Primer +# POKT Primer -## POKT Network: A Quick Overview -1. Apps (App developers): Individuals or entities that stake into the Pocket Network and obtain access to external blockchain nodes in return. -2. Node Runners: Individuals or entities that stake into the network and provide access to external blockchain nodes, such as Ethereum & Polygon in return for $POKT. +- [POKT Network: A Quick Overview](#pokt-network-a-quick-overview) + - [The Challenge: Interacting with the Protocol](#the-challenge-interacting-with-the-protocol) + - [The Solution: Gateway Operators](#the-solution-gateway-operators) + - [Conclusion](#conclusion) +- [Footnotes](#footnotes) + +## POKT Network: A Quick Overview + +1. **Apps (App developers)**: Individuals or entities that stake into the Pocket Network and obtain access to external blockchain nodes in return. +2. **Node Runners**: Individuals or entities that stake into the network and provide access to external blockchain nodes, such as Ethereum & Polygon in return for $POKT. ### The Challenge: Interacting with the Protocol + For application developers, directly engaging with the POKT Network can be intimidating due to its inherent complexities. The technical barriers and protocol nuances can dissuade many from integrating and adopting the network. **Challenges faced by App Developers using the protocol:** -1. Managing Throughput: The network supports approximately 250 app stakes and around 10B relays. With each app stake being limited to roughly 20M requests per hour, developers who surpass this need to stake multiple applications and balance the load among these stakes. -2. Determining Quality of Service (QoS): The network doesn't currently enforce QoS standards. Apps are assigned a set of pseudo-randomly selected node runners, rotated over specified intervals, also known as sessions. It falls on the application developer to implement strategies, such as filtering and predictive analysis, to select node runners that align with their criteria for reliability, availability, and data integrity. -3. Protocol Interaction: Unlike the straightforward procedure of sending requests to an HTTP JSON-RPC server, interacting with the POKT Network requires far more complexities given its blockchain nature. (i.e. signing a request for a relay proof) + +1. **Managing Throughput**: The network supports approximately `250 app stakes` and around `10B relays`. With each app stake being limited to roughly `20M requests per hour`, developers who surpass this need to stake multiple applications and balance the load among these stakes. +2. **Determining Quality of Service (QoS)**: The network doesn't currently enforce QoS standards. Apps are assigned a set of pseudo-randomly selected node runners, rotated over specified intervals, also known as sessions. It falls on the application developer to implement strategies, such as filtering and predictive analysis, to select node runners that align with their criteria for reliability, availability, and data integrity. +3. **Protocol Interaction**: Unlike the straightforward procedure of sending requests to an HTTP JSON-RPC server, interacting with the POKT Network requires far more complexities given its blockchain nature. (i.e. signing a request for a relay proof) ### The Solution: Gateway Operators + Gateway Operators act as a conduit between app developers and the POKT Network, streamlining the process by abstracting the network's complexities. Their operations on a high level can be seen as: -1. Managing Throughput: By staking and load-balancing app stakes, Gateway Operators ensure the required throughput for smooth network interactions. -2. Determining the Quality of Service: Gateway Operators filter malicious, out-of-sync, offline, or underperforming nodes. -3. Protocol Interaction: Gateway Operators offer a seamless HTTP JSON-RPC Server interface, making it simpler for developers to send and receive requests, akin to interactions with conventional servers. Under the hood, the web server will contain the necessary business logic in order to interact with the protocol. + +1. **Managing Throughput**: By staking and load-balancing app stakes, Gateway Operators ensure the required throughput for smooth network interactions. +2. **Determining the Quality of Service**: Gateway Operators filter malicious, out-of-sync, offline, or under-performing nodes. +3. **Protocol Interaction**: Gateway Operators offer a seamless HTTP JSON-RPC Server interface, making it simpler for developers to send and receive requests, akin to interactions with conventional servers. Under the hood, the web server will contain the necessary business logic in order to interact with the protocol. ### Conclusion + Engaging with the POKT Network's capabilities doesn't have to be an uphill task. Thanks to Gateway Operators, app developers can concentrate on their core competencies—developing remarkable applications using a familiar HTTP interface, like traditional RPC providers—all while reaping the benefits of a decentralized RPC platform. --- -###### Footnotes: +## Footnotes 1. _As of 9/14/2023, the app stakes are permissioned and overseen by the Pocket Network Foundation for security considerations._ 2. _The amount of POKT staked into an app doesn't carry significant implications as all gateway operators are charged a fee for every request sent through an app stake._ 3. _Historically, Grove (formerly known as Pocket Network Inc.) has been the sole gateway operator. This will change by 2024 Q2 as more gateway operators join the network._ -4. _Our research aims to invite more gateway operators to join the network in a sustainable fashion by documenting the protocol specifications and limitations and leveraging and providing open-source software and noncloud vendor-lock-in services._ \ No newline at end of file +4. _Our research aims to invite more gateway operators to join the network in a sustainable fashion by documenting the protocol specifications and limitations and leveraging and providing open-source software and noncloud vendor-lock-in services._ diff --git a/docs/pokt-relay-specification.md b/docs/pokt-relay-specification.md index 3be179b..e02f6cb 100644 --- a/docs/pokt-relay-specification.md +++ b/docs/pokt-relay-specification.md @@ -1,23 +1,41 @@ -# POKT Gateway Server Endpoints +# POKT Gateway Server Endpoints + +- [Overview](#overview) +- [What are AATs?](#what-are-aats) +- [Generating an AAT](#generating-an-aat) +- [Retrieve the latest session for your application](#retrieve-the-latest-session-for-your-application) +- [What are relay proofs?](#what-are-relay-proofs) +- [Generating a relay to the network](#generating-a-relay-to-the-network) + - [**Relay Meta Data structure:**](#relay-meta-data-structure) + - [**Relay Metadata structure:**](#relay-metadata-structure) + - [**Relay Proof Data structure**](#relay-proof-data-structure) +- [Final Steps](#final-steps) ## Overview + To truly understand why gateway operators and the Nodies Gateway Stack are important, let's review what steps application developers have to perform in order to send a single request to Pocket Network without gateways. The chronological steps assuming the application is staked are: -* Generate an Application Authentication Token (AAT) -* Send a request to a Pocket full node for the latest nodes in a session -* Construct and sign a Relay Proof and submit it to one of the nodes in a session -* Receive a response from a node -* Determine if the response is legit or valid -* Proxy it back to your Web Application + +- Generate an Application Authentication Token (AAT) +- Send a request to a Pocket full node for the latest nodes in a session +- Construct and sign a Relay Proof and submit it to one of the nodes in a session +- Receive a response from a node +- Determine if the response is legit or valid +- Proxy it back to your Web Application + + + --- ## What are AATs? + The AAT is an auth token that allows application clients to access the network without the need to expose their private keys. _Note: AATs are non-revocable and do not have a time expiration date. The only way to revoke a token is to unstake the entire application_ **AAT's Data Structure** + ```go type AAT struct { Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"version"` @@ -26,17 +44,22 @@ type AAT struct { ApplicationSignature string `protobuf:"bytes,4,opt,name=applicationSignature,proto3" json:"signature"` } ``` + **JSON AAT Example** + ```json { - "version": "0.0.1", - "app_pub_key": "eb0cf2a891382677f03c1b080ec270c693dda7a4c3ee4bcac259ad47c5fe0743", - "client_pub_key": "eb0cf2a891382677f03c1b080ec270c693dda7a4c3ee4bcac259ad47c5fe0743", - "signature": "5309f66a22ace63e8b4f94220151feabad11d4f3c22f50f6e395c72f1df96111da9bb25eceb11361d7e7074e7105d57dd2ec1d85cf962460608ef4bc5d35a80a", + "version": "0.0.1", + "app_pub_key": "eb0cf2a891382677f03c1b080ec270c693dda7a4c3ee4bcac259ad47c5fe0743", + "client_pub_key": "eb0cf2a891382677f03c1b080ec270c693dda7a4c3ee4bcac259ad47c5fe0743", + "signature": "5309f66a22ace63e8b4f94220151feabad11d4f3c22f50f6e395c72f1df96111da9bb25eceb11361d7e7074e7105d57dd2ec1d85cf962460608ef4bc5d35a80a" } ``` + --- + ## Generating an AAT + 1. The AAT specification can be found here, but in simple terms: 2. JSON Encode AAT with an empty string signature field: 3. SHA3_256 the JSON bytes (bytes of the stringified JSON Object) @@ -45,10 +68,14 @@ type AAT struct { 6. Replace the empty AAT.signature field with the hex-encoded result. An example code implementation of this can be found in both Javascript and Golang + 1. [Javascript Implementation](https://github.com/pokt-network/pocket-aat-js/tree/staging) 2. [Golang Implementation](https://github.com/pokt-network/pocket-core/blob/staging/x/pocketcore/keeper/aat.go#L14) + --- + ## Retrieve the latest session for your application + An application is supported by 24 randomly chosen node runners for a short duration called a `session`, which currently lasts 4 blocks. Once that session is over, the application gets a new set of node runners. To find out who the latest node runners are, the application connects to a Pocket node and sends it a request. This process is called dispatching. @@ -57,12 +84,13 @@ Request (POST): `{pocket_host}/v1/client/dispatch` with the following payload: ```json { - "app_public_key":"514810e9139c5571905c642564b18cfb67899af2da05e638031075033da091a5", - "chain":"0074" + "app_public_key": "514810e9139c5571905c642564b18cfb67899af2da05e638031075033da091a5", + "chain": "0074" } ``` Response: + ```json { "block_height": 108183, @@ -106,58 +134,70 @@ Response: _Note: Given that retrieving a session requires a full node, this means staked applications will need to source full nodes or run one themselves! As well, this acts as a failure point for app/gateway operators if they only rely on one full node for dispatching. Without a session, application developers cannot send a request. Thankfully thanks to pruning efforts and more full nodes entering the networks, this should become a lower risk._ --- + ## What are relay proofs? + At its core, a relay proof is like a digital receipt proving that an application sent a request to a node runner. Here's how it works: -* **Generation and Validation:** When an application makes a request, it creates and signs a 'relay proof'. This is like a digital signature, ensuring the request is genuine and hasn't been tampered with. -* **Verification by Servicer:** These servicers check the relay proof to make sure it's from a legitimate application in the network. -* **Storing the Proof:** Once verified, node runners store this proof in a data structure called a 'Merle Sum Index (MSI) Tree'. -* **Processing the Request:** The node runner then processes the request and sends the information back to the application. -* **Claim and Proof Lifecycle:** In the process of getting paid for their work, node runners go through a two-step 'claim and proof' cycle. First, they submit the 'root' of the MSI Tree as part of a claim transaction, indicating they have served several requests. Then, to provide evidence of their work, they must submit a randomly selected index along with Merkle proof from the branch to root, ensuring fairness and verification. This allows the network to trust that the node runners aren't just selecting an easy-to-prove transaction but deterministically chosen by the network in a secure way. Ultimately, this allows for a compute and space-efficient blockchain as validators of the network do not have to store nor verify every single request served. + +- **Generation and Validation:** When an application makes a request, it creates and signs a 'relay proof'. This is like a digital signature, ensuring the request is genuine and hasn't been tampered with. +- **Verification by Servicer:** These servicers check the relay proof to make sure it's from a legitimate application in the network. +- **Storing the Proof:** Once verified, node runners store this proof in a data structure called a 'Merle Sum Index (MSI) Tree'. +- **Processing the Request:** The node runner then processes the request and sends the information back to the application. +- **Claim and Proof Lifecycle:** In the process of getting paid for their work, node runners go through a two-step 'claim and proof' cycle. First, they submit the 'root' of the MSI Tree as part of a claim transaction, indicating they have served several requests. Then, to provide evidence of their work, they must submit a randomly selected index along with Merkle proof from the branch to root, ensuring fairness and verification. This allows the network to trust that the node runners aren't just selecting an easy-to-prove transaction but deterministically chosen by the network in a secure way. Ultimately, this allows for a compute and space-efficient blockchain as validators of the network do not have to store nor verify every single request served. ## Generating a relay to the network + Now that we have access to a set of node runners (and assuming all node runners are actually operational), we still need to send the JSON-RPC request in a data structure that node runners will understand and accept. Unfortunately, node runners will not accept a simple HTTP JSON-RPC request as you would expect with other node providers, so we must construct a relay along with a relay proof. POST request to `{pocket_host}/v1/client/relay` with the following payload + ```json { - "payload":"relay_payload", - "meta":"relay_meta", - "proof":"relay_proof" + "payload": "relay_payload", + "meta": "relay_meta", + "proof": "relay_proof" } ``` ### **Relay Meta Data structure:** + ```go type Payload struct { - Data string `json:"data"` // the actual data string for the external chain - Method string `json:"method"` // the http CRUD method - Path string `json:"path"` // the REST Path - Headers map[string]string `json:"headers,omitempty"` // http headers + Data string `json:"data"` // the actual data string for the external chain + Method string `json:"method"` // the http CRUD method + Path string `json:"path"` // the REST Path + Headers map[string]string `json:"headers,omitempty"` // http headers } ``` + ```json { - "data": "{\"jsonrpc\":\"2.0\",\"method\":\"eth_getBalance\",\"params\":[\"0x050ea4ab4183E41129B7D72A492DaBf52B27EdB5\",\"latest\"],\"id\":67}", - "method": "POST", - "path": "", - "headers": null + "data": "{\"jsonrpc\":\"2.0\",\"method\":\"eth_getBalance\",\"params\":[\"0x050ea4ab4183E41129B7D72A492DaBf52B27EdB5\",\"latest\"],\"id\":67}", + "method": "POST", + "path": "", + "headers": null } ``` ### **Relay Metadata structure:** + Relay metadata is simply when the request is made based on POKT's block time. This can be simplified to the session block height. However, if possible, the application developer should return the block height from /v1/query/height + ```go type RelayMeta struct { BlockHeight int64 `json:"block_height"` // the block height when the request is made } ``` + ```json { - "block_height": 100 + "block_height": 100 } ``` ### **Relay Proof Data structure** + The generation of the relay proof is the most complex and crucial piece to ensuring the application's request is recognized and accepted by the node runners. + ```go type RelayProof struct { RequestHash string `protobuf:"bytes,1,opt,name=requestHash,proto3" json:"request_hash"` @@ -169,69 +209,81 @@ type RelayProof struct { Signature string `protobuf:"bytes,7,opt,name=signature,proto3" json:"signature"` } ``` + 1. Entropy Generate a random integer from [0, INT64] 2. `SessionBlockHeight`, `Blockchain`, `ServicerPubKey`, `Token` are retrievable from the above steps. -3. `RequestHash` is generated by with the following psuedo code -```text -// requestHash -{ - "payload": relay_payload, - "meta": relay_meta -} -requestHashBytes := json.Marshal(requestHash) (bytes of the stringified JSON Object) -SHA3_256(requestHashBytes) -HexEncode to string -``` +3. `RequestHash` is generated by with the following pseudo-code + + ```text + // requestHash + { + "payload": relay_payload, + "meta": relay_meta + } + requestHashBytes := json.Marshal(requestHash) (bytes of the stringified JSON Object) + SHA3_256(requestHashBytes) + HexEncode to string + ``` + 4. Once the relay object is generated, construct an ordered version of the relay object to hash and sign with the application private key for a signature using the following order: ng the following order: -```text -// relayProof -{ - "entropy": "1234567890123456", - "session_block_height": 108181, - "servicer_pub_key": "a1b2c3d4e5f67890a1b2c3d4e5f67890a1b2c3d4e5f67890", - "blockchain": "0074", - "signature": "", - "token": "SHA3_256-aat-without-signature", - "request_hash": "sha-256-request-hash" -} -// json encode relay proof -relayProofJsonBytes := json.Marshal(relayProof) -hashedRelayProof = SHA3_256(rrelayProofJsonBytes) -SIGN(hashedRelayProof, appPrivateKey) -> c1d2e3f4a5b67890c1d2e3f4a5b67890c1d2e3f4a5b67890c1d2e3f4a5b67890 -``` + + ```text + // relayProof + { + "entropy": "1234567890123456", + "session_block_height": 108181, + "servicer_pub_key": "a1b2c3d4e5f67890a1b2c3d4e5f67890a1b2c3d4e5f67890", + "blockchain": "0074", + "signature": "", + "token": "SHA3_256-aat-without-signature", + "request_hash": "sha-256-request-hash" + } + // json encode relay proof + relayProofJsonBytes := json.Marshal(relayProof) + hashedRelayProof = SHA3_256(relayProofJsonBytes) + SIGN(hashedRelayProof, appPrivateKey) -> c1d2e3f4a5b67890c1d2e3f4a5b67890c1d2e3f4a5b67890c1d2e3f4a5b67890 + ``` + 5. Use the generated signature to fill out the missing Signature field **_NOTE: Ordering of the JSON object matters because the values are hashed. If the ordering changes, so will the hash._** --- + ## Final Steps + With all the fields now generated, the valid relay proof can be constructed as below: 1. Relay Proof with Signature -```json -{ - "request_hash": "b1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", - "entropy": 1234567890123456, - "session_block_height": 108181, - "servicer_pub_key": "a1b2c3d4e5f67890a1b2c3d4e5f67890a1b2c3d4e5f67890", - "blockchain": "0074", - "aat": { + + ```json + { + "request_hash": "b1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + "entropy": 1234567890123456, + "session_block_height": 108181, + "servicer_pub_key": "a1b2c3d4e5f67890a1b2c3d4e5f67890a1b2c3d4e5f67890", + "blockchain": "0074", + "aat": { ... // includes signature - }, - "signature": "c1d2e3f4a5b67890c1d2e3f4a5b67890c1d2e3f4a5b67890c1d2e3f4a5b67890" -} -``` -2. Send to /v1/client/relay -```json -{ - "payload":"relay_payload", - "meta":"relay_meta", - "proof":"relay_proof_with_signature" -} -``` + }, + "signature": "c1d2e3f4a5b67890c1d2e3f4a5b67890c1d2e3f4a5b67890c1d2e3f4a5b67890" + } + ``` + +2. Send to /v1/client/relay + + ```json + { + "payload": "relay_payload", + "meta": "relay_meta", + "proof": "relay_proof_with_signature" + } + ``` + If all goes well, the application should receive a response from the node runner! ----- -After delving into the complexities of selecting a reliable source of dispatchers to retrieve a session, considering the network does not offer QoS assurances, and grasping the intricacies of sending requests to node runners, it becomes evident how crucial it is for software to abstract away the protocol and foster true developer adoption. This highlights the importance of the Gateway Operators and ultimately the Gateway server vision. \ No newline at end of file +--- + +After delving into the complexities of selecting a reliable source of dispatchers to retrieve a session, considering the network does not offer QoS assurances, and grasping the intricacies of sending requests to node runners, it becomes evident how crucial it is for software to abstract away the protocol and foster true developer adoption. This highlights the importance of the Gateway Operators and ultimately the Gateway server vision. diff --git a/docs/quick-onboarding-guide.md b/docs/quick-onboarding-guide.md index 0435568..6a4546a 100644 --- a/docs/quick-onboarding-guide.md +++ b/docs/quick-onboarding-guide.md @@ -1,21 +1,44 @@ -# Quick Onboarding Guide +# Quick Onboarding Guide + +- [Quick Onboarding Guide ](#quick-onboarding-guide-) + - [1. Preparing the Application Stakes](#1-preparing-the-application-stakes) + - [2. Create Encryption Password](#2-create-encryption-password) + - [3. Configure Global Configuration Variables](#3-configure-global-configuration-variables) + - [Env Variables Description](#env-variables-description) + - [4. Run Migration Script](#4-run-migration-script) + - [5. Insert App Stake Private Keys](#5-insert-app-stake-private-keys) + - [6. Run Gateway Server](#6-run-gateway-server) + - [6.1 Locally From Source](#61-locally-from-source) + - [6.2 From A Docker Image](#62-from-a-docker-image) + - [6.3 From docker-compose (on remote server)](#63-from-docker-compose-on-remote-server) + - [7. Send a Relay](#7-send-a-relay) + - [POKT TestNet](#pokt-testnet) + - [POKT MainNet](#pokt-mainnet) + - [8.1 Chain \& Altruist Setup \& Configuration](#81-chain--altruist-setup--configuration) ## 1. Preparing the Application Stakes + Application stakes provides Gateway Operators access to the POKT Network for sending traffic. +
+ + + Testnet Instructions
  1. Generate 5 accounts (wallets) through the testnet wallet URL
  2. Distribute POKT to all the wallets generated through the testnet faucet
    3. -
  3. Stake each account into the network as an application stake with the chain id `0007` (a test chain that represents ETH Network). +
  4. Stake each account into the network as an application stake with the chain id `0007` (a test chain that represents ETH Network).
  5. You can use the application stake script to simplify the process if you don't have access to the Pocket Core Executable or not familiar with the CLI commands.
-Staking application stakes too complicated for you? No worries, we prestaked some shared applications stakes into POKT Testnet to help you get onboarded quicker. + +**Staking application stakes too complicated for you?** No worries, we prestaked some shared applications stakes into POKT Testnet to help you get onboarded quicker. **Please do not submit stake transactions to avoid disruption for other gateway operator testers as the applications are already staked on your behalf in the correct chain**. All applications are staked into chain 0007 with 10M POKT. Testnet Application Private Keys: +
+ Mainnet Instructions + + + Application stakes in Morse are permissioned, therefore you must receive application stakes through the Pocket Network Foundation. If you are an authorized gateway operator, the Foundation will assist you in receiving the application stakes private keys. -
+ ## 2. Create Encryption Password -Create an encryption password for your app stake keys. This password will be used to encrypt/decrypt your app-stake private keys stored in a Postgres database. It can be any combination of plaintext, symbols, letters, etc. + +Prepare an encryption password for your app stake keys. + +This password will be used to encrypt/decrypt your app-stake private keys stored in a Postgres database. It can be any combination of plaintext, symbols, letters, etc. ## 3. Configure Global Configuration Variables -Fill out the `.env` variables for the gateway server. This can be done by injecting environment variables directly or using a `.env` file. + +Run `cp .env.sample .env` and update the `.env` appropriately. ### Env Variables Description -| Variable Name | Description | Example Value | -|------------------------------------|------------------------------------------------------------|---------------------------------------------------------------------------------------------------| -| `POKT_RPC_FULL_HOST` | Used for dispatching sessions | `https://pokt-testnet-rpc.nodies.org` (a complimentary testnet dispatcher URL provided by Nodies) | -| `HTTP_SERVER_PORT` | Gateway server port | `8080` | -| `POKT_RPC_TIMEOUT` | Max response time for a POKT node to respond | `10s` | -| `ALTRUIST_REQUEST_TIMEOUT` | Max response time for an altruist backup to respond | `10s` | -| `ENVIRONMENT_STAGE` | Log verbosity | `development`, `production` | -| `SESSION_CACHE_TTL` | Duration for sessions to stay in cache | `75m` | -| `POKT_APPLICATIONS_ENCRYPTION_KEY` | User-generated encryption key | `a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6` | -| `DB_CONNECTION_URL` | PostgreSQL Database connection URL | `postgres://user:password@localhost:5432/postgres` | -| `EMIT_SERVICE_URL_PROM_METRICS` | Boolean flag to enable service url for relay metrics | `false`, `true` | -| `CHAIN_NETWORK` | Identifies which network the gateway server is running on. | `morse_mainnet`, `morse_testnet` | -See [.env.sample](..%2F.env.sample) for a sample. + +See [.env.sample](../.env.sample) for a sample. + +| Variable Name | Description | Example Value | +| ---------------------------------- | --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------- | +| `POKT_RPC_FULL_HOST` | Used for dispatching sessions | `https://pokt-testnet-rpc.nodies.org` (a complimentary testnet dispatcher URL provided by Nodies) | +| `HTTP_SERVER_PORT` | Gateway server port | `8080` | +| `POKT_RPC_TIMEOUT` | Max response time for a POKT node to respond | `10s` | +| `ALTRUIST_REQUEST_TIMEOUT` | Max response time for an altruist backup to respond | `10s` | +| `ENVIRONMENT_STAGE` | Log verbosity | `development`, `production` | +| `SESSION_CACHE_TTL` | Duration for sessions to stay in cache | `75m` | +| `POKT_APPLICATIONS_ENCRYPTION_KEY` | User-generated encryption key | `a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6` | +| `POKT_APPLICATION_PRIVATE_KEY` | Optional - Pocket application private key | `1d06f04dcf5199a7f93f625d4fa507c2e0aca2f94fa3ebc2022c5e589406a9133d7ec4fef2ef676b340ce1df6ec5d0264ce1f40fae7fe9e07c415fa06fc1ffd6` | + +| `DB_CONNECTION_URL` | PostgreSQL Database connection URL | `postgres://user:password@localhost:5432/postgres` | +| `EMIT_SERVICE_URL_PROM_METRICS` | Boolean flag to enable service url for relay metrics | `false`, `true` | +| `CHAIN_NETWORK` | Identifies which network the gateway server is running on. | `morse_mainnet`, `morse_testnet` | +| `API_KEY` | Any user generated key used to authenticate the user when calling the `poktapps` and `qosnodes` endpoints | `efe8eVTcWtXhp9ZfeTZcQuy49oDND4gh`, | ## 4. Run Migration Script + Run the migration script to seed your PostgreSQL database. + ```sh ./scripts/migration.sh -u ``` + + ## 5. Insert App Stake Private Keys + Copy and paste the following SQL query to insert app stake private keys into the database: + ```sql INSERT INTO pokt_applications (encrypted_private_key) VALUES (pgp_sym_encrypt('{private_key}', '{encryption_key}')); ``` + _Note: Replace {private_key} and {encryption_key} in the SQL query with your actual private key and encryption key._ -## 6. Compile and Run Gateway Server -Copy and paste the following code to compile and run the gateway server. Hit the endpoint http://localhost/relay/{chain_id} with a JSON-RPC payload. +:::tip + +If you've fully configured your `.env`, you can run `make db_insert_app_private_key` to insert the app private keys into the database. + +::: -If using testnet, send a request to chain `0007`, http://localhost/relay/0007. This testnet chain id represents Ethereum Node Operators and Nodies is currently supporting the chain id for reliable testing. +## 6. Run Gateway Server + +### 6.1 Locally From Source + +Compile the gateway server by running the following command: ```sh go build cmd/gateway_server/main.go ``` -Alternatively, you can leverage our published [docker images](https://github.com/pokt-network/gateway-server/pkgs/container/pocket-gateway-server) +And run it with: + +```sh +./main +``` + +### 6.2 From A Docker Image + +You can leverage our published [docker images](https://github.com/pokt-network/gateway-server/pkgs/container/pocket-gateway-server) + +### 6.3 From docker-compose (on remote server) + +See the rough notes in [docs](./docker-compose.md) for running the gateway server on a remote server. + +## 7. Send a Relay + +_Note: The requests below assume the applications are stake correctly._ + +### POKT TestNet + +If using testnet, send a request to chain `0007` at `http://localhost/relay/0007`. + +This testnet chain id represents Ethereum Node Operators and Nodies is currently supporting the chain id for reliable testing. + +For example, assuming the gateway server is running locally, you can send a request like so: + +```sh +curl -X POST -H "Content-Type: application/json" \ + --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \ + http://localhost:8080/relay/0007 +``` + +### POKT MainNet + +Similarly, you can sent a request to ethereum mainnet at `0021` like so: + +```sh +curl -X POST -H "Content-Type: application/json" \ + --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \ + http://localhost:8080/relay/0021 +``` + +### 8.1 Chain & Altruist Setup & Configuration + +Upon startup of the gateway server, nodes within a session take time to be marked as healthy (~10 seconds), therefore the requests will be routed to the altruist (failover) node first. -## 7. Setup Chain Configuration and Altruist -Upon startup of the gateway server, nodes within a session take time to be marked as healthy (~10 seconds), therefore the requests will be routed to the altruist (failover) node. It is suggested each chain that you support has a configured altruist. You can learn more about supplying altruist configurations through [altruist-chain-configuration.md](altruist-chain-configuration.md). +It is suggested each chain that you support has a configured altruist. You can learn more about supplying altruist configurations through [altruist-chain-configuration.md](altruist-chain-configuration.md). _Remember to keep sensitive information secure and follow best practices for handling private keys and passwords._ diff --git a/docs/system-architecture.md b/docs/system-architecture.md index 108cc66..d8b3482 100644 --- a/docs/system-architecture.md +++ b/docs/system-architecture.md @@ -1,14 +1,30 @@ -# POKT Gateway Server Architecture +# POKT Gateway Server Architecture ![gateway-server-architecture.png](resources/gateway-server-architecture.png) -## Gateway Server Managed Responsibilities -Under the hood, the gateway server handles everything in regard to protocol interaction abstract away the complexity of retrieving a session, signing a relay and sending it to a respective node and other functionality such as: -- Node Selection & Routing (QoS) - determining which nodes are healthy based off responses -- Provides underlying Prometheus metrics endpoint for relay performance metadata -- Providing an efficient HTTP endpoint to send requests to +- [Gateway Server Responsibilities](#gateway-server-responsibilities) + - [Primary Features](#primary-features) + - [Secondary Features](#secondary-features) +- [Gateway Operator Responsibilities](#gateway-operator-responsibilities) -## Gateway Operator Managed Responsibilities -- Keeping the encryption key and respectively the app stakes keys secure. -- Staking in the approriate chains -- Any features in regard to a SaaS business as mentioned in the [overview](overview.md). \ No newline at end of file +## Gateway Server Responsibilities + +### Primary Features + +Under the hood, the gateway server handles everything in regard to protocol interaction to abstract away the complexity of: + +1. Retrieving a session +2. Signing a relay +3. Sending a relay to a node operator & receiving a response + +### Secondary Features + +1. **Node Selection & Routing (QoS)** - determining which nodes are healthy based off responses +2. **Metrics** - Provides underlying Prometheus metrics endpoint for relay performance metadata +3. **HTTP Interface** - Providing an efficient HTTP endpoint to send requests to + +## Gateway Operator Responsibilities + +1. **Key management** - Keeping the encryption key and respectively the app stakes keys secure. +2. **App stake management** - Staking in the approriate chains +3. **SaaS business support** - Any features in regard to a SaaS business as mentioned in the [overview](overview.md). diff --git a/pkg/pokt/common/json-rpc_ffjson.go b/pkg/pokt/common/json-rpc_ffjson.go index 7785822..eec3b04 100644 --- a/pkg/pokt/common/json-rpc_ffjson.go +++ b/pkg/pokt/common/json-rpc_ffjson.go @@ -6,6 +6,7 @@ package common import ( "bytes" "fmt" + fflib "github.com/pquerna/ffjson/fflib/v1" )