From 131d5248294de63404cc5b1a43fbb2634d651e53 Mon Sep 17 00:00:00 2001 From: Thomas Kammerlocher Date: Tue, 11 Jun 2024 12:44:54 +0200 Subject: [PATCH] feat: added configuration to use a local token registry (#873) * feat: added configuration to use a local token registry * feat: made local token registry default. * chore: adjusted token registry port to not disturb any process at port 8080, so switched to 8091 * chore: adjusted token registry port * feat: added configurability which repository to use for the token-registry --- Dockerfile | 7 ++- README.md | 8 +++ docker-compose.yml | 19 +++++- .../src/MetadataClient.ts | 58 +++++++++++++++---- scripts/export_env.sh | 8 ++- scripts/token-registry-init.sh | 21 +++++++ 6 files changed, 108 insertions(+), 13 deletions(-) create mode 100755 scripts/token-registry-init.sh diff --git a/Dockerfile b/Dockerfile index 460d6752..89496f65 100644 --- a/Dockerfile +++ b/Dockerfile @@ -59,7 +59,8 @@ WORKDIR /src FROM ubuntu-nodejs as background ARG NETWORK=mainnet -ARG METADATA_SERVER_URI="https://tokens.cardano.org" +# using local token registry as default +ARG METADATA_SERVER_URI="http://token-metadata-registry:8091" RUN apt-get update -y && apt-get install lsb-release -y RUN curl --proto '=https' --tlsv1.2 -sSf -L https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - &&\ echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list &&\ @@ -115,3 +116,7 @@ COPY config/network/${NETWORK}/cardano-node /config/cardano-node/ WORKDIR /app/packages/server/dist EXPOSE 3100 CMD ["node", "index.js"] + +FROM cardanofoundation/cf-token-metadata-registry-api:latest as token-registry +ADD scripts/token-registry-init.sh /app/entrypoint.sh +ENTRYPOINT sh /app/entrypoint.sh diff --git a/README.md b/README.md index adac6441..b9def4f7 100644 --- a/README.md +++ b/README.md @@ -207,6 +207,14 @@ docker compose -p preview down +### Use global Token Metadata Registry +The public Token metadata registry has a limit of daily requests, this can lead to long sync times, when resyncing from scratch. +If it's still needed to run with the global environment it's possible by removing the `token-metadata-registry` from `docker-compose.yml`. +And start it with: +``` +METADATA_SERVER_URI="https://tokens.cardano.org" docker compose up -d +``` + ### Upgrade Database to Postgres 14 If you are upgrading from Postgres 11 to 14: A resync will be needed. To speed up the process you can use the following snapshots: diff --git a/docker-compose.yml b/docker-compose.yml index edd08902..46879f64 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -106,7 +106,7 @@ services: - "postgres" environment: - LOGGER_MIN_SEVERITY=${LOGGER_MIN_SEVERITY:-info} - - METADATA_SERVER_URI=${METADATA_SERVER_URI:-https://tokens.cardano.org} + - METADATA_SERVER_URI=${METADATA_SERVER_URI:-http://token-metadata-registry:8080} restart: on-failure secrets: - postgres_db @@ -142,6 +142,23 @@ services: max-size: "200k" max-file: "10" + token-metadata-registry: + build: + context: . + target: token-registry + ports: + - ${TOKEN_REGISTRY_PORT:-8080}:8080 + environment: + - TOKEN_METADATA_SYNC_JOB=true + - POSTGRES_PORT=${POSTGRES_PORT:-5432} + - POSTGRES_HOST=postgres + - DB_SCHEMA=tokenregistry + - NETWORK=${NETWORK:-mainnet} + secrets: + - postgres_db + - postgres_password + - postgres_user + secrets: postgres_db: file: ./placeholder-secrets/postgres_db diff --git a/packages/api-cardano-db-hasura/src/MetadataClient.ts b/packages/api-cardano-db-hasura/src/MetadataClient.ts index 19efb6b7..13317bb8 100644 --- a/packages/api-cardano-db-hasura/src/MetadataClient.ts +++ b/packages/api-cardano-db-hasura/src/MetadataClient.ts @@ -3,6 +3,7 @@ import { errors, RunnableModuleState } from '@cardano-graphql/util' import { dummyLogger, Logger } from 'ts-log' import { AssetMetadata } from './AssetMetadata' import { Asset } from './graphql_types' +import pRetry from 'p-retry' const MODULE_NAME = 'MetadataFetchClient' @@ -20,16 +21,51 @@ export class MetadataClient { }) } - private async ensureMetadataServerIsAvailable (): Promise { - try { - await this.axiosClient.get('/metadata/healthcheck') - } catch (error) { - if (error.code === 'ENOTFOUND') { - throw new errors.HostDoesNotExist('metadata server') - } else if (error.response?.status !== 404) { - throw error + private async ensureLocalMetadataServerIsAvailable (): Promise { + await pRetry( + async () => { + try { + await this.axiosClient.get('/health') + } catch (error) { + if (error.code === 'ENOTFOUND') { + this.logger.info('Waiting for TokenRegistry to be available') + throw new errors.HostDoesNotExist('metadata server') + } else if (error.response?.status === 400) { // Needed until TokenRegistry is updated + this.logger.info('Token Registry is up') + } else if (error.response?.status !== 404) { + this.logger.info('Metadata Server unreachable.') + throw error + } + } + }, { + factor: 1.5, + retries: 10 } - } + ) + } + + private async waitForLocalMetadataServerSynced (): Promise { + await pRetry( + async () => { + try { + const result = await this.axiosClient.get('/health') + if (!result.data.synced) { + this.logger.info('Metadata registry is still syncing. This can take up to 15min...') + throw new Error('') + } + } catch (error) { + if (error.response?.status === 400) { + this.logger.info('external Registry is up and running') // Needed until TokenRegistry is updated + } else { + throw new Error('') + } + } + }, { + factor: 1.5, + retries: 1000, + minTimeout: 60000 // first try after one minute + } + ) } public async fetch (assetIds: Asset['assetId'][]): Promise { @@ -62,7 +98,9 @@ export class MetadataClient { if (this.state !== null) return this.state = 'initializing' this.logger.info({ module: MODULE_NAME }, 'Initializing') - await this.ensureMetadataServerIsAvailable() + await this.ensureLocalMetadataServerIsAvailable() + this.logger.info({ module: MODULE_NAME }, 'Metadata Server is up and running. Checking Sync Status.') + await this.waitForLocalMetadataServerSynced() this.state = 'initialized' this.logger.info({ module: MODULE_NAME }, 'Initialized') } diff --git a/scripts/export_env.sh b/scripts/export_env.sh index 9ea9863d..2e8dd7b8 100755 --- a/scripts/export_env.sh +++ b/scripts/export_env.sh @@ -11,10 +11,11 @@ case "$NETWORK" in mainnet) API_PORT=3100 HASURA_PORT=8090 - METADATA_SERVER_URI="https://tokens.cardano.org" + METADATA_SERVER_URI="http://localhost:8080" OGMIOS_PORT=1337 PG_ADMIN_PORT=8442 POSTGRES_PORT=5432 + TOKEN_REGISTRY_PORT=8080 ;; testnet) API_PORT=3101 @@ -23,6 +24,7 @@ case "$NETWORK" in OGMIOS_PORT=1338 PG_ADMIN_PORT=8443 POSTGRES_PORT=5443 + TOKEN_REGISTRY_PORT=8081 export CARDANO_NODE_VERSION=8.7.3 ;; preprod) @@ -32,6 +34,7 @@ case "$NETWORK" in OGMIOS_PORT=1339 PG_ADMIN_PORT=8444 POSTGRES_PORT=5444 + TOKEN_REGISTRY_PORT=8082 ;; preview) API_PORT=3103 @@ -40,6 +43,7 @@ case "$NETWORK" in OGMIOS_PORT=1340 PG_ADMIN_PORT=8445 POSTGRES_PORT=5445 + TOKEN_REGISTRY_PORT=8083 ;; vasil-dev) API_PORT=3104 @@ -48,6 +52,7 @@ case "$NETWORK" in OGMIOS_PORT=1341 PG_ADMIN_PORT=8446 POSTGRES_PORT=5446 + TOKEN_REGISTRY_PORT=8084 ;; esac @@ -70,3 +75,4 @@ export PG_ADMIN_PORT export POSTGRES_PORT export POSTGRES_USER_FILE=${SECRETS_DIR}/postgres_user export POSTGRES_HOST=localhost +export TOKEN_REGISTRY_TAG=latest diff --git a/scripts/token-registry-init.sh b/scripts/token-registry-init.sh new file mode 100755 index 00000000..1e902082 --- /dev/null +++ b/scripts/token-registry-init.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +export DB_USERNAME=$(cat /run/secrets/postgres_user) +export DB_PASSWORD=$(cat /run/secrets/postgres_password) +export DB_NAME=$(cat /run/secrets/postgres_db) +export DB_URL=jdbc:postgresql://${POSTGRES_HOST}:${POSTGRES_PORT}/${DB_NAME}?currentSchema=${DB_SCHEMA} + +case "$NETWORK" in + mainnet) + GITHUB_ORGANIZATION=cardano-foundation + GITHUB_PROJECT_NAME=cardano-token-registry + GITHUB_MAPPINGS_FOLDER=mappings + ;; + preprod|preview|testnet) + GITHUB_ORGANIZATION=input-output-hk + GITHUB_PROJECT_NAME=metadata-registry-testnet + GITHUB_MAPPINGS_FOLDER=registry + ;; +esac +echo Using Github Repository $GITHUB_ORGANIZATION/$GITHUB_PROJECT_NAME as token registry +java -jar /app/app.jar \ No newline at end of file