From 2e0fede424b95c4b60e9430f5bf6e84f9d9726fe Mon Sep 17 00:00:00 2001 From: Ragot Geoffrey Date: Fri, 26 Jan 2024 09:32:12 +0100 Subject: [PATCH] fix: search ingestion (#1155) --- .../search-benthos-resources.yaml | 1 + .../configmaps--v1/search-benthos-streams.yaml | 4 ++-- .../monopod-disabled-one-service.yaml | 2 +- .../search-benthos-resources.yaml | 1 + .../configmaps--v1/search-benthos-streams.yaml | 4 ++-- .../deployments-apps-v1/monopod-latest.yaml | 2 +- .../search-benthos-resources.yaml | 1 + .../configmaps--v1/search-benthos-streams.yaml | 4 ++-- .../deployments-apps-v1/monopod-ledgerv1.yaml | 2 +- .../search-benthos-resources.yaml | 1 + .../configmaps--v1/search-benthos-streams.yaml | 4 ++-- .../monopod-payments-after-v1-0-0-alpha-6.yaml | 2 +- .../search-benthos-resources.yaml | 1 + .../configmaps--v1/search-benthos-streams.yaml | 4 ++-- .../monopod-payments-before-v1-0-0-alpha-6.yaml | 2 +- .../search-benthos-resources.yaml | 1 + .../configmaps--v1/search-benthos-streams.yaml | 4 ++-- .../monopod-search-before-v0-7-0.yaml | 2 +- .../search-benthos-resources.yaml | 1 + .../configmaps--v1/search-benthos-streams.yaml | 4 ++-- .../deployments-apps-v1/search-benthos.yaml | 4 ++-- .../search-benthos-resources.yaml | 1 + .../configmaps--v1/search-benthos-streams.yaml | 4 ++-- .../deployments-apps-v1/search-benthos.yaml | 4 ++-- .../search-benthos-resources.yaml | 1 + .../configmaps--v1/search-benthos-streams.yaml | 4 ++-- .../deployments-apps-v1/search-benthos.yaml | 4 ++-- .../search-benthos-resources.yaml | 1 + .../configmaps--v1/search-benthos-streams.yaml | 4 ++-- .../deployments-apps-v1/search-benthos.yaml | 4 ++-- .../search-benthos-resources.yaml | 1 + .../configmaps--v1/search-benthos-streams.yaml | 4 ++-- .../deployments-apps-v1/search-benthos.yaml | 4 ++-- .../search-benthos-resources.yaml | 1 + .../configmaps--v1/search-benthos-streams.yaml | 4 ++-- .../deployments-apps-v1/search-benthos.yaml | 4 ++-- .../search-benthos-resources.yaml | 1 + .../configmaps--v1/search-benthos-streams.yaml | 4 ++-- .../deployments-apps-v1/search-benthos.yaml | 4 ++-- .../search-benthos-resources.yaml | 1 + .../configmaps--v1/search-benthos-streams.yaml | 4 ++-- .../deployments-apps-v1/search-benthos.yaml | 4 ++-- .../search-benthos-resources.yaml | 1 + .../configmaps--v1/search-benthos-streams.yaml | 4 ++-- .../deployments-apps-v1/search-benthos.yaml | 4 ++-- .../search-benthos-resources.yaml | 1 + .../configmaps--v1/search-benthos-streams.yaml | 4 ++-- .../deployments-apps-v1/search-benthos.yaml | 4 ++-- .../search-benthos-resources.yaml | 1 + .../configmaps--v1/search-benthos-streams.yaml | 4 ++-- .../deployments-apps-v1/search-benthos.yaml | 4 ++-- .../search-benthos-resources.yaml | 1 + .../configmaps--v1/search-benthos-streams.yaml | 4 ++-- .../deployments-apps-v1/search-benthos.yaml | 4 ++-- .../search-benthos-resources.yaml | 1 + .../configmaps--v1/search-benthos-streams.yaml | 4 ++-- .../deployments-apps-v1/search-benthos.yaml | 4 ++-- .../search-benthos-resources.yaml | 1 + .../configmaps--v1/search-benthos-streams.yaml | 4 ++-- .../deployments-apps-v1/search-benthos.yaml | 4 ++-- .../search-benthos-resources.yaml | 1 + .../configmaps--v1/search-benthos-streams.yaml | 4 ++-- .../deployments-apps-v1/search-benthos.yaml | 4 ++-- .../search-benthos-resources.yaml | 1 + .../configmaps--v1/search-benthos-streams.yaml | 4 ++-- .../deployments-apps-v1/search-benthos.yaml | 4 ++-- .../operator/internal/modules/search/handler.go | 2 +- .../benthos/resources/output_elasticsearch.yaml | 3 ++- .../ledger/v1.0.0/ledger_ingestion_v1.yaml | 6 +++++- .../ledger/v2.0.0/ledger_ingestion_v2.yaml | 17 ++++++++++++++--- tests/integration/internal/modules/search.go | 2 +- 71 files changed, 127 insertions(+), 89 deletions(-) diff --git a/components/operator/internal/controllers/stack/testdata/monopod-disabled-one-service/results/configmaps--v1/search-benthos-resources.yaml b/components/operator/internal/controllers/stack/testdata/monopod-disabled-one-service/results/configmaps--v1/search-benthos-resources.yaml index 8eac725cfd..85f2716104 100644 --- a/components/operator/internal/controllers/stack/testdata/monopod-disabled-one-service/results/configmaps--v1/search-benthos-resources.yaml +++ b/components/operator/internal/controllers/stack/testdata/monopod-disabled-one-service/results/configmaps--v1/search-benthos-resources.yaml @@ -42,6 +42,7 @@ data: doc: ${! if json().exists("document") { json("document") } else {""} } script: ${! if json().exists("script") { json("script") } else {""} } upsert: ${! if json().exists("upsert") { json("upsert") } else {""} } + params: ${! if json().exists("params") { json("params") } else {""} } kind: ConfigMap metadata: labels: diff --git a/components/operator/internal/controllers/stack/testdata/monopod-disabled-one-service/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/monopod-disabled-one-service/results/configmaps--v1/search-benthos-streams.yaml index b5a9fcf4d1..a802f3f19d 100644 --- a/components/operator/internal/controllers/stack/testdata/monopod-disabled-one-service/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/monopod-disabled-one-service/results/configmaps--v1/search-benthos-streams.yaml @@ -1,7 +1,7 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" - ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value.string()\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": metadata.key,\n \"value\": metadata.value\n },\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(params.key)\",\n \"params\": {\n \"key\": this.payload.key\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_all_v1.yaml: | diff --git a/components/operator/internal/controllers/stack/testdata/monopod-disabled-one-service/results/deployments-apps-v1/monopod-disabled-one-service.yaml b/components/operator/internal/controllers/stack/testdata/monopod-disabled-one-service/results/deployments-apps-v1/monopod-disabled-one-service.yaml index ad5f9cdb80..c71f63e93c 100644 --- a/components/operator/internal/controllers/stack/testdata/monopod-disabled-one-service/results/deployments-apps-v1/monopod-disabled-one-service.yaml +++ b/components/operator/internal/controllers/stack/testdata/monopod-disabled-one-service/results/deployments-apps-v1/monopod-disabled-one-service.yaml @@ -575,7 +575,7 @@ spec: value: $(LEDGER_POSTGRES_NO_DATABASE_URI)/$(LEDGER_POSTGRES_DATABASE)?sslmode=disable - name: BROKER value: kafka - image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.0-es + image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.1-es imagePullPolicy: IfNotPresent name: search-benthos ports: diff --git a/components/operator/internal/controllers/stack/testdata/monopod-latest/results/configmaps--v1/search-benthos-resources.yaml b/components/operator/internal/controllers/stack/testdata/monopod-latest/results/configmaps--v1/search-benthos-resources.yaml index 3cd7de8d7f..72f3f1cf36 100644 --- a/components/operator/internal/controllers/stack/testdata/monopod-latest/results/configmaps--v1/search-benthos-resources.yaml +++ b/components/operator/internal/controllers/stack/testdata/monopod-latest/results/configmaps--v1/search-benthos-resources.yaml @@ -42,6 +42,7 @@ data: doc: ${! if json().exists("document") { json("document") } else {""} } script: ${! if json().exists("script") { json("script") } else {""} } upsert: ${! if json().exists("upsert") { json("upsert") } else {""} } + params: ${! if json().exists("params") { json("params") } else {""} } kind: ConfigMap metadata: labels: diff --git a/components/operator/internal/controllers/stack/testdata/monopod-latest/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/monopod-latest/results/configmaps--v1/search-benthos-streams.yaml index 99b149e96b..0d0c47639d 100644 --- a/components/operator/internal/controllers/stack/testdata/monopod-latest/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/monopod-latest/results/configmaps--v1/search-benthos-streams.yaml @@ -1,7 +1,7 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" - ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value.string()\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": metadata.key,\n \"value\": metadata.value\n },\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(params.key)\",\n \"params\": {\n \"key\": this.payload.key\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_all_v1.yaml: | diff --git a/components/operator/internal/controllers/stack/testdata/monopod-latest/results/deployments-apps-v1/monopod-latest.yaml b/components/operator/internal/controllers/stack/testdata/monopod-latest/results/deployments-apps-v1/monopod-latest.yaml index fb9bef382e..47646cdc72 100644 --- a/components/operator/internal/controllers/stack/testdata/monopod-latest/results/deployments-apps-v1/monopod-latest.yaml +++ b/components/operator/internal/controllers/stack/testdata/monopod-latest/results/deployments-apps-v1/monopod-latest.yaml @@ -822,7 +822,7 @@ spec: value: $(LEDGER_POSTGRES_NO_DATABASE_URI)/$(LEDGER_POSTGRES_DATABASE)?sslmode=disable - name: BROKER value: kafka - image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.0-es + image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.1-es imagePullPolicy: IfNotPresent name: search-benthos ports: diff --git a/components/operator/internal/controllers/stack/testdata/monopod-ledgerv1/results/configmaps--v1/search-benthos-resources.yaml b/components/operator/internal/controllers/stack/testdata/monopod-ledgerv1/results/configmaps--v1/search-benthos-resources.yaml index 3bc1357de9..d4870b7b4d 100644 --- a/components/operator/internal/controllers/stack/testdata/monopod-ledgerv1/results/configmaps--v1/search-benthos-resources.yaml +++ b/components/operator/internal/controllers/stack/testdata/monopod-ledgerv1/results/configmaps--v1/search-benthos-resources.yaml @@ -42,6 +42,7 @@ data: doc: ${! if json().exists("document") { json("document") } else {""} } script: ${! if json().exists("script") { json("script") } else {""} } upsert: ${! if json().exists("upsert") { json("upsert") } else {""} } + params: ${! if json().exists("params") { json("params") } else {""} } kind: ConfigMap metadata: labels: diff --git a/components/operator/internal/controllers/stack/testdata/monopod-ledgerv1/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/monopod-ledgerv1/results/configmaps--v1/search-benthos-streams.yaml index 3591f216bf..0a1d349cc5 100644 --- a/components/operator/internal/controllers/stack/testdata/monopod-ledgerv1/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/monopod-ledgerv1/results/configmaps--v1/search-benthos-streams.yaml @@ -1,7 +1,7 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" - ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value.string()\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": metadata.key,\n \"value\": metadata.value\n },\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(params.key)\",\n \"params\": {\n \"key\": this.payload.key\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_all_v1.yaml: | diff --git a/components/operator/internal/controllers/stack/testdata/monopod-ledgerv1/results/deployments-apps-v1/monopod-ledgerv1.yaml b/components/operator/internal/controllers/stack/testdata/monopod-ledgerv1/results/deployments-apps-v1/monopod-ledgerv1.yaml index af7cc478c1..efc0f264c4 100644 --- a/components/operator/internal/controllers/stack/testdata/monopod-ledgerv1/results/deployments-apps-v1/monopod-ledgerv1.yaml +++ b/components/operator/internal/controllers/stack/testdata/monopod-ledgerv1/results/deployments-apps-v1/monopod-ledgerv1.yaml @@ -824,7 +824,7 @@ spec: value: $(LEDGER_POSTGRES_NO_DATABASE_URI)/$(LEDGER_POSTGRES_DATABASE)?sslmode=disable - name: BROKER value: kafka - image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.0-es + image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.1-es imagePullPolicy: IfNotPresent name: search-benthos ports: diff --git a/components/operator/internal/controllers/stack/testdata/monopod-payments-after-v1.0.0-alpha.6/results/configmaps--v1/search-benthos-resources.yaml b/components/operator/internal/controllers/stack/testdata/monopod-payments-after-v1.0.0-alpha.6/results/configmaps--v1/search-benthos-resources.yaml index a944863240..d8eb0e5bbe 100644 --- a/components/operator/internal/controllers/stack/testdata/monopod-payments-after-v1.0.0-alpha.6/results/configmaps--v1/search-benthos-resources.yaml +++ b/components/operator/internal/controllers/stack/testdata/monopod-payments-after-v1.0.0-alpha.6/results/configmaps--v1/search-benthos-resources.yaml @@ -42,6 +42,7 @@ data: doc: ${! if json().exists("document") { json("document") } else {""} } script: ${! if json().exists("script") { json("script") } else {""} } upsert: ${! if json().exists("upsert") { json("upsert") } else {""} } + params: ${! if json().exists("params") { json("params") } else {""} } kind: ConfigMap metadata: labels: diff --git a/components/operator/internal/controllers/stack/testdata/monopod-payments-after-v1.0.0-alpha.6/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/monopod-payments-after-v1.0.0-alpha.6/results/configmaps--v1/search-benthos-streams.yaml index c293590e51..a75b68f31a 100644 --- a/components/operator/internal/controllers/stack/testdata/monopod-payments-after-v1.0.0-alpha.6/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/monopod-payments-after-v1.0.0-alpha.6/results/configmaps--v1/search-benthos-streams.yaml @@ -1,7 +1,7 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" - ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value.string()\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": metadata.key,\n \"value\": metadata.value\n },\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(params.key)\",\n \"params\": {\n \"key\": this.payload.key\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_all_v1.yaml: | diff --git a/components/operator/internal/controllers/stack/testdata/monopod-payments-after-v1.0.0-alpha.6/results/deployments-apps-v1/monopod-payments-after-v1-0-0-alpha-6.yaml b/components/operator/internal/controllers/stack/testdata/monopod-payments-after-v1.0.0-alpha.6/results/deployments-apps-v1/monopod-payments-after-v1-0-0-alpha-6.yaml index e47e9353f9..6958ecb746 100644 --- a/components/operator/internal/controllers/stack/testdata/monopod-payments-after-v1.0.0-alpha.6/results/deployments-apps-v1/monopod-payments-after-v1-0-0-alpha-6.yaml +++ b/components/operator/internal/controllers/stack/testdata/monopod-payments-after-v1.0.0-alpha.6/results/deployments-apps-v1/monopod-payments-after-v1-0-0-alpha-6.yaml @@ -822,7 +822,7 @@ spec: value: $(LEDGER_POSTGRES_NO_DATABASE_URI)/$(LEDGER_POSTGRES_DATABASE)?sslmode=disable - name: BROKER value: kafka - image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.0-es + image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.1-es imagePullPolicy: IfNotPresent name: search-benthos ports: diff --git a/components/operator/internal/controllers/stack/testdata/monopod-payments-before-v1.0.0-alpha.6/results/configmaps--v1/search-benthos-resources.yaml b/components/operator/internal/controllers/stack/testdata/monopod-payments-before-v1.0.0-alpha.6/results/configmaps--v1/search-benthos-resources.yaml index be4826a273..fdc33ec725 100644 --- a/components/operator/internal/controllers/stack/testdata/monopod-payments-before-v1.0.0-alpha.6/results/configmaps--v1/search-benthos-resources.yaml +++ b/components/operator/internal/controllers/stack/testdata/monopod-payments-before-v1.0.0-alpha.6/results/configmaps--v1/search-benthos-resources.yaml @@ -42,6 +42,7 @@ data: doc: ${! if json().exists("document") { json("document") } else {""} } script: ${! if json().exists("script") { json("script") } else {""} } upsert: ${! if json().exists("upsert") { json("upsert") } else {""} } + params: ${! if json().exists("params") { json("params") } else {""} } kind: ConfigMap metadata: labels: diff --git a/components/operator/internal/controllers/stack/testdata/monopod-payments-before-v1.0.0-alpha.6/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/monopod-payments-before-v1.0.0-alpha.6/results/configmaps--v1/search-benthos-streams.yaml index 1accc20e81..c5d1801e60 100644 --- a/components/operator/internal/controllers/stack/testdata/monopod-payments-before-v1.0.0-alpha.6/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/monopod-payments-before-v1.0.0-alpha.6/results/configmaps--v1/search-benthos-streams.yaml @@ -1,7 +1,7 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" - ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value.string()\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": metadata.key,\n \"value\": metadata.value\n },\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(params.key)\",\n \"params\": {\n \"key\": this.payload.key\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_all_v1.yaml: | diff --git a/components/operator/internal/controllers/stack/testdata/monopod-payments-before-v1.0.0-alpha.6/results/deployments-apps-v1/monopod-payments-before-v1-0-0-alpha-6.yaml b/components/operator/internal/controllers/stack/testdata/monopod-payments-before-v1.0.0-alpha.6/results/deployments-apps-v1/monopod-payments-before-v1-0-0-alpha-6.yaml index 757ee08474..b1dbe233da 100644 --- a/components/operator/internal/controllers/stack/testdata/monopod-payments-before-v1.0.0-alpha.6/results/deployments-apps-v1/monopod-payments-before-v1-0-0-alpha-6.yaml +++ b/components/operator/internal/controllers/stack/testdata/monopod-payments-before-v1.0.0-alpha.6/results/deployments-apps-v1/monopod-payments-before-v1-0-0-alpha-6.yaml @@ -732,7 +732,7 @@ spec: value: $(LEDGER_POSTGRES_NO_DATABASE_URI)/$(LEDGER_POSTGRES_DATABASE)?sslmode=disable - name: BROKER value: kafka - image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.0-es + image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.1-es imagePullPolicy: IfNotPresent name: search-benthos ports: diff --git a/components/operator/internal/controllers/stack/testdata/monopod-search-before-v0.7.0/results/configmaps--v1/search-benthos-resources.yaml b/components/operator/internal/controllers/stack/testdata/monopod-search-before-v0.7.0/results/configmaps--v1/search-benthos-resources.yaml index 7a4c49aeb9..d48148570a 100644 --- a/components/operator/internal/controllers/stack/testdata/monopod-search-before-v0.7.0/results/configmaps--v1/search-benthos-resources.yaml +++ b/components/operator/internal/controllers/stack/testdata/monopod-search-before-v0.7.0/results/configmaps--v1/search-benthos-resources.yaml @@ -42,6 +42,7 @@ data: doc: ${! if json().exists("document") { json("document") } else {""} } script: ${! if json().exists("script") { json("script") } else {""} } upsert: ${! if json().exists("upsert") { json("upsert") } else {""} } + params: ${! if json().exists("params") { json("params") } else {""} } kind: ConfigMap metadata: labels: diff --git a/components/operator/internal/controllers/stack/testdata/monopod-search-before-v0.7.0/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/monopod-search-before-v0.7.0/results/configmaps--v1/search-benthos-streams.yaml index af881e985a..8216fba369 100644 --- a/components/operator/internal/controllers/stack/testdata/monopod-search-before-v0.7.0/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/monopod-search-before-v0.7.0/results/configmaps--v1/search-benthos-streams.yaml @@ -1,7 +1,7 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" - ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value.string()\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": metadata.key,\n \"value\": metadata.value\n },\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(params.key)\",\n \"params\": {\n \"key\": this.payload.key\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_all_v1.yaml: | diff --git a/components/operator/internal/controllers/stack/testdata/monopod-search-before-v0.7.0/results/deployments-apps-v1/monopod-search-before-v0-7-0.yaml b/components/operator/internal/controllers/stack/testdata/monopod-search-before-v0.7.0/results/deployments-apps-v1/monopod-search-before-v0-7-0.yaml index 5b8075b7f9..008d266a46 100644 --- a/components/operator/internal/controllers/stack/testdata/monopod-search-before-v0.7.0/results/deployments-apps-v1/monopod-search-before-v0-7-0.yaml +++ b/components/operator/internal/controllers/stack/testdata/monopod-search-before-v0.7.0/results/deployments-apps-v1/monopod-search-before-v0-7-0.yaml @@ -822,7 +822,7 @@ spec: value: $(LEDGER_POSTGRES_NO_DATABASE_URI)/$(LEDGER_POSTGRES_DATABASE)?sslmode=disable - name: BROKER value: kafka - image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.0-es + image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.1-es imagePullPolicy: IfNotPresent name: search-benthos ports: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-debug/results/configmaps--v1/search-benthos-resources.yaml b/components/operator/internal/controllers/stack/testdata/multipod-debug/results/configmaps--v1/search-benthos-resources.yaml index 6a17dae801..da5b5f5b5c 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-debug/results/configmaps--v1/search-benthos-resources.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-debug/results/configmaps--v1/search-benthos-resources.yaml @@ -42,6 +42,7 @@ data: doc: ${! if json().exists("document") { json("document") } else {""} } script: ${! if json().exists("script") { json("script") } else {""} } upsert: ${! if json().exists("upsert") { json("upsert") } else {""} } + params: ${! if json().exists("params") { json("params") } else {""} } kind: ConfigMap metadata: labels: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-debug/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-debug/results/configmaps--v1/search-benthos-streams.yaml index 3d6a19484f..785dcffd72 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-debug/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-debug/results/configmaps--v1/search-benthos-streams.yaml @@ -1,7 +1,7 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" - ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value.string()\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": metadata.key,\n \"value\": metadata.value\n },\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(params.key)\",\n \"params\": {\n \"key\": this.payload.key\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_all_v1.yaml: | diff --git a/components/operator/internal/controllers/stack/testdata/multipod-debug/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-debug/results/deployments-apps-v1/search-benthos.yaml index 093fb767e9..a485347a22 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-debug/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-debug/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: Zas1IqJbhfN4PIGns5xxVW6s2zu7A-qQ0uJgR82mbNQ= + stack.formance.cloud/volumes-hash: oAXxeJ9yZ42WF5KLqqbY5iIu425cXeLlIyAQnwTJV_4= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos @@ -103,7 +103,7 @@ spec: value: $(LEDGER_POSTGRES_NO_DATABASE_URI)/$(LEDGER_POSTGRES_DATABASE)?sslmode=disable - name: BROKER value: kafka - image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.0-es + image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.1-es imagePullPolicy: IfNotPresent name: search-benthos ports: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-disabled-one-service/results/configmaps--v1/search-benthos-resources.yaml b/components/operator/internal/controllers/stack/testdata/multipod-disabled-one-service/results/configmaps--v1/search-benthos-resources.yaml index 17e52fcd70..aeb9f9b433 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-disabled-one-service/results/configmaps--v1/search-benthos-resources.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-disabled-one-service/results/configmaps--v1/search-benthos-resources.yaml @@ -42,6 +42,7 @@ data: doc: ${! if json().exists("document") { json("document") } else {""} } script: ${! if json().exists("script") { json("script") } else {""} } upsert: ${! if json().exists("upsert") { json("upsert") } else {""} } + params: ${! if json().exists("params") { json("params") } else {""} } kind: ConfigMap metadata: labels: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-disabled-one-service/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-disabled-one-service/results/configmaps--v1/search-benthos-streams.yaml index 04c1110e96..c9302422e3 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-disabled-one-service/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-disabled-one-service/results/configmaps--v1/search-benthos-streams.yaml @@ -1,7 +1,7 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" - ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value.string()\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": metadata.key,\n \"value\": metadata.value\n },\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(params.key)\",\n \"params\": {\n \"key\": this.payload.key\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_all_v1.yaml: | diff --git a/components/operator/internal/controllers/stack/testdata/multipod-disabled-one-service/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-disabled-one-service/results/deployments-apps-v1/search-benthos.yaml index a4788c207a..2a5e940c20 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-disabled-one-service/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-disabled-one-service/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: Zas1IqJbhfN4PIGns5xxVW6s2zu7A-qQ0uJgR82mbNQ= + stack.formance.cloud/volumes-hash: oAXxeJ9yZ42WF5KLqqbY5iIu425cXeLlIyAQnwTJV_4= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos @@ -103,7 +103,7 @@ spec: value: $(LEDGER_POSTGRES_NO_DATABASE_URI)/$(LEDGER_POSTGRES_DATABASE)?sslmode=disable - name: BROKER value: kafka - image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.0-es + image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.1-es imagePullPolicy: IfNotPresent name: search-benthos ports: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-latest-no-monitoring/results/configmaps--v1/search-benthos-resources.yaml b/components/operator/internal/controllers/stack/testdata/multipod-latest-no-monitoring/results/configmaps--v1/search-benthos-resources.yaml index 816c547281..c8ee6903ad 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-latest-no-monitoring/results/configmaps--v1/search-benthos-resources.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-latest-no-monitoring/results/configmaps--v1/search-benthos-resources.yaml @@ -42,6 +42,7 @@ data: doc: ${! if json().exists("document") { json("document") } else {""} } script: ${! if json().exists("script") { json("script") } else {""} } upsert: ${! if json().exists("upsert") { json("upsert") } else {""} } + params: ${! if json().exists("params") { json("params") } else {""} } kind: ConfigMap metadata: labels: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-latest-no-monitoring/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-latest-no-monitoring/results/configmaps--v1/search-benthos-streams.yaml index 41674e812f..ec0aed2494 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-latest-no-monitoring/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-latest-no-monitoring/results/configmaps--v1/search-benthos-streams.yaml @@ -1,7 +1,7 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" - ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value.string()\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": metadata.key,\n \"value\": metadata.value\n },\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(params.key)\",\n \"params\": {\n \"key\": this.payload.key\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_all_v1.yaml: | diff --git a/components/operator/internal/controllers/stack/testdata/multipod-latest-no-monitoring/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-latest-no-monitoring/results/deployments-apps-v1/search-benthos.yaml index 3dabe64b36..8896ccee27 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-latest-no-monitoring/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-latest-no-monitoring/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: EzO6HC1Z4jk4Ld2Ua4vIfjNEaYrtFQPY3G06P14GkrU= + stack.formance.cloud/volumes-hash: SKMuqJo00TeiIbsAJrChX-gmffYXjf1omWjMmImsS4k= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos @@ -84,7 +84,7 @@ spec: value: $(LEDGER_POSTGRES_NO_DATABASE_URI)/$(LEDGER_POSTGRES_DATABASE)?sslmode=disable - name: BROKER value: kafka - image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.0-es + image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.1-es imagePullPolicy: IfNotPresent name: search-benthos ports: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-check-scopes/results/configmaps--v1/search-benthos-resources.yaml b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-check-scopes/results/configmaps--v1/search-benthos-resources.yaml index 301e66f907..f8289834b7 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-check-scopes/results/configmaps--v1/search-benthos-resources.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-check-scopes/results/configmaps--v1/search-benthos-resources.yaml @@ -42,6 +42,7 @@ data: doc: ${! if json().exists("document") { json("document") } else {""} } script: ${! if json().exists("script") { json("script") } else {""} } upsert: ${! if json().exists("upsert") { json("upsert") } else {""} } + params: ${! if json().exists("params") { json("params") } else {""} } kind: ConfigMap metadata: labels: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-check-scopes/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-check-scopes/results/configmaps--v1/search-benthos-streams.yaml index afac799af1..6526146fd1 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-check-scopes/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-check-scopes/results/configmaps--v1/search-benthos-streams.yaml @@ -1,7 +1,7 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" - ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value.string()\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": metadata.key,\n \"value\": metadata.value\n },\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(params.key)\",\n \"params\": {\n \"key\": this.payload.key\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_all_v1.yaml: | diff --git a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-check-scopes/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-check-scopes/results/deployments-apps-v1/search-benthos.yaml index 33a97b9677..92e8b3c0b9 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-check-scopes/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-check-scopes/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: Zas1IqJbhfN4PIGns5xxVW6s2zu7A-qQ0uJgR82mbNQ= + stack.formance.cloud/volumes-hash: oAXxeJ9yZ42WF5KLqqbY5iIu425cXeLlIyAQnwTJV_4= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos @@ -103,7 +103,7 @@ spec: value: $(LEDGER_POSTGRES_NO_DATABASE_URI)/$(LEDGER_POSTGRES_DATABASE)?sslmode=disable - name: BROKER value: kafka - image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.0-es + image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.1-es imagePullPolicy: IfNotPresent name: search-benthos ports: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-custom-registry/results/configmaps--v1/search-benthos-resources.yaml b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-custom-registry/results/configmaps--v1/search-benthos-resources.yaml index 61316bb407..c1373b3989 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-custom-registry/results/configmaps--v1/search-benthos-resources.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-custom-registry/results/configmaps--v1/search-benthos-resources.yaml @@ -42,6 +42,7 @@ data: doc: ${! if json().exists("document") { json("document") } else {""} } script: ${! if json().exists("script") { json("script") } else {""} } upsert: ${! if json().exists("upsert") { json("upsert") } else {""} } + params: ${! if json().exists("params") { json("params") } else {""} } kind: ConfigMap metadata: labels: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-custom-registry/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-custom-registry/results/configmaps--v1/search-benthos-streams.yaml index 43ab792fa8..2995e702dc 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-custom-registry/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-custom-registry/results/configmaps--v1/search-benthos-streams.yaml @@ -1,7 +1,7 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" - ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value.string()\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": metadata.key,\n \"value\": metadata.value\n },\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(params.key)\",\n \"params\": {\n \"key\": this.payload.key\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_all_v1.yaml: | diff --git a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-custom-registry/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-custom-registry/results/deployments-apps-v1/search-benthos.yaml index 7d9979cfbe..22d6037c27 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-custom-registry/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-custom-registry/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: Zas1IqJbhfN4PIGns5xxVW6s2zu7A-qQ0uJgR82mbNQ= + stack.formance.cloud/volumes-hash: oAXxeJ9yZ42WF5KLqqbY5iIu425cXeLlIyAQnwTJV_4= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos @@ -103,7 +103,7 @@ spec: value: $(LEDGER_POSTGRES_NO_DATABASE_URI)/$(LEDGER_POSTGRES_DATABASE)?sslmode=disable - name: BROKER value: kafka - image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.0-es + image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.1-es imagePullPolicy: IfNotPresent name: search-benthos ports: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-gateway-fallback/results/configmaps--v1/search-benthos-resources.yaml b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-gateway-fallback/results/configmaps--v1/search-benthos-resources.yaml index 0160b04c3c..9a36e8a4ad 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-gateway-fallback/results/configmaps--v1/search-benthos-resources.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-gateway-fallback/results/configmaps--v1/search-benthos-resources.yaml @@ -42,6 +42,7 @@ data: doc: ${! if json().exists("document") { json("document") } else {""} } script: ${! if json().exists("script") { json("script") } else {""} } upsert: ${! if json().exists("upsert") { json("upsert") } else {""} } + params: ${! if json().exists("params") { json("params") } else {""} } kind: ConfigMap metadata: labels: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-gateway-fallback/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-gateway-fallback/results/configmaps--v1/search-benthos-streams.yaml index bb650d12bc..f2625ce247 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-gateway-fallback/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-gateway-fallback/results/configmaps--v1/search-benthos-streams.yaml @@ -1,7 +1,7 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" - ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value.string()\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": metadata.key,\n \"value\": metadata.value\n },\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(params.key)\",\n \"params\": {\n \"key\": this.payload.key\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_all_v1.yaml: | diff --git a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-gateway-fallback/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-gateway-fallback/results/deployments-apps-v1/search-benthos.yaml index 117793b6cd..4d318e9dc1 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-gateway-fallback/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-gateway-fallback/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: Zas1IqJbhfN4PIGns5xxVW6s2zu7A-qQ0uJgR82mbNQ= + stack.formance.cloud/volumes-hash: oAXxeJ9yZ42WF5KLqqbY5iIu425cXeLlIyAQnwTJV_4= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos @@ -103,7 +103,7 @@ spec: value: $(LEDGER_POSTGRES_NO_DATABASE_URI)/$(LEDGER_POSTGRES_DATABASE)?sslmode=disable - name: BROKER value: kafka - image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.0-es + image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.1-es imagePullPolicy: IfNotPresent name: search-benthos ports: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-secrets/results/configmaps--v1/search-benthos-resources.yaml b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-secrets/results/configmaps--v1/search-benthos-resources.yaml index 936795e857..d11d0d4431 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-secrets/results/configmaps--v1/search-benthos-resources.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-secrets/results/configmaps--v1/search-benthos-resources.yaml @@ -42,6 +42,7 @@ data: doc: ${! if json().exists("document") { json("document") } else {""} } script: ${! if json().exists("script") { json("script") } else {""} } upsert: ${! if json().exists("upsert") { json("upsert") } else {""} } + params: ${! if json().exists("params") { json("params") } else {""} } kind: ConfigMap metadata: labels: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-secrets/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-secrets/results/configmaps--v1/search-benthos-streams.yaml index 2abc68bab8..495caa8a1d 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-secrets/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-secrets/results/configmaps--v1/search-benthos-streams.yaml @@ -1,7 +1,7 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" - ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value.string()\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": metadata.key,\n \"value\": metadata.value\n },\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(params.key)\",\n \"params\": {\n \"key\": this.payload.key\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_all_v1.yaml: | diff --git a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-secrets/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-secrets/results/deployments-apps-v1/search-benthos.yaml index ecf1ec8903..f94e99c30e 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-latest-with-secrets/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-latest-with-secrets/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: Zas1IqJbhfN4PIGns5xxVW6s2zu7A-qQ0uJgR82mbNQ= + stack.formance.cloud/volumes-hash: oAXxeJ9yZ42WF5KLqqbY5iIu425cXeLlIyAQnwTJV_4= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos @@ -109,7 +109,7 @@ spec: value: $(LEDGER_POSTGRES_NO_DATABASE_URI)/$(LEDGER_POSTGRES_DATABASE)?sslmode=disable - name: BROKER value: kafka - image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.0-es + image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.1-es imagePullPolicy: IfNotPresent name: search-benthos ports: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-latest/results/configmaps--v1/search-benthos-resources.yaml b/components/operator/internal/controllers/stack/testdata/multipod-latest/results/configmaps--v1/search-benthos-resources.yaml index 7577669007..661a8d8010 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-latest/results/configmaps--v1/search-benthos-resources.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-latest/results/configmaps--v1/search-benthos-resources.yaml @@ -42,6 +42,7 @@ data: doc: ${! if json().exists("document") { json("document") } else {""} } script: ${! if json().exists("script") { json("script") } else {""} } upsert: ${! if json().exists("upsert") { json("upsert") } else {""} } + params: ${! if json().exists("params") { json("params") } else {""} } kind: ConfigMap metadata: labels: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-latest/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-latest/results/configmaps--v1/search-benthos-streams.yaml index 63a287cfae..f2e1a4f24a 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-latest/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-latest/results/configmaps--v1/search-benthos-streams.yaml @@ -1,7 +1,7 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" - ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value.string()\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": metadata.key,\n \"value\": metadata.value\n },\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(params.key)\",\n \"params\": {\n \"key\": this.payload.key\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_all_v1.yaml: | diff --git a/components/operator/internal/controllers/stack/testdata/multipod-latest/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-latest/results/deployments-apps-v1/search-benthos.yaml index a43276fd94..7ffbdbfd03 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-latest/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-latest/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: Zas1IqJbhfN4PIGns5xxVW6s2zu7A-qQ0uJgR82mbNQ= + stack.formance.cloud/volumes-hash: oAXxeJ9yZ42WF5KLqqbY5iIu425cXeLlIyAQnwTJV_4= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos @@ -103,7 +103,7 @@ spec: value: $(LEDGER_POSTGRES_NO_DATABASE_URI)/$(LEDGER_POSTGRES_DATABASE)?sslmode=disable - name: BROKER value: kafka - image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.0-es + image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.1-es imagePullPolicy: IfNotPresent name: search-benthos ports: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-ledger-v2.0.0-beta.7/results/configmaps--v1/search-benthos-resources.yaml b/components/operator/internal/controllers/stack/testdata/multipod-ledger-v2.0.0-beta.7/results/configmaps--v1/search-benthos-resources.yaml index 89d5ddee89..af5e2939d7 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-ledger-v2.0.0-beta.7/results/configmaps--v1/search-benthos-resources.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-ledger-v2.0.0-beta.7/results/configmaps--v1/search-benthos-resources.yaml @@ -42,6 +42,7 @@ data: doc: ${! if json().exists("document") { json("document") } else {""} } script: ${! if json().exists("script") { json("script") } else {""} } upsert: ${! if json().exists("upsert") { json("upsert") } else {""} } + params: ${! if json().exists("params") { json("params") } else {""} } kind: ConfigMap metadata: labels: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-ledger-v2.0.0-beta.7/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-ledger-v2.0.0-beta.7/results/configmaps--v1/search-benthos-streams.yaml index dab2a46792..950fe472f4 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-ledger-v2.0.0-beta.7/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-ledger-v2.0.0-beta.7/results/configmaps--v1/search-benthos-streams.yaml @@ -1,7 +1,7 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" - ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value.string()\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": metadata.key,\n \"value\": metadata.value\n },\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(params.key)\",\n \"params\": {\n \"key\": this.payload.key\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_all_v1.yaml: | diff --git a/components/operator/internal/controllers/stack/testdata/multipod-ledger-v2.0.0-beta.7/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-ledger-v2.0.0-beta.7/results/deployments-apps-v1/search-benthos.yaml index ba8f246c0e..858611a465 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-ledger-v2.0.0-beta.7/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-ledger-v2.0.0-beta.7/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: Zas1IqJbhfN4PIGns5xxVW6s2zu7A-qQ0uJgR82mbNQ= + stack.formance.cloud/volumes-hash: oAXxeJ9yZ42WF5KLqqbY5iIu425cXeLlIyAQnwTJV_4= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos @@ -103,7 +103,7 @@ spec: value: $(LEDGER_POSTGRES_NO_DATABASE_URI)/$(LEDGER_POSTGRES_DATABASE)?sslmode=disable - name: BROKER value: kafka - image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.0-es + image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.1-es imagePullPolicy: IfNotPresent name: search-benthos ports: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-orchestration-v0.2.0/results/configmaps--v1/search-benthos-resources.yaml b/components/operator/internal/controllers/stack/testdata/multipod-orchestration-v0.2.0/results/configmaps--v1/search-benthos-resources.yaml index d146d81ce3..a786a08f40 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-orchestration-v0.2.0/results/configmaps--v1/search-benthos-resources.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-orchestration-v0.2.0/results/configmaps--v1/search-benthos-resources.yaml @@ -42,6 +42,7 @@ data: doc: ${! if json().exists("document") { json("document") } else {""} } script: ${! if json().exists("script") { json("script") } else {""} } upsert: ${! if json().exists("upsert") { json("upsert") } else {""} } + params: ${! if json().exists("params") { json("params") } else {""} } kind: ConfigMap metadata: labels: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-orchestration-v0.2.0/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-orchestration-v0.2.0/results/configmaps--v1/search-benthos-streams.yaml index ad9e5db172..a092b1fac0 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-orchestration-v0.2.0/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-orchestration-v0.2.0/results/configmaps--v1/search-benthos-streams.yaml @@ -1,7 +1,7 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" - ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value.string()\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": metadata.key,\n \"value\": metadata.value\n },\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(params.key)\",\n \"params\": {\n \"key\": this.payload.key\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_all_v1.yaml: | diff --git a/components/operator/internal/controllers/stack/testdata/multipod-orchestration-v0.2.0/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-orchestration-v0.2.0/results/deployments-apps-v1/search-benthos.yaml index e99fb354c9..4b1ae1dd3a 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-orchestration-v0.2.0/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-orchestration-v0.2.0/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: Zas1IqJbhfN4PIGns5xxVW6s2zu7A-qQ0uJgR82mbNQ= + stack.formance.cloud/volumes-hash: oAXxeJ9yZ42WF5KLqqbY5iIu425cXeLlIyAQnwTJV_4= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos @@ -103,7 +103,7 @@ spec: value: $(LEDGER_POSTGRES_NO_DATABASE_URI)/$(LEDGER_POSTGRES_DATABASE)?sslmode=disable - name: BROKER value: kafka - image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.0-es + image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.1-es imagePullPolicy: IfNotPresent name: search-benthos ports: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-service-annotation/results/configmaps--v1/search-benthos-resources.yaml b/components/operator/internal/controllers/stack/testdata/multipod-service-annotation/results/configmaps--v1/search-benthos-resources.yaml index 6ec4d941c0..b0f7e739ae 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-service-annotation/results/configmaps--v1/search-benthos-resources.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-service-annotation/results/configmaps--v1/search-benthos-resources.yaml @@ -42,6 +42,7 @@ data: doc: ${! if json().exists("document") { json("document") } else {""} } script: ${! if json().exists("script") { json("script") } else {""} } upsert: ${! if json().exists("upsert") { json("upsert") } else {""} } + params: ${! if json().exists("params") { json("params") } else {""} } kind: ConfigMap metadata: labels: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-service-annotation/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-service-annotation/results/configmaps--v1/search-benthos-streams.yaml index 76f2d33b1f..483887861d 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-service-annotation/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-service-annotation/results/configmaps--v1/search-benthos-streams.yaml @@ -1,7 +1,7 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" - ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value.string()\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": metadata.key,\n \"value\": metadata.value\n },\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(params.key)\",\n \"params\": {\n \"key\": this.payload.key\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_all_v1.yaml: | diff --git a/components/operator/internal/controllers/stack/testdata/multipod-service-annotation/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-service-annotation/results/deployments-apps-v1/search-benthos.yaml index 9b1a686951..fd1ce25f42 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-service-annotation/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-service-annotation/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: EzO6HC1Z4jk4Ld2Ua4vIfjNEaYrtFQPY3G06P14GkrU= + stack.formance.cloud/volumes-hash: SKMuqJo00TeiIbsAJrChX-gmffYXjf1omWjMmImsS4k= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos @@ -84,7 +84,7 @@ spec: value: $(LEDGER_POSTGRES_NO_DATABASE_URI)/$(LEDGER_POSTGRES_DATABASE)?sslmode=disable - name: BROKER value: kafka - image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.0-es + image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.1-es imagePullPolicy: IfNotPresent name: search-benthos ports: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log-unvalid-tag/results/configmaps--v1/search-benthos-resources.yaml b/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log-unvalid-tag/results/configmaps--v1/search-benthos-resources.yaml index 1ee3661c6e..a1ac6927c9 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log-unvalid-tag/results/configmaps--v1/search-benthos-resources.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log-unvalid-tag/results/configmaps--v1/search-benthos-resources.yaml @@ -42,6 +42,7 @@ data: doc: ${! if json().exists("document") { json("document") } else {""} } script: ${! if json().exists("script") { json("script") } else {""} } upsert: ${! if json().exists("upsert") { json("upsert") } else {""} } + params: ${! if json().exists("params") { json("params") } else {""} } kind: ConfigMap metadata: labels: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log-unvalid-tag/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log-unvalid-tag/results/configmaps--v1/search-benthos-streams.yaml index 91bdd5e8ea..7e0b705f47 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log-unvalid-tag/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log-unvalid-tag/results/configmaps--v1/search-benthos-streams.yaml @@ -1,7 +1,7 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" - ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value.string()\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": metadata.key,\n \"value\": metadata.value\n },\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(params.key)\",\n \"params\": {\n \"key\": this.payload.key\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_all_v1.yaml: | diff --git a/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log-unvalid-tag/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log-unvalid-tag/results/deployments-apps-v1/search-benthos.yaml index 108b11c9c2..ab6da607be 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log-unvalid-tag/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log-unvalid-tag/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: N51Q5ybfoJgUQ5aJzKW0j9antWOEQn2YykvrD29h3no= + stack.formance.cloud/volumes-hash: NHqcEkmmUN10HDFaBGIFMaiPZrBqtcjAqSYG2G1r6Ek= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos @@ -85,7 +85,7 @@ spec: value: $(LEDGER_POSTGRES_NO_DATABASE_URI)/$(LEDGER_POSTGRES_DATABASE)?sslmode=disable - name: BROKER value: kafka - image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.0-es + image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.1-es imagePullPolicy: IfNotPresent name: search-benthos ports: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log/results/configmaps--v1/search-benthos-resources.yaml b/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log/results/configmaps--v1/search-benthos-resources.yaml index 51301b97e6..e8f9eb9455 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log/results/configmaps--v1/search-benthos-resources.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log/results/configmaps--v1/search-benthos-resources.yaml @@ -42,6 +42,7 @@ data: doc: ${! if json().exists("document") { json("document") } else {""} } script: ${! if json().exists("script") { json("script") } else {""} } upsert: ${! if json().exists("upsert") { json("upsert") } else {""} } + params: ${! if json().exists("params") { json("params") } else {""} } kind: ConfigMap metadata: labels: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log/results/configmaps--v1/search-benthos-streams.yaml index 01e166ba7e..220d7f75e1 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log/results/configmaps--v1/search-benthos-streams.yaml @@ -1,7 +1,7 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" - ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value.string()\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": metadata.key,\n \"value\": metadata.value\n },\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(params.key)\",\n \"params\": {\n \"key\": this.payload.key\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_all_v1.yaml: | diff --git a/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log/results/deployments-apps-v1/search-benthos.yaml index a75ccb2172..7bd89a12e1 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-with-audit-log/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: N51Q5ybfoJgUQ5aJzKW0j9antWOEQn2YykvrD29h3no= + stack.formance.cloud/volumes-hash: NHqcEkmmUN10HDFaBGIFMaiPZrBqtcjAqSYG2G1r6Ek= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos @@ -85,7 +85,7 @@ spec: value: $(LEDGER_POSTGRES_NO_DATABASE_URI)/$(LEDGER_POSTGRES_DATABASE)?sslmode=disable - name: BROKER value: kafka - image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.0-es + image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.1-es imagePullPolicy: IfNotPresent name: search-benthos ports: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-with-new-versions-per-module-gateway/results/configmaps--v1/search-benthos-resources.yaml b/components/operator/internal/controllers/stack/testdata/multipod-with-new-versions-per-module-gateway/results/configmaps--v1/search-benthos-resources.yaml index 8200ceebdc..1e3a168d32 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-with-new-versions-per-module-gateway/results/configmaps--v1/search-benthos-resources.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-with-new-versions-per-module-gateway/results/configmaps--v1/search-benthos-resources.yaml @@ -42,6 +42,7 @@ data: doc: ${! if json().exists("document") { json("document") } else {""} } script: ${! if json().exists("script") { json("script") } else {""} } upsert: ${! if json().exists("upsert") { json("upsert") } else {""} } + params: ${! if json().exists("params") { json("params") } else {""} } kind: ConfigMap metadata: labels: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-with-new-versions-per-module-gateway/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-with-new-versions-per-module-gateway/results/configmaps--v1/search-benthos-streams.yaml index fa206d4090..d6b4001941 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-with-new-versions-per-module-gateway/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-with-new-versions-per-module-gateway/results/configmaps--v1/search-benthos-streams.yaml @@ -1,7 +1,7 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" - ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value.string()\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": metadata.key,\n \"value\": metadata.value\n },\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(params.key)\",\n \"params\": {\n \"key\": this.payload.key\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_all_v1.yaml: | diff --git a/components/operator/internal/controllers/stack/testdata/multipod-with-new-versions-per-module-gateway/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-with-new-versions-per-module-gateway/results/deployments-apps-v1/search-benthos.yaml index 6f9a9bfbb4..ddaf813709 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-with-new-versions-per-module-gateway/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-with-new-versions-per-module-gateway/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: EzO6HC1Z4jk4Ld2Ua4vIfjNEaYrtFQPY3G06P14GkrU= + stack.formance.cloud/volumes-hash: SKMuqJo00TeiIbsAJrChX-gmffYXjf1omWjMmImsS4k= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos @@ -84,7 +84,7 @@ spec: value: $(LEDGER_POSTGRES_NO_DATABASE_URI)/$(LEDGER_POSTGRES_DATABASE)?sslmode=disable - name: BROKER value: kafka - image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.0-es + image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.1-es imagePullPolicy: IfNotPresent name: search-benthos ports: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-without-audit-log/results/configmaps--v1/search-benthos-resources.yaml b/components/operator/internal/controllers/stack/testdata/multipod-without-audit-log/results/configmaps--v1/search-benthos-resources.yaml index 58cc2778f2..8cbbac33ff 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-without-audit-log/results/configmaps--v1/search-benthos-resources.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-without-audit-log/results/configmaps--v1/search-benthos-resources.yaml @@ -42,6 +42,7 @@ data: doc: ${! if json().exists("document") { json("document") } else {""} } script: ${! if json().exists("script") { json("script") } else {""} } upsert: ${! if json().exists("upsert") { json("upsert") } else {""} } + params: ${! if json().exists("params") { json("params") } else {""} } kind: ConfigMap metadata: labels: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-without-audit-log/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-without-audit-log/results/configmaps--v1/search-benthos-streams.yaml index 9cc26077e0..12d0c49309 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-without-audit-log/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-without-audit-log/results/configmaps--v1/search-benthos-streams.yaml @@ -1,7 +1,7 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" - ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value.string()\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": metadata.key,\n \"value\": metadata.value\n },\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(params.key)\",\n \"params\": {\n \"key\": this.payload.key\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_all_v1.yaml: | diff --git a/components/operator/internal/controllers/stack/testdata/multipod-without-audit-log/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-without-audit-log/results/deployments-apps-v1/search-benthos.yaml index b9141de9c9..cd0fd3f944 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-without-audit-log/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-without-audit-log/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: EzO6HC1Z4jk4Ld2Ua4vIfjNEaYrtFQPY3G06P14GkrU= + stack.formance.cloud/volumes-hash: SKMuqJo00TeiIbsAJrChX-gmffYXjf1omWjMmImsS4k= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos @@ -84,7 +84,7 @@ spec: value: $(LEDGER_POSTGRES_NO_DATABASE_URI)/$(LEDGER_POSTGRES_DATABASE)?sslmode=disable - name: BROKER value: kafka - image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.0-es + image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.1-es imagePullPolicy: IfNotPresent name: search-benthos ports: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-without-new-versions-per-module-gateway/results/configmaps--v1/search-benthos-resources.yaml b/components/operator/internal/controllers/stack/testdata/multipod-without-new-versions-per-module-gateway/results/configmaps--v1/search-benthos-resources.yaml index 780e50a1ab..6ab64b7661 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-without-new-versions-per-module-gateway/results/configmaps--v1/search-benthos-resources.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-without-new-versions-per-module-gateway/results/configmaps--v1/search-benthos-resources.yaml @@ -42,6 +42,7 @@ data: doc: ${! if json().exists("document") { json("document") } else {""} } script: ${! if json().exists("script") { json("script") } else {""} } upsert: ${! if json().exists("upsert") { json("upsert") } else {""} } + params: ${! if json().exists("params") { json("params") } else {""} } kind: ConfigMap metadata: labels: diff --git a/components/operator/internal/controllers/stack/testdata/multipod-without-new-versions-per-module-gateway/results/configmaps--v1/search-benthos-streams.yaml b/components/operator/internal/controllers/stack/testdata/multipod-without-new-versions-per-module-gateway/results/configmaps--v1/search-benthos-streams.yaml index a090836f24..471f5bffb3 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-without-new-versions-per-module-gateway/results/configmaps--v1/search-benthos-streams.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-without-new-versions-per-module-gateway/results/configmaps--v1/search-benthos-streams.yaml @@ -1,7 +1,7 @@ apiVersion: v1 data: - ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value.string() + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value.string() + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" - ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[\\\"\" + metadata.key + \"\\\"]=\\\"\" + metadata.value + \"\\\"\",\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['\" + item.key + \"': '\" + item.value + \"'] } ctx._source.data.metadata['\" + item.key + \"']='\" + item.value + \"'\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(\\\"\" + this.payload.key + \"\\\")\",\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v1.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search\n\npipeline:\n processors:\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v1\n processors:\n - bloblang: |\n map account {\n root = this.map_each(v -> v.value.map_each(v2 -> {\n \"action\": \"upsert\",\n \"id\": v.key,\n \"document\": {\n \"data\": {\n \"address\": v.key\n },\n \"indexed\": {\n \"address\": v.key\n },\n \"kind\": \"ACCOUNT\"\n }\n }).values()).values().flatten()\n }\n\n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"%s\".format(this.txid),\n \"document\": {\n \"data\": {\n \"postings\": this.postings,\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"metadata\": if this.metadata { this.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.reference,\n \"txid\": this.txid,\n \"timestamp\": this.timestamp,\n \"asset\": this.postings.map_each(p -> p.asset),\n \"source\": this.postings.map_each(p -> p.source),\n \"destination\": this.postings.map_each(p -> p.destination),\n \"amount\": this.postings.map_each(p -> if p.asset.contains(\"/\") {\n [\n p.amount,\n p.amount / if p.asset.split(\"/\").index(1).number(){ range(0, p.asset.split(\"/\").index(1).number()).fold(1, t -> t.tally * 10) } else { 1 } # amount / pow(10, decimal part of asset)\n ]\n } else { [ p.amount ] }).flatten().map_each(v -> \"%v\".format(v))\n },\n \"kind\": \"TRANSACTION\"\n }\n }\n }\n\n map committedTransactions {\n root = [\n this.payload.transactions.map_each(t -> t.apply(\"tx\")).map_each(t -> t.assign({\n \"id\": \"TRANSACTION-%s-%s\".format(this.payload.ledger, t.id)\n })),\n this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.source),\n \"document\": {\n \"data\": {\n \"address\": p.source,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.source\n },\n \"kind\": \"ACCOUNT\"\n }\n }, {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, p.destination),\n \"document\": {\n \"data\": {\n \"address\": p.destination,\n \"metadata\": {}\n },\n \"indexed\": {\n \"address\": p.destination\n },\n \"kind\": \"ACCOUNT\"\n }\n }])).flatten().flatten()\n ].flatten().map_each(t -> t.merge({\n \"document\": {\n \"when\": this.date,\n \"ledger\": this.payload.ledger,\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger\n }\n },\n }))\n }\n\n root = this.apply(\"committedTransactions\")\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v1\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value.string()\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": {\n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value.string() },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n\noutput:\n resource: elasticsearch\n" + ledger_ingestion_v2.yaml: "input:\n event_bus:\n topic: ledger\n consumer_group: search-ledgerv2\n\npipeline:\n processors:\n - log:\n message: \"receive message: ${! this }\"\n - switch_event_type:\n events:\n - label: COMMITTED_TRANSACTIONS\n version: v2\n processors:\n - bloblang: |\n map amount {\n root = [this.amount]\n let hasDecimals = this.asset.split(\"/\").length() > 1\n let decimals = if $hasDecimals { this.asset.split(\"/\").index(1).number() } else { 0 }\n root = if $decimals > 0 {\n root.append(\n this.amount / range(0, $decimals).fold(1, t -> t.tally * 10) # Just a pow...\n )\n }\n root = root.flatten()\n }\n \n map tx {\n root = {\n \"action\": \"index\",\n \"id\": \"TRANSACTION-%s-%s\".format(this.ledger, this.transaction.id),\n \"document\": {\n \"data\": {\n \"postings\": this.transaction.postings,\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"metadata\": if this.transaction.metadata { this.transaction.metadata } else {{}}\n },\n \"indexed\": {\n \"reference\": this.transaction.reference,\n \"txid\": this.transaction.id,\n \"timestamp\": this.transaction.timestamp,\n \"asset\": this.transaction.postings.map_each(p -> p.asset),\n \"source\": this.transaction.postings.map_each(p -> p.source),\n \"destination\": this.transaction.postings.map_each(p -> p.destination),\n \"amount\": this.transaction.postings.map_each(p -> p.apply(\"amount\"))\n },\n \"kind\": \"TRANSACTION\",\n \"when\": this.date\n }\n }\n }\n \n map account {\n root = {\n \"action\": \"upsert\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.ledger, this.account),\n \"document\": {\n \"data\": { \n \"address\": this.account,\n \"metadata\": {}\n },\n \"indexed\": { \n \"address\": this.account\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }\n }\n \n root = []\n root = root.append(\n this.payload.transactions.map_each(transaction -> {\n \"transaction\": transaction,\n \"ledger\": this.payload.ledger\n }.apply(\"tx\"))\n )\n root = root.append(\n this.payload.transactions.\n map_each(transaction -> transaction.postings.map_each(posting -> [\n posting.source,\n posting.destination\n ]).\n flatten().\n map_each(account -> {\n \"account\": account,\n \"ledger\": this.payload.ledger\n }.apply(\"account\"))\n ).\n flatten()\n )\n root = root.append(\n this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> {\n \"script\": \"ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": metadata.key,\n \"value\": metadata.value\n },\n \"action\": \"update\",\n \"id\": \"ACCOUNT-%s-%s\".format(this.payload.ledger, item.key),\n \"upsert\": {\n \"data\": { \n \"address\": item.key,\n \"metadata\": { metadata.key: metadata.value }\n },\n \"indexed\": { \n \"address\": item.key\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()).values().flatten()\n )\n root = root.flatten()\n \n let overlay = {\n \"data\": {\n \"ledger\": this.payload.ledger\n },\n \"indexed\": {\n \"ledger\": this.payload.ledger \n }\n }\n \n root = root.map_each(cmd -> match cmd.action {\n cmd.action == \"update\" => if cmd.exists(\"upsert\") { cmd.merge({\n \"upsert\": $overlay\n }) } else { cmd },\n _ => cmd.merge({\n \"document\": $overlay\n }) \n })\n - log:\n message: \"Computed: ${! this }\"\n - unarchive:\n format: json_array\n - label: SAVED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = this.payload.metadata.map_each(item -> {\n \"script\": \"if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value\",\n \"params\": {\n \"key\": item.key,\n \"value\": item.value\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId),\n \"upsert\": {\n \"data\": { \n \"address\": this.payload.targetId,\n \"metadata\": { item.key: item.value },\n \"ledger\": this.payload.ledger\n },\n \"indexed\": { \n \"address\": this.payload.targetId,\n \"ledger\": this.payload.ledger\n },\n \"kind\": \"ACCOUNT\",\n \"when\": this.date\n }\n }).values()\n - unarchive:\n format: json_array\n - label: DELETED_METADATA\n version: v2\n processors:\n - bloblang: |\n root = {\n \"script\": \"ctx._source.data.metadata.remove(params.key)\",\n \"params\": {\n \"key\": this.payload.key\n },\n \"action\": \"update\",\n \"id\": \"%s-%s-%s\".format(this.payload.targetType, this.payload.ledger, this.payload.targetId)\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v1.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n meta batchSize = 100\n - postgres_query:\n service: ledger\n query: 'select count(*) as accounts_count from \"${! meta(\"ledger\") }\".accounts'\n - unarchive:\n format: json_array\n - bloblang: |\n meta loopCount = (this.accounts_count.number() / meta(\"batchSize\").number()).ceil()\n meta loopIndex = 0\n - bloblang: |\n root = if meta(\"loopCount\") == \"0\" {\n deleted() \n }\n - while:\n check: 'meta(\"loopIndex\") < meta(\"loopCount\")'\n processors:\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n offset ${! meta(\"loopIndex\").number() * meta(\"batchSize\").number() }\n limit ${! meta(\"batchSize\") }\n - bloblang:\n meta loopIndex = meta(\"loopIndex\").number() + 1\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\")\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_accounts_v2.yaml: "input:\n http_server:\n path: /\n\npipeline:\n processors:\n - bloblang: |\n meta ledger = this.ledger\n - postgres_query:\n service: ledger\n query: |\n select address, metadata\n from \"${! meta(\"ledger\") }\".accounts\n - unarchive:\n format: json_array\n - bloblang: |\n root = this.assign({\n \"metadata\": this.metadata.parse_json()\n })\n - bloblang: |\n root = {\n \"document\": {\n \"data\": { \n \"address\": this.address,\n \"ledger\": meta(\"ledger\"),\n \"metadata\": this.metadata\n },\n \"indexed\": {\n \"address\": this.address,\n \"ledger\": meta(\"ledger\")\n },\n \"kind\": \"ACCOUNT\",\n \"ledger\": meta(\"ledger\"),\n \"when\": this.date\n },\n \"id\": \"ACCOUNT-%s-%s\".format(meta(\"ledger\"), this.address),\n \"action\": \"upsert\"\n }\n\noutput:\n resource: elasticsearch\n" ledger_reindex_all_v1.yaml: | diff --git a/components/operator/internal/controllers/stack/testdata/multipod-without-new-versions-per-module-gateway/results/deployments-apps-v1/search-benthos.yaml b/components/operator/internal/controllers/stack/testdata/multipod-without-new-versions-per-module-gateway/results/deployments-apps-v1/search-benthos.yaml index 450ed22824..8dac823fc4 100644 --- a/components/operator/internal/controllers/stack/testdata/multipod-without-new-versions-per-module-gateway/results/deployments-apps-v1/search-benthos.yaml +++ b/components/operator/internal/controllers/stack/testdata/multipod-without-new-versions-per-module-gateway/results/deployments-apps-v1/search-benthos.yaml @@ -20,7 +20,7 @@ spec: template: metadata: annotations: - stack.formance.cloud/volumes-hash: EzO6HC1Z4jk4Ld2Ua4vIfjNEaYrtFQPY3G06P14GkrU= + stack.formance.cloud/volumes-hash: SKMuqJo00TeiIbsAJrChX-gmffYXjf1omWjMmImsS4k= creationTimestamp: null labels: app.kubernetes.io/name: search-benthos @@ -84,7 +84,7 @@ spec: value: $(LEDGER_POSTGRES_NO_DATABASE_URI)/$(LEDGER_POSTGRES_DATABASE)?sslmode=disable - name: BROKER value: kafka - image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.0-es + image: public.ecr.aws/formance-internal/jeffail/benthos:v4.23.1-es imagePullPolicy: IfNotPresent name: search-benthos ports: diff --git a/components/operator/internal/modules/search/handler.go b/components/operator/internal/modules/search/handler.go index e92074a707..51b6ece880 100644 --- a/components/operator/internal/modules/search/handler.go +++ b/components/operator/internal/modules/search/handler.go @@ -19,7 +19,7 @@ import ( ) const ( - benthosImage = "public.ecr.aws/formance-internal/jeffail/benthos:v4.23.0-es" + benthosImage = "public.ecr.aws/formance-internal/jeffail/benthos:v4.23.1-es" ) type module struct{} diff --git a/ee/search/benthos/resources/output_elasticsearch.yaml b/ee/search/benthos/resources/output_elasticsearch.yaml index 9b3007cddb..6c760ee364 100644 --- a/ee/search/benthos/resources/output_elasticsearch.yaml +++ b/ee/search/benthos/resources/output_elasticsearch.yaml @@ -38,4 +38,5 @@ output_resources: multipart: true doc: ${! if json().exists("document") { json("document") } else {""} } script: ${! if json().exists("script") { json("script") } else {""} } - upsert: ${! if json().exists("upsert") { json("upsert") } else {""} } \ No newline at end of file + upsert: ${! if json().exists("upsert") { json("upsert") } else {""} } + params: ${! if json().exists("params") { json("params") } else {""} } \ No newline at end of file diff --git a/ee/search/benthos/streams/ledger/v1.0.0/ledger_ingestion_v1.yaml b/ee/search/benthos/streams/ledger/v1.0.0/ledger_ingestion_v1.yaml index 7136fc918c..ba2dd20a8a 100644 --- a/ee/search/benthos/streams/ledger/v1.0.0/ledger_ingestion_v1.yaml +++ b/ee/search/benthos/streams/ledger/v1.0.0/ledger_ingestion_v1.yaml @@ -112,7 +112,11 @@ pipeline: processors: - bloblang: | root = this.payload.metadata.map_each(item -> { - "script": "if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['" + item.key + "': '" + item.value.string() + "'] } ctx._source.data.metadata['" + item.key + "']='" + item.value.string() + "'", + "script": "if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value", + "params": { + "key": item.key, + "value": item.value.string() + }, "action": "update", "id": "%s-%s-%s".format(this.payload.targetType, this.payload.ledger, this.payload.targetId), "upsert": { diff --git a/ee/search/benthos/streams/ledger/v2.0.0/ledger_ingestion_v2.yaml b/ee/search/benthos/streams/ledger/v2.0.0/ledger_ingestion_v2.yaml index 66b8e07891..debec86131 100644 --- a/ee/search/benthos/streams/ledger/v2.0.0/ledger_ingestion_v2.yaml +++ b/ee/search/benthos/streams/ledger/v2.0.0/ledger_ingestion_v2.yaml @@ -93,7 +93,11 @@ pipeline: ) root = root.append( this.payload.accountMetadata.map_each(item -> item.value.map_each(metadata -> { - "script": "ctx._source.data.metadata[\"" + metadata.key + "\"]=\"" + metadata.value + "\"", + "script": "ctx._source.data.metadata[params.key]=params.value", + "params": { + "key": metadata.key, + "value": metadata.value + }, "action": "update", "id": "ACCOUNT-%s-%s".format(this.payload.ledger, item.key), "upsert": { @@ -137,7 +141,11 @@ pipeline: processors: - bloblang: | root = this.payload.metadata.map_each(item -> { - "script": "if (ctx._source.data.metadata == null) { ctx._source.data.metadata = ['" + item.key + "': '" + item.value + "'] } ctx._source.data.metadata['" + item.key + "']='" + item.value + "'", + "script": "if (ctx._source.data.metadata == null) { ctx._source.data.metadata = [params.key: params.value] } ctx._source.data.metadata[params.key]=params.value", + "params": { + "key": item.key, + "value": item.value + }, "action": "update", "id": "%s-%s-%s".format(this.payload.targetType, this.payload.ledger, this.payload.targetId), "upsert": { @@ -161,7 +169,10 @@ pipeline: processors: - bloblang: | root = { - "script": "ctx._source.data.metadata.remove(\"" + this.payload.key + "\")", + "script": "ctx._source.data.metadata.remove(params.key)", + "params": { + "key": this.payload.key + }, "action": "update", "id": "%s-%s-%s".format(this.payload.targetType, this.payload.ledger, this.payload.targetId) } diff --git a/tests/integration/internal/modules/search.go b/tests/integration/internal/modules/search.go index f5991f8c15..d1b54b9584 100644 --- a/tests/integration/internal/modules/search.go +++ b/tests/integration/internal/modules/search.go @@ -26,7 +26,7 @@ var Search = internal.NewModule("search"). //"--mapping-init-disabled", } }), - internal.NewDockerService("public.ecr.aws/formance-internal/jeffail/benthos", "v4.23.0-es"). + internal.NewDockerService("public.ecr.aws/formance-internal/jeffail/benthos", "v4.23.1-es"). WithEntrypoint([]string{ "/benthos", "-c", "/config/config.yml",