From aabfc9b4f2e610c67edc52113bc2df7fcbda8461 Mon Sep 17 00:00:00 2001 From: Hyun Seungmin Date: Thu, 9 May 2024 10:06:06 +0900 Subject: [PATCH 1/6] configure root namespace of project --- Mimir.Worker/Mimir.Worker.csproj | 1 + Mimir/Mimir.csproj | 1 + 2 files changed, 2 insertions(+) diff --git a/Mimir.Worker/Mimir.Worker.csproj b/Mimir.Worker/Mimir.Worker.csproj index 4e0dc050..e9b2a755 100644 --- a/Mimir.Worker/Mimir.Worker.csproj +++ b/Mimir.Worker/Mimir.Worker.csproj @@ -1,6 +1,7 @@ + Mimir.Worker net8.0 enable enable diff --git a/Mimir/Mimir.csproj b/Mimir/Mimir.csproj index a50ed0e9..cb4688a6 100644 --- a/Mimir/Mimir.csproj +++ b/Mimir/Mimir.csproj @@ -1,6 +1,7 @@ + Mimir net8.0 enable enable From 087efdc5244ed5cf309caf37c72602fde8170296 Mon Sep 17 00:00:00 2001 From: Hyun Seungmin Date: Thu, 9 May 2024 10:40:13 +0900 Subject: [PATCH 2/6] fix warnings minor --- Mimir.Worker/BlockPoller.cs | 9 +++++---- Mimir.Worker/Initializer.cs | 7 +++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Mimir.Worker/BlockPoller.cs b/Mimir.Worker/BlockPoller.cs index 5303b34c..779b096f 100644 --- a/Mimir.Worker/BlockPoller.cs +++ b/Mimir.Worker/BlockPoller.cs @@ -25,12 +25,13 @@ public async Task RunAsync(CancellationToken cancellationToken) continue; } - var rawArenaTxsResp = await headlessGqlClient.GetBattleArenaTransactions.ExecuteAsync(processBlockIndex); - + var rawArenaTxsResp = await headlessGqlClient.GetBattleArenaTransactions.ExecuteAsync( + processBlockIndex, + cancellationToken); if (rawArenaTxsResp.Data is null) { - Serilog.Log.Error("Failed to get arena txs. errors:\n" + - string.Join("\n", rawArenaTxsResp.Errors.Select(x => "- " + x.Message))); + var errors = rawArenaTxsResp.Errors.Select(e => e.Message); + Serilog.Log.Error("Failed to get arena txs. response data is null. errors:\n{Errors}", errors); await mongoDbWorker.UpdateLatestBlockIndex(syncedBlockIndex + 1); continue; } diff --git a/Mimir.Worker/Initializer.cs b/Mimir.Worker/Initializer.cs index a3573972..a44b7797 100644 --- a/Mimir.Worker/Initializer.cs +++ b/Mimir.Worker/Initializer.cs @@ -29,11 +29,10 @@ public Initializer( protected override async Task ExecuteAsync(CancellationToken stoppingToken) { var started = DateTime.UtcNow; - await _scrapper.ExecuteAsync(stoppingToken); - - var totalElapsedMinutes = DateTime.UtcNow.Subtract(started).Minutes; - _logger.LogInformation($"Finished Initializer background service. Elapsed {totalElapsedMinutes} minutes."); + _logger.LogInformation( + "Finished Initializer background service. Elapsed {TotalElapsedMinutes} minutes", + DateTime.UtcNow.Subtract(started).Minutes); var poller = new BlockPoller(_stateService, _headlessGqlClient, _worker); await poller.RunAsync(stoppingToken); From 05d315cde22ab4796f6f8ab3594ffb39fcf09f5b Mon Sep 17 00:00:00 2001 From: Hyun Seungmin Date: Thu, 9 May 2024 19:03:24 +0900 Subject: [PATCH 3/6] add GetTransactions query --- Mimir.Worker/Queries.graphql | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Mimir.Worker/Queries.graphql b/Mimir.Worker/Queries.graphql index f07394ac..5b6d8a44 100644 --- a/Mimir.Worker/Queries.graphql +++ b/Mimir.Worker/Queries.graphql @@ -17,3 +17,16 @@ query GetBattleArenaTransactions($blockIndex: Long!) { } } } + +query GetTransactions { + chainQuery { + blockQuery { + blocks(desc: true, limit: 1) { + index + transactions { + signer + } + } + } + } +} From 53fa53af0dad35e774f5b70a00003ae60c330949 Mon Sep 17 00:00:00 2001 From: Hyun Seungmin Date: Thu, 9 May 2024 19:04:49 +0900 Subject: [PATCH 4/6] save all AvatarData from every transactions in the block --- Mimir.Worker/BlockPoller.cs | 134 ++++++++++++++++++++++++++---------- 1 file changed, 96 insertions(+), 38 deletions(-) diff --git a/Mimir.Worker/BlockPoller.cs b/Mimir.Worker/BlockPoller.cs index 779b096f..632cdfbc 100644 --- a/Mimir.Worker/BlockPoller.cs +++ b/Mimir.Worker/BlockPoller.cs @@ -5,10 +5,12 @@ using Mimir.Worker.Models; using Mimir.Worker.Scrapper; using Mimir.Worker.Services; +using Nekoyume; +using StrawberryShake; namespace Mimir.Worker; -public class BlockPoller(IStateService stateService, HeadlessGQLClient headlessGqlClient, MongoDbWorker mongoDbWorker) +public class BlockPoller(IStateService stateService, IHeadlessGQLClient headlessGqlClient, MongoDbWorker mongoDbWorker) { public async Task RunAsync(CancellationToken cancellationToken) { @@ -18,53 +20,109 @@ public async Task RunAsync(CancellationToken cancellationToken) var syncedBlockIndex = await mongoDbWorker.GetLatestBlockIndex(); var currentBlockIndex = await stateService.GetLatestIndex(); var processBlockIndex = syncedBlockIndex + 1; - if (processBlockIndex >= currentBlockIndex) { await Task.Delay(TimeSpan.FromMilliseconds(3000), cancellationToken); continue; } - var rawArenaTxsResp = await headlessGqlClient.GetBattleArenaTransactions.ExecuteAsync( - processBlockIndex, - cancellationToken); - if (rawArenaTxsResp.Data is null) - { - var errors = rawArenaTxsResp.Errors.Select(e => e.Message); - Serilog.Log.Error("Failed to get arena txs. response data is null. errors:\n{Errors}", errors); - await mongoDbWorker.UpdateLatestBlockIndex(syncedBlockIndex + 1); - continue; - } + await AllInventoryAsync(processBlockIndex, stateGetter, cancellationToken); + await BattleArenaAsync(processBlockIndex, stateGetter, cancellationToken); + await mongoDbWorker.UpdateLatestBlockIndex(processBlockIndex); + } + } - try - { - var arenaTxs = rawArenaTxsResp.Data.Transaction - .NcTransactions!.Select(raw => - TxMarshaler.DeserializeTransactionWithoutVerification( - Convert.FromBase64String(raw.SerializedPayload))) - .ToList(); + private async Task AllInventoryAsync( + long processBlockIndex, + StateGetter stateGetter, + CancellationToken cancellationToken) + { + var operationResult = await headlessGqlClient.GetTransactions.ExecuteAsync(cancellationToken); + if (operationResult.Data is null) + { + HandleErrors(operationResult); + return; + } - foreach (var arenaTx in arenaTxs) - { - var arenaAction = (Dictionary)arenaTx.Actions.First(); - var arenaActionValues = (Dictionary)arenaAction["values"]; - var enemyAvatarAddress = new Address(arenaActionValues["eaa"]); - var myAvatarAddress = new Address(arenaActionValues["maa"]); + if (operationResult.Data.ChainQuery.BlockQuery is null || + operationResult.Data.ChainQuery.BlockQuery.Blocks.Count == 0) + { + return; + } - var roundData = await stateGetter.GetArenaRoundData(processBlockIndex); - var enemyAvatarData = await stateGetter.GetAvatarData(enemyAvatarAddress); - var myAvatarData = await stateGetter.GetAvatarData(myAvatarAddress); - var myArenaData = await stateGetter.GetArenaData(roundData, myAvatarAddress); - var enemyArenaData = await stateGetter.GetArenaData(roundData, enemyAvatarAddress); + var block = operationResult.Data.ChainQuery.BlockQuery.Blocks[0]; + if (block.Index < processBlockIndex) + { + return; + } - await mongoDbWorker.BulkUpsertAvatarDataAsync(new List { myAvatarData, enemyAvatarData }); - await mongoDbWorker.BulkUpsertArenaDataAsync(new List { myArenaData, enemyArenaData }); - } - } - finally - { - await mongoDbWorker.UpdateLatestBlockIndex(syncedBlockIndex + 1); - } + foreach (var transaction in block.Transactions) + { + var agentAddress = new Address(transaction.Signer); + var avatarAddresses = Enumerable.Range(0, GameConfig.SlotCount) + .Select(e => Addresses.GetAvatarAddress(agentAddress, e)); + var avatarDataArray = await Task.WhenAll(avatarAddresses.Select(stateGetter.GetAvatarData)); + await mongoDbWorker.BulkUpsertAvatarDataAsync( + avatarDataArray + .Where(e => e is not null) + .OfType() + .ToList()); + } + } + + private async Task BattleArenaAsync( + long processBlockIndex, + StateGetter stateGetter, + CancellationToken cancellationToken) + { + var rawArenaTxsResp = await headlessGqlClient.GetBattleArenaTransactions.ExecuteAsync( + processBlockIndex, + cancellationToken); + if (rawArenaTxsResp.Data is null) + { + HandleErrors(rawArenaTxsResp); + return; + } + + if (rawArenaTxsResp.Data.Transaction.NcTransactions is null) + { + return; } + + var arenaTxs = rawArenaTxsResp.Data.Transaction.NcTransactions + .Where(raw => raw is not null) + .Select(raw => TxMarshaler.DeserializeTransactionWithoutVerification( + Convert.FromBase64String(raw!.SerializedPayload))) + .ToList(); + foreach (var arenaTx in arenaTxs) + { + var arenaAction = (Dictionary)arenaTx.Actions[0]; + var arenaActionValues = (Dictionary)arenaAction["values"]; + var myAvatarAddress = new Address(arenaActionValues["maa"]); + var enemyAvatarAddress = new Address(arenaActionValues["eaa"]); + + var roundData = await stateGetter.GetArenaRoundData(processBlockIndex); + var myAvatarData = await stateGetter.GetAvatarData(myAvatarAddress); + var enemyAvatarData = await stateGetter.GetAvatarData(enemyAvatarAddress); + var myArenaData = await stateGetter.GetArenaData(roundData, myAvatarAddress); + var enemyArenaData = await stateGetter.GetArenaData(roundData, enemyAvatarAddress); + + await mongoDbWorker.BulkUpsertAvatarDataAsync( + new[] { myAvatarData, enemyAvatarData } + .Where(e => e is not null) + .OfType() + .ToList()); + await mongoDbWorker.BulkUpsertArenaDataAsync( + new[] { myArenaData, enemyArenaData } + .Where(e => e is not null) + .OfType() + .ToList()); + } + } + + private static void HandleErrors(IOperationResult operationResult) + { + var errors = operationResult.Errors.Select(e => e.Message); + Serilog.Log.Error("Failed to get txs. response data is null. errors:\n{Errors}", errors); } } From d64826061ba72c3a1f2bfa03448d11a45dae1f0a Mon Sep 17 00:00:00 2001 From: Hyun Seungmin Date: Fri, 10 May 2024 10:19:13 +0900 Subject: [PATCH 5/6] rename minor --- Mimir.Worker/BlockPoller.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Mimir.Worker/BlockPoller.cs b/Mimir.Worker/BlockPoller.cs index 632cdfbc..4329b6ea 100644 --- a/Mimir.Worker/BlockPoller.cs +++ b/Mimir.Worker/BlockPoller.cs @@ -26,13 +26,13 @@ public async Task RunAsync(CancellationToken cancellationToken) continue; } - await AllInventoryAsync(processBlockIndex, stateGetter, cancellationToken); + await EveryAvatarAsync(processBlockIndex, stateGetter, cancellationToken); await BattleArenaAsync(processBlockIndex, stateGetter, cancellationToken); await mongoDbWorker.UpdateLatestBlockIndex(processBlockIndex); } } - private async Task AllInventoryAsync( + private async Task EveryAvatarAsync( long processBlockIndex, StateGetter stateGetter, CancellationToken cancellationToken) From 5f69cd245d88aec3341f3d0aed14ef8b18d8edaa Mon Sep 17 00:00:00 2001 From: Hyun Seungmin Date: Fri, 10 May 2024 10:31:55 +0900 Subject: [PATCH 6/6] change query --- Mimir.Worker/BlockPoller.cs | 21 +++++++++++---------- Mimir.Worker/Queries.graphql | 13 ++++--------- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/Mimir.Worker/BlockPoller.cs b/Mimir.Worker/BlockPoller.cs index 4329b6ea..65dfa03e 100644 --- a/Mimir.Worker/BlockPoller.cs +++ b/Mimir.Worker/BlockPoller.cs @@ -37,28 +37,29 @@ private async Task EveryAvatarAsync( StateGetter stateGetter, CancellationToken cancellationToken) { - var operationResult = await headlessGqlClient.GetTransactions.ExecuteAsync(cancellationToken); + var operationResult = await headlessGqlClient.GetTransactionSigners.ExecuteAsync( + processBlockIndex, + cancellationToken); if (operationResult.Data is null) { HandleErrors(operationResult); return; } - if (operationResult.Data.ChainQuery.BlockQuery is null || - operationResult.Data.ChainQuery.BlockQuery.Blocks.Count == 0) + var txs = operationResult.Data.Transaction.NcTransactions; + if (txs is null || txs.Count == 0) { return; } - var block = operationResult.Data.ChainQuery.BlockQuery.Blocks[0]; - if (block.Index < processBlockIndex) + foreach (var tx in txs) { - return; - } + if (tx is null) + { + continue; + } - foreach (var transaction in block.Transactions) - { - var agentAddress = new Address(transaction.Signer); + var agentAddress = new Address(tx.Signer); var avatarAddresses = Enumerable.Range(0, GameConfig.SlotCount) .Select(e => Addresses.GetAvatarAddress(agentAddress, e)); var avatarDataArray = await Task.WhenAll(avatarAddresses.Select(stateGetter.GetAvatarData)); diff --git a/Mimir.Worker/Queries.graphql b/Mimir.Worker/Queries.graphql index 5b6d8a44..bdbfb73e 100644 --- a/Mimir.Worker/Queries.graphql +++ b/Mimir.Worker/Queries.graphql @@ -18,15 +18,10 @@ query GetBattleArenaTransactions($blockIndex: Long!) { } } -query GetTransactions { - chainQuery { - blockQuery { - blocks(desc: true, limit: 1) { - index - transactions { - signer - } - } +query GetTransactionSigners($blockIndex: Long!) { + transaction { + ncTransactions(startingBlockIndex: $blockIndex, limit: 1, actionType: "^[a-zA-Z0-9]*$") { + signer } } }