From 4d4c6dd93cdd08e665dc650239fe8c80ba1a838f Mon Sep 17 00:00:00 2001 From: area363 Date: Sun, 21 Apr 2024 20:03:16 -0400 Subject: [PATCH 1/4] add PLUGIN_PATH env to avoid downloading the same plugin file each restart --- .../Hosting/LibplanetNodeService.cs | 74 +++++++++++++++---- 1 file changed, 58 insertions(+), 16 deletions(-) diff --git a/Libplanet.Headless/Hosting/LibplanetNodeService.cs b/Libplanet.Headless/Hosting/LibplanetNodeService.cs index 4e0f063ce..a0bc99b9a 100644 --- a/Libplanet.Headless/Hosting/LibplanetNodeService.cs +++ b/Libplanet.Headless/Hosting/LibplanetNodeService.cs @@ -5,6 +5,8 @@ using System.IO.Compression; using System.Linq; using System.Net.Http; +using System.Security.Cryptography; +using System.Text; using System.Threading; using System.Threading.Tasks; using Bencodex; @@ -635,29 +637,53 @@ public override void Dispose() Log.Debug("Store disposed."); } - private string ResolvePluginPath(string path) => - Uri.IsWellFormedUriString(path, UriKind.Absolute) - ? DownloadPlugin(path).Result - : path; + private string ResolvePluginPath(string path) + { + if (Uri.IsWellFormedUriString(path, UriKind.Absolute)) + { + // Run the download on a background thread + return Task.Run(() => DownloadPlugin(path, SwarmCancellationToken)).GetAwaiter().GetResult(); + } - private async Task DownloadPlugin(string url) + return path; + } + + private async Task DownloadPlugin(string url, CancellationToken cancellationToken) { - var path = Path.Combine(Environment.CurrentDirectory, "plugins"); + var basePath = Environment.GetEnvironmentVariable("PLUGIN_PATH") ?? Environment.CurrentDirectory; + var path = Path.Combine(basePath, "plugins"); Directory.CreateDirectory(path); - var hashed = url.GetHashCode().ToString(); + var hashed = CreateSha256Hash(url); var logger = Log.ForContext("LibplanetNodeService", hashed); using var httpClient = new HttpClient(); var downloadPath = Path.Join(path, hashed + ".zip"); var extractPath = Path.Join(path, hashed); - logger.Debug("Downloading..."); - await File.WriteAllBytesAsync( - downloadPath, - await httpClient.GetByteArrayAsync(url, SwarmCancellationToken), - SwarmCancellationToken); - logger.Debug("Finished downloading."); - logger.Debug("Extracting..."); - ZipFile.ExtractToDirectory(downloadPath, extractPath); - logger.Debug("Finished extracting."); + logger.Debug("Download Path: {downloadPath}", downloadPath); + logger.Debug("Extract Path: {extractPath}", extractPath); + + if (!File.Exists(downloadPath)) + { + logger.Debug("Downloading..."); + var content = await httpClient.GetByteArrayAsync(url, cancellationToken); + await File.WriteAllBytesAsync(downloadPath, content, cancellationToken); + logger.Debug("Finished downloading."); + } + else + { + logger.Debug("Download skipped, file already exists."); + } + + if (!Directory.Exists(extractPath)) + { + logger.Debug("Extracting..."); + ZipFile.ExtractToDirectory(downloadPath, extractPath); + logger.Debug("Finished extracting."); + } + else + { + logger.Debug("Extraction skipped, folder already exists."); + } + return Path.Combine(extractPath, "Lib9c.Plugin.dll"); } @@ -671,5 +697,21 @@ public ActionTypeLoaderContext(long index) public long Index { get; } } + + private static string CreateSha256Hash(string input) + { + using SHA256 sha256Hash = SHA256.Create(); + + // ComputeHash - returns byte array + byte[] bytes = sha256Hash.ComputeHash(Encoding.UTF8.GetBytes(input)); + + // Convert byte array to a string + StringBuilder builder = new StringBuilder(); + foreach (var t in bytes) + { + builder.Append(t.ToString("x2")); + } + return builder.ToString(); + } } } From 2e06152c012211aa8a60d44da0d5169b1f42e34c Mon Sep 17 00:00:00 2001 From: area363 Date: Sun, 21 Apr 2024 23:41:27 -0400 Subject: [PATCH 2/4] add checksum method --- .../Hosting/LibplanetNodeService.cs | 43 ++++++++++++++++--- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/Libplanet.Headless/Hosting/LibplanetNodeService.cs b/Libplanet.Headless/Hosting/LibplanetNodeService.cs index a0bc99b9a..486513ee6 100644 --- a/Libplanet.Headless/Hosting/LibplanetNodeService.cs +++ b/Libplanet.Headless/Hosting/LibplanetNodeService.cs @@ -642,13 +642,13 @@ private string ResolvePluginPath(string path) if (Uri.IsWellFormedUriString(path, UriKind.Absolute)) { // Run the download on a background thread - return Task.Run(() => DownloadPlugin(path, SwarmCancellationToken)).GetAwaiter().GetResult(); + return Task.Run(() => DownloadPlugin(path)).GetAwaiter().GetResult(); } return path; } - private async Task DownloadPlugin(string url, CancellationToken cancellationToken) + private async Task DownloadPlugin(string url) { var basePath = Environment.GetEnvironmentVariable("PLUGIN_PATH") ?? Environment.CurrentDirectory; var path = Path.Combine(basePath, "plugins"); @@ -658,14 +658,16 @@ private async Task DownloadPlugin(string url, CancellationToken cancella using var httpClient = new HttpClient(); var downloadPath = Path.Join(path, hashed + ".zip"); var extractPath = Path.Join(path, hashed); + var checksumPath = Path.Join(extractPath, "checksum.txt"); + logger.Debug("Download Path: {downloadPath}", downloadPath); logger.Debug("Extract Path: {extractPath}", extractPath); if (!File.Exists(downloadPath)) { logger.Debug("Downloading..."); - var content = await httpClient.GetByteArrayAsync(url, cancellationToken); - await File.WriteAllBytesAsync(downloadPath, content, cancellationToken); + var content = await httpClient.GetByteArrayAsync(url); + await File.WriteAllBytesAsync(downloadPath, content); logger.Debug("Finished downloading."); } else @@ -673,20 +675,47 @@ private async Task DownloadPlugin(string url, CancellationToken cancella logger.Debug("Download skipped, file already exists."); } - if (!Directory.Exists(extractPath)) + if (!Directory.Exists(extractPath) || (File.Exists(checksumPath) && CalculateDirectoryChecksum(extractPath) != await File.ReadAllTextAsync(checksumPath))) { + Directory.CreateDirectory(extractPath); logger.Debug("Extracting..."); - ZipFile.ExtractToDirectory(downloadPath, extractPath); + ZipFile.ExtractToDirectory(downloadPath, extractPath, true); logger.Debug("Finished extracting."); + + // Calculate checksum of extracted files and save it + var checksum = CalculateDirectoryChecksum(extractPath); + await File.WriteAllTextAsync(checksumPath, checksum); } else { - logger.Debug("Extraction skipped, folder already exists."); + logger.Debug("Extraction skipped, folder already exists and is verified."); } return Path.Combine(extractPath, "Lib9c.Plugin.dll"); } + private string CalculateDirectoryChecksum(string directoryPath) + { + using var md5 = MD5.Create(); + var files = Directory.GetFiles(directoryPath, "*", SearchOption.AllDirectories) + .OrderBy(p => p).ToArray(); + + foreach (var file in files) + { + // hash path + var pathBytes = Encoding.UTF8.GetBytes(file.Substring(directoryPath.Length)); + md5.TransformBlock(pathBytes, 0, pathBytes.Length, pathBytes, 0); + + // hash contents + var contentBytes = File.ReadAllBytes(file); + md5.TransformBlock(contentBytes, 0, contentBytes.Length, contentBytes, 0); + } + + // Handles empty content. + md5.TransformFinalBlock(Array.Empty(), 0, 0); + return BitConverter.ToString(md5.Hash!).Replace("-", "").ToLowerInvariant(); + } + // FIXME: Request libplanet provide default implementation. private sealed class ActionTypeLoaderContext : IActionTypeLoaderContext { From 7932a8d973bc5eb6dc745ec322a7b43079d1b715 Mon Sep 17 00:00:00 2001 From: hyeon Date: Tue, 23 Apr 2024 18:01:34 +0900 Subject: [PATCH 3/4] Bump lib9c --- Lib9c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib9c b/Lib9c index e1f447941..13023b0d8 160000 --- a/Lib9c +++ b/Lib9c @@ -1 +1 @@ -Subproject commit e1f447941bd8d5bdb30384617c4fe1dd9edcd49a +Subproject commit 13023b0d8cd6b9d049ce15c93683dcf5d604e1e4 From 5ab3e34ddc15b86363405e686889a330447bc987 Mon Sep 17 00:00:00 2001 From: hyeon Date: Wed, 24 Apr 2024 16:44:24 +0900 Subject: [PATCH 4/4] Bump lib9c --- Lib9c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib9c b/Lib9c index 13023b0d8..7c90f7d82 160000 --- a/Lib9c +++ b/Lib9c @@ -1 +1 @@ -Subproject commit 13023b0d8cd6b9d049ce15c93683dcf5d604e1e4 +Subproject commit 7c90f7d8243b1fc80a2e770abd0891a7a4f00bed