From 3f9039ed165c8014bb465f74d4638c3222d9ab01 Mon Sep 17 00:00:00 2001 From: Sebastian Erives Date: Mon, 4 Nov 2024 21:25:02 -0600 Subject: [PATCH 1/4] Optimize plugin class loader for caching and resource reusage --- .../plugin/loader/PluginClassLoader.kt | 139 ++++++++++++------ .../eocvsim/plugin/loader/PluginLoader.kt | 4 - .../superaccess/SuperAccessDaemonClient.kt | 58 ++------ 3 files changed, 110 insertions(+), 91 deletions(-) diff --git a/EOCV-Sim/src/main/java/io/github/deltacv/eocvsim/plugin/loader/PluginClassLoader.kt b/EOCV-Sim/src/main/java/io/github/deltacv/eocvsim/plugin/loader/PluginClassLoader.kt index 5aae9f2..9119b6a 100644 --- a/EOCV-Sim/src/main/java/io/github/deltacv/eocvsim/plugin/loader/PluginClassLoader.kt +++ b/EOCV-Sim/src/main/java/io/github/deltacv/eocvsim/plugin/loader/PluginClassLoader.kt @@ -53,6 +53,11 @@ class PluginClassLoader( private var additionalZipFiles = mutableListOf>() + private var zipFiles = mutableMapOf() + + private var classpathCacheLock = Any() + private var classpathCache = mutableMapOf() + private val zipFile = try { ZipFile(pluginJar) } catch (e: Exception) { @@ -129,13 +134,13 @@ class PluginClassLoader( clazz = Class.forName(name) } - } catch(e: Throwable) { + } catch (e: Throwable) { clazz = try { loadClassStrict(name) - } catch(_: Throwable) { + } catch (_: Throwable) { val classpathClass = classFromClasspath(name) - if(classpathClass != null) { + if (classpathClass != null) { classpathClass } else { throw e @@ -182,48 +187,62 @@ class PluginClassLoader( * Get a resource from the classpath specified in the constructor */ fun resourceAsStreamFromClasspath(name: String): InputStream? { - for (file in classpath) { - if (file == pluginJar) continue - - val zipFile = ZipFile(file) - - val entry = zipFile.getEntry(name) - - if (entry != null) { - try { - additionalZipFiles.add(WeakReference(zipFile)) - return zipFile.getInputStream(entry) - } catch (_: Exception) { - zipFile.close() - } - } else { - zipFile.close() - } - } - - return null + return resourceFromClasspath(name)?.openStream() } /** * Get a resource from the classpath specified in the constructor */ fun resourceFromClasspath(name: String): URL? { - for (file in classpath) { - if (file == pluginJar) continue - - val zipFile = ZipFile(file) + fun resFromJarFile(file: File): URL? { + if (file == pluginJar) return null try { + val zipFile = zipFiles[file] ?: ZipFile(file) + zipFiles[file] = zipFile + val entry = zipFile.getEntry(name) + if (entry == null) return null - if (entry != null) { - try { - return URL("jar:file:${file.absolutePath}!/$name") - } catch (_: Exception) { - } + val packages = name.split("/") + + val firstPackage = if (packages.size > 3) + packages[0] + "." + packages[1] + "." + packages[3] + else if (packages.size > 2) + packages[0] + "." + packages[1] + else name + + synchronized(classpathCacheLock) { + classpathCache[firstPackage] = file } - } finally { - zipFile.close() + + try { + return URL("jar:file:${file.absolutePath}!/$name") + } catch (_: Exception) { + } + } catch (_: Exception) { + } + + return null + } + + val cache = synchronized(classpathCacheLock) { + classpathCache.filterKeys { name.contains(it.replace("/", ".")) } + } + + for ((cacheName, file) in cache) { + if (name.contains(cacheName)) { + val res = resFromJarFile(file) + if (res != null) { + return res + } + } + } + + for (file in classpath) { + val res = resFromJarFile(file) + if (res != null) { + return res } } @@ -234,30 +253,64 @@ class PluginClassLoader( * Load a class from the classpath specified in the constructor */ fun classFromClasspath(className: String): Class<*>? { - for (file in classpath) { - if (file == pluginJar) continue - - val zipFile = ZipFile(file) + fun classFromJarFile(file: File): Class<*>? { + if (file == pluginJar) return null try { + val zipFile = zipFiles[file] ?: ZipFile(file) + zipFiles[file] = zipFile + val entry = zipFile.getEntry(className.replace('.', '/') + ".class") + if (entry == null) return null + + val packages = className.split(".") + + val firstPackage = if (packages.size > 3) + packages[0] + "." + packages[1] + "." + packages[3] + else if (packages.size > 2) + packages[0] + "." + packages[1] + else className - if (entry != null) { - return loadClass(entry, zipFile = zipFile) + synchronized(classpathCacheLock) { + classpathCache[firstPackage] = file + } + + return loadClass(entry, zipFile = zipFile) + } catch (_: Exception) { + } + + return null + } + + val cache = synchronized(classpathCacheLock) { + classpathCache.filterKeys { className.contains(it) } + } + + for ((name, file) in cache) { + if (className.contains(name)) { + val clazz = classFromJarFile(file) + if (clazz != null) { + return clazz } - } finally { - zipFile.close() } } + for (file in classpath) { + val clazz = classFromJarFile(file) + if (clazz != null) return clazz + } + return null } fun close() { zipFile.close() - for(ref in additionalZipFiles) { + for (ref in additionalZipFiles) { ref.get()?.close() } + for ((_, zipFile) in zipFiles) { + zipFile.close() + } } override fun toString() = "PluginClassLoader@\"${pluginJar.name}\"" diff --git a/EOCV-Sim/src/main/java/io/github/deltacv/eocvsim/plugin/loader/PluginLoader.kt b/EOCV-Sim/src/main/java/io/github/deltacv/eocvsim/plugin/loader/PluginLoader.kt index 2ae9631..0fc6672 100644 --- a/EOCV-Sim/src/main/java/io/github/deltacv/eocvsim/plugin/loader/PluginLoader.kt +++ b/EOCV-Sim/src/main/java/io/github/deltacv/eocvsim/plugin/loader/PluginLoader.kt @@ -27,21 +27,17 @@ import com.github.serivesmejia.eocvsim.EOCVSim import com.github.serivesmejia.eocvsim.config.ConfigLoader import com.github.serivesmejia.eocvsim.gui.dialog.AppendDelegate import com.github.serivesmejia.eocvsim.gui.dialog.PluginOutput -import com.github.serivesmejia.eocvsim.util.SysUtil import com.github.serivesmejia.eocvsim.util.event.EventHandler -import com.github.serivesmejia.eocvsim.util.extension.fileHash import com.github.serivesmejia.eocvsim.util.extension.hashString import com.github.serivesmejia.eocvsim.util.extension.plus import com.github.serivesmejia.eocvsim.util.loggerForThis import com.moandjiezana.toml.Toml import io.github.deltacv.common.util.ParsedVersion import io.github.deltacv.eocvsim.plugin.EOCVSimPlugin -import io.github.deltacv.eocvsim.plugin.security.PluginSignature import io.github.deltacv.eocvsim.plugin.security.PluginSignatureVerifier import io.github.deltacv.eocvsim.sandbox.nio.SandboxFileSystem import net.lingala.zip4j.ZipFile import java.io.File -import java.security.MessageDigest enum class PluginSource { REPOSITORY, diff --git a/EOCV-Sim/src/main/java/io/github/deltacv/eocvsim/plugin/security/superaccess/SuperAccessDaemonClient.kt b/EOCV-Sim/src/main/java/io/github/deltacv/eocvsim/plugin/security/superaccess/SuperAccessDaemonClient.kt index 8eefdb8..e71a6e4 100644 --- a/EOCV-Sim/src/main/java/io/github/deltacv/eocvsim/plugin/security/superaccess/SuperAccessDaemonClient.kt +++ b/EOCV-Sim/src/main/java/io/github/deltacv/eocvsim/plugin/security/superaccess/SuperAccessDaemonClient.kt @@ -42,21 +42,23 @@ typealias ResponseReceiver = (SuperAccessDaemon.SuperAccessResponse) -> Unit typealias ResponseCondition = (SuperAccessDaemon.SuperAccessResponse) -> Boolean private data class AccessCache( - val head: String, val file: String, val hasAccess: Boolean, - val timestamp: Long, - val footer: String, - val count: Int, - val countIntegrity: String + val timestamp: Long ) { + init { + // get calling class + val stackTrace = Thread.currentThread().stackTrace + val callingClass = stackTrace[2].className - fun hash() = (head + file + hasAccess.toString() + timestamp.toString() + footer + count.toString() + countIntegrity).hashString - + if(callingClass != SuperAccessDaemonClient::class.java.name) { + throw IllegalAccessException("AccessCache should only be created from SuperAccessDaemonClient") + } + } } class SuperAccessDaemonClient( - val cacheTTLMillis: Long = 5000 // 5 seconds + val cacheTTLMillis: Long = 3_000 ) { val logger by loggerForThis() @@ -64,11 +66,8 @@ class SuperAccessDaemonClient( private val startLock = ReentrantLock() val startCondition = startLock.newCondition() - private var cacheCount = Random().nextInt(Int.MAX_VALUE) - private var integrity = "" - private val cacheLock = Any() - private val accessCache = mutableMapOf>() + private val accessCache = mutableMapOf() // create a new WebSocket server private val server = WsServer(startLock, startCondition) @@ -103,28 +102,9 @@ class SuperAccessDaemonClient( fun checkAccess(file: File): Boolean { synchronized(cacheLock) { if (accessCache.containsKey(file.absolutePath)) { - val currentCaches = accessCache[file.absolutePath]!! - val currentCache = currentCaches.last() + val currentCache = accessCache[file.absolutePath]!! if (System.currentTimeMillis() - currentCache.timestamp < cacheTTLMillis) { - val sortedCache = mutableListOf() - - for(caches in accessCache.values) { - sortedCache.addAll(caches) - } - - sortedCache.sortBy { it.timestamp } - - var expectedIntegrity = "" - - for(cache in sortedCache) { - expectedIntegrity = (cache.hash() + expectedIntegrity).hashString - } - - if (expectedIntegrity != integrity) { - throw SecurityException("Access cache has been tampered with") - } - return currentCache.hasAccess } } @@ -159,18 +139,8 @@ class SuperAccessDaemonClient( } synchronized(cacheLock) { - val list = accessCache.getOrPut(file.absolutePath) { mutableListOf() } - - val head = list.lastOrNull()?.hash() ?: "" - - var countIntegrity = (cacheCount + (cacheCount - 1)).hashString - - val newCache = AccessCache(head, file.absolutePath, hasAccess, System.currentTimeMillis(), integrity, cacheCount, countIntegrity).apply { - integrity = (hash() + integrity).hashString - cacheCount++ - } - - list.add(newCache) + val newCache = AccessCache(file.absolutePath, hasAccess, System.currentTimeMillis()) + accessCache[file.absolutePath] = newCache } return hasAccess From 64e18f6d5c1a3d99322f99df859f7f9d36eb6f51 Mon Sep 17 00:00:00 2001 From: Sebastian Erives Date: Mon, 4 Nov 2024 22:06:17 -0600 Subject: [PATCH 2/4] Change version to 3.8.3 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index bc4f9e4..b989655 100644 --- a/build.gradle +++ b/build.gradle @@ -44,7 +44,7 @@ plugins { allprojects { group 'com.github.deltacv' - version '3.8.2' + version '3.8.3' apply plugin: 'java' From 5b03e128c26d3e3f429c11752989509109ef0f6e Mon Sep 17 00:00:00 2001 From: Sebastian Erives Date: Mon, 4 Nov 2024 23:44:26 -0600 Subject: [PATCH 3/4] Remove log file creation by superaccessdaemon --- .../security/superaccess/SuperAccessDaemon.kt | 8 +++++-- .../superaccess/SuperAccessDaemonClient.kt | 3 ++- EOCV-Sim/src/main/resources/log4j2.xml | 6 +++++ EOCV-Sim/src/main/resources/log4j2_nofile.xml | 24 +++++++++++++++++++ 4 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 EOCV-Sim/src/main/resources/log4j2_nofile.xml diff --git a/EOCV-Sim/src/main/java/io/github/deltacv/eocvsim/plugin/security/superaccess/SuperAccessDaemon.kt b/EOCV-Sim/src/main/java/io/github/deltacv/eocvsim/plugin/security/superaccess/SuperAccessDaemon.kt index 19a45c4..3df7ab7 100644 --- a/EOCV-Sim/src/main/java/io/github/deltacv/eocvsim/plugin/security/superaccess/SuperAccessDaemon.kt +++ b/EOCV-Sim/src/main/java/io/github/deltacv/eocvsim/plugin/security/superaccess/SuperAccessDaemon.kt @@ -23,9 +23,8 @@ package io.github.deltacv.eocvsim.plugin.security.superaccess -import com.github.serivesmejia.eocvsim.util.extension.plus import com.github.serivesmejia.eocvsim.util.extension.fileHash -import com.github.serivesmejia.eocvsim.util.io.EOCVSimFolder +import com.github.serivesmejia.eocvsim.util.extension.plus import com.github.serivesmejia.eocvsim.util.loggerForThis import com.github.serivesmejia.eocvsim.util.serialization.PolymorphicAdapter import com.google.gson.GsonBuilder @@ -38,11 +37,16 @@ import io.github.deltacv.eocvsim.plugin.loader.PluginParser import io.github.deltacv.eocvsim.plugin.security.Authority import io.github.deltacv.eocvsim.plugin.security.AuthorityFetcher import io.github.deltacv.eocvsim.plugin.security.MutablePluginSignature +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.core.LoggerContext +import org.apache.logging.log4j.core.appender.FileAppender import org.java_websocket.client.WebSocketClient import org.java_websocket.handshake.ServerHandshake import java.io.File import java.lang.Exception import java.net.URI +import java.nio.file.Files +import java.nio.file.Paths import java.util.concurrent.Executors import java.util.zip.ZipFile import javax.swing.SwingUtilities diff --git a/EOCV-Sim/src/main/java/io/github/deltacv/eocvsim/plugin/security/superaccess/SuperAccessDaemonClient.kt b/EOCV-Sim/src/main/java/io/github/deltacv/eocvsim/plugin/security/superaccess/SuperAccessDaemonClient.kt index e71a6e4..c4c990a 100644 --- a/EOCV-Sim/src/main/java/io/github/deltacv/eocvsim/plugin/security/superaccess/SuperAccessDaemonClient.kt +++ b/EOCV-Sim/src/main/java/io/github/deltacv/eocvsim/plugin/security/superaccess/SuperAccessDaemonClient.kt @@ -221,7 +221,8 @@ class SuperAccessDaemonClient( val exitCode = JavaProcess.exec( SuperAccessDaemon::class.java, JavaProcess.SLF4JIOReceiver(logger), - null, listOf(port.toString()) + listOf("-Dlog4j.configurationFile=log4j2_nofile.xml"), + listOf(port.toString()) ) if(processRestarts == 6) { diff --git a/EOCV-Sim/src/main/resources/log4j2.xml b/EOCV-Sim/src/main/resources/log4j2.xml index 59ac6df..2c78ef5 100644 --- a/EOCV-Sim/src/main/resources/log4j2.xml +++ b/EOCV-Sim/src/main/resources/log4j2.xml @@ -20,6 +20,12 @@ + + + + + + diff --git a/EOCV-Sim/src/main/resources/log4j2_nofile.xml b/EOCV-Sim/src/main/resources/log4j2_nofile.xml new file mode 100644 index 0000000..e91ff1f --- /dev/null +++ b/EOCV-Sim/src/main/resources/log4j2_nofile.xml @@ -0,0 +1,24 @@ + + + + + [%d{HH:mm:ss}] [%t/%level]: [%c{1}] %msg%n + + + + + + + + + + + + + + + + + + + \ No newline at end of file From 04ba83ed5ea89a50f61480c4e066b4a3df0a0432 Mon Sep 17 00:00:00 2001 From: Sebastian Erives Date: Tue, 5 Nov 2024 00:14:31 -0600 Subject: [PATCH 4/4] Add changelog for 3.8.3 --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 5df0bec..d739d50 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,11 @@ Join the [deltacv discord server](https://discord.gg/A3RMYzf6DA) ! ### Formerly, EOCV-Sim was hosted on a [personal account repo](https://github.com/serivesmejia/EOCV-Sim/). Released prior to 3.0.0 can be found there for historic purposes. +## [v3.8.3 - Plugin Classloader major optimization](https://github.com/deltacv/EOCV-Sim/releases/tag/v3.8.3) +- This is the 28th release for EOCV-Sim + - Improves plugin classloader classpath loading by caching and resource reusage, this brings an extremely noticeable performance boost for plugins that load a lot of class files from different Maven dependencies in a short amount of time (I'm looking at you Javalin) + - Stops SuperAccessDaemon from creating a separate log file + ## [v3.8.2 - Synchronization bug fixes](https://github.com/deltacv/EOCV-Sim/releases/tag/v3.8.2) - This is the 27th release for EOCV-Sim - Internal changes: