Skip to content

Commit

Permalink
Allow AVX-512 instructions when jdks are bundled with distributions (#…
Browse files Browse the repository at this point in the history 
…1517)

Allow AVX-512 instructions when jdks are bundled with distributions
  • Loading branch information
@carterkozak
carterkozak authored Sep 5, 2023
1 parent 6eff589 commit e82d0e5
Show file tree
Hide file tree
Showing 4 changed files with 168 additions and 50 deletions.
5 changes: 5 additions & 0 deletions changelog/@unreleased/pr-1517.v2.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
type: improvement
improvement:
description: Allow AVX-512 instructions when jdks are bundled with distributions
links:
- https://github.com/palantir/sls-packaging/pull/1517
6 changes: 6 additions & 0 deletions ...ckaging/src/main/java/com/palantir/gradle/dist/service/JavaServiceDistributionPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,12 @@ public void execute(Task _task) {
task.getAddJava8GcLogging().set(distributionExtension.getAddJava8GcLogging());
task.getJavaHome().set(distributionExtension.getJavaHome());
task.getJavaVersion().set(distributionExtension.getJavaVersion());
task.getBundledJdks()
.set(distributionExtension
.getJdks()
.map(javaVersionObjectMap -> javaVersionObjectMap.containsKey(distributionExtension
.getJavaVersion()
.get())));
task.getEnv().set(userConfiguredEnvWithJdkEnvVars(distributionExtension));
});

Expand Down
27 changes: 23 additions & 4 deletions gradle-sls-packaging/src/main/java/com/palantir/gradle/dist/service/tasks/LaunchConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,17 +78,18 @@ public final class LaunchConfig {
// AWS-managed systems that modify DNS records on failover.
"-Dsun.net.inetaddr.ttl=20",
"-XX:+UnlockDiagnosticVMOptions",
// Disable AVX-512 intrinsics due to AES/CTR corruption bug in https://bugs.openjdk.org/browse/JDK-8292158
"-XX:+IgnoreUnrecognizedVMOptions",
// UseAVX is not recognized on some platforms (arm), so we must include 'IgnoreUnrecognizedVMOptions'.
// When a system supports UseAVX=N, setting UseAVX=N+1 will set the flag to the highest supported value.
"-XX:UseAVX=2",
"-XX:NativeMemoryTracking=summary",
// Increase default JFR stack depth beyond the default (conservative) 64 frames.
// This can be overridden by user-provided options.
// See sls-packaging#1230
"-XX:FlightRecorderOptions=stackdepth=256");

// Disable AVX-512 intrinsics due to AES/CTR corruption bug in https://bugs.openjdk.org/browse/JDK-8292158
// UseAVX is not recognized on some platforms (arm), so we must include 'IgnoreUnrecognizedVMOptions' above.
// When a system supports UseAVX=N, setting UseAVX=N+1 will set the flag to the highest supported value.
private static final ImmutableList<String> disableAvx512 = ImmutableList.of("-XX:UseAVX=2");

// Reduce memory usage for some versions of glibc.
// Default value is 8 * CORES.
// See https://issues.apache.org/jira/browse/HADOOP-7154
Expand All @@ -114,6 +115,9 @@ interface Params {
@Input
Property<JavaVersion> getJavaVersion();

@Input
Property<Boolean> getBundledJdks();

@Input
ListProperty<String> getArgs();

Expand Down Expand Up @@ -150,6 +154,7 @@ interface Params {

static void action(Params params) {
JavaVersion javaVersion = params.getJavaVersion().get();
List<String> avxOptions = getAvxOptions(params);

writeConfig(
LaunchConfigInfo.builder()
Expand All @@ -160,6 +165,7 @@ static void action(Params params) {
.classpath(relativizeToServiceLibDirectory(params.getClasspath()))
.addAllJvmOpts(javaAgentArgs(params))
.addAllJvmOpts(alwaysOnJvmOptions)
.addAllJvmOpts(avxOptions)
.addAllJvmOpts(params.getAddJava8GcLogging().get() ? java8gcLoggingOptions : ImmutableList.of())
// Java 11.0.16 introduced a potential memory leak issues when using the C2
// compiler
Expand Down Expand Up @@ -208,12 +214,25 @@ static void action(Params params) {
.classpath(relativizeToServiceLibDirectory(params.getClasspath()))
.addAllJvmOpts(javaAgentArgs(params))
.addAllJvmOpts(alwaysOnJvmOptions)
.addAllJvmOpts(avxOptions)
.addAllJvmOpts(params.getDefaultJvmOpts().get())
.env(defaultEnvironment)
.build(),
params.getCheckLauncher().get().getAsFile());
}

// When a specific jdk is provided, we can assume a modern versions including the
// bugfix for JDK-8292158. Only Java versions 11-19 were impacted by this bug, so
// we don't need to worry about newer releases.
private static List<String> getAvxOptions(Params params) {
JavaVersion javaVersion = params.getJavaVersion().get();
return !params.getBundledJdks().get()
&& javaVersion.compareTo(JavaVersion.toVersion("11")) >= 0
&& javaVersion.compareTo(JavaVersion.toVersion("19")) <= 0
? disableAvx512
: ImmutableList.of();
}

private static void writeConfig(LaunchConfigInfo config, File scriptFile) {
try {
Files.createDirectories(scriptFile.getParentFile().toPath());
Expand Down
180 changes: 134 additions & 46 deletions ...rc/test/groovy/com/palantir/gradle/dist/service/JavaServiceDistributionPluginTests.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -378,6 +378,7 @@ class JavaServiceDistributionPluginTests extends GradleIntegrationSpec {
dependencies { implementation files("${EXTERNAL_JAR}") }
tasks.jar.archiveBaseName = "internal"
distribution {
javaVersion 11
javaHome 'foo'
args 'myArg1', 'myArg2'
checkArgs 'myCheckArg1', 'myCheckArg2'
Expand All @@ -391,59 +392,147 @@ class JavaServiceDistributionPluginTests extends GradleIntegrationSpec {

then:
def expectedStaticConfig = LaunchConfig.LaunchConfigInfo.builder()
.mainClass("test.Test")
.serviceName("service-name")
.javaHome("foo")
.args(["myArg1", "myArg2"])
.classpath(['service/lib/internal-0.0.1.jar', 'service/lib/external.jar'])
.jvmOpts([
'-XX:+CrashOnOutOfMemoryError',
'-Djava.io.tmpdir=var/data/tmp',
'-XX:ErrorFile=var/log/hs_err_pid%p.log',
'-XX:HeapDumpPath=var/log',
'-Dsun.net.inetaddr.ttl=20',
'-XX:+UnlockDiagnosticVMOptions',
'-XX:+IgnoreUnrecognizedVMOptions',
'-XX:UseAVX=2',
'-XX:NativeMemoryTracking=summary',
'-XX:FlightRecorderOptions=stackdepth=256',
'-XX:+UseParallelGC',
'-Xmx4M',
'-Djavax.net.ssl.trustStore=truststore.jks'])
.env(LaunchConfig.defaultEnvironment + [
"key1": "val1",
"key2": "val2"])
.dirs(["var/data/tmp"])
.build()
.mainClass("test.Test")
.serviceName("service-name")
.javaHome("foo")
.args(["myArg1", "myArg2"])
.classpath(['service/lib/internal-0.0.1.jar', 'service/lib/external.jar'])
.jvmOpts([
'-XX:+CrashOnOutOfMemoryError',
'-Djava.io.tmpdir=var/data/tmp',
'-XX:ErrorFile=var/log/hs_err_pid%p.log',
'-XX:HeapDumpPath=var/log',
'-Dsun.net.inetaddr.ttl=20',
'-XX:+UnlockDiagnosticVMOptions',
'-XX:+IgnoreUnrecognizedVMOptions',
'-XX:NativeMemoryTracking=summary',
'-XX:FlightRecorderOptions=stackdepth=256',
'-XX:UseAVX=2',
'-XX:CompileCommand=exclude,sun/security/ssl/SSLEngineInputRecord.decodeInputRecord',
'-XX:-UseBiasedLocking',
'-XX:+IgnoreUnrecognizedVMOptions',
'-XX:+UseContainerCpuShares',
'-XX:+UseParallelGC',
'-Xmx4M',
'-Djavax.net.ssl.trustStore=truststore.jks'])
.env(LaunchConfig.defaultEnvironment + [
"key1": "val1",
"key2": "val2"])
.dirs(["var/data/tmp"])
.build()
def actualStaticConfig = OBJECT_MAPPER.readValue(
file('dist/service-name-0.0.1/service/bin/launcher-static.yml'), LaunchConfig.LaunchConfigInfo)

def expectedCheckConfig = LaunchConfig.LaunchConfigInfo.builder()
.mainClass(actualStaticConfig.mainClass())
.serviceName(actualStaticConfig.serviceName())
.javaHome(actualStaticConfig.javaHome())
.args(["myCheckArg1", "myCheckArg2"])
.classpath(actualStaticConfig.classpath())
.jvmOpts([
'-XX:+CrashOnOutOfMemoryError',
'-Djava.io.tmpdir=var/data/tmp',
'-XX:ErrorFile=var/log/hs_err_pid%p.log',
'-XX:HeapDumpPath=var/log',
'-Dsun.net.inetaddr.ttl=20',
'-XX:+UnlockDiagnosticVMOptions',
'-XX:+IgnoreUnrecognizedVMOptions',
'-XX:UseAVX=2',
'-XX:NativeMemoryTracking=summary',
'-XX:FlightRecorderOptions=stackdepth=256',
'-Xmx4M',
'-Djavax.net.ssl.trustStore=truststore.jks'])
.env(LaunchConfig.defaultEnvironment)
.dirs(actualStaticConfig.dirs())
.build()
.mainClass(actualStaticConfig.mainClass())
.serviceName(actualStaticConfig.serviceName())
.javaHome(actualStaticConfig.javaHome())
.args(["myCheckArg1", "myCheckArg2"])
.classpath(actualStaticConfig.classpath())
.jvmOpts([
'-XX:+CrashOnOutOfMemoryError',
'-Djava.io.tmpdir=var/data/tmp',
'-XX:ErrorFile=var/log/hs_err_pid%p.log',
'-XX:HeapDumpPath=var/log',
'-Dsun.net.inetaddr.ttl=20',
'-XX:+UnlockDiagnosticVMOptions',
'-XX:+IgnoreUnrecognizedVMOptions',
'-XX:NativeMemoryTracking=summary',
'-XX:FlightRecorderOptions=stackdepth=256',
'-XX:UseAVX=2',
'-Xmx4M',
'-Djavax.net.ssl.trustStore=truststore.jks'])
.env(LaunchConfig.defaultEnvironment)
.dirs(actualStaticConfig.dirs())
.build()

def actualCheckConfig = OBJECT_MAPPER.readValue(
file('dist/service-name-0.0.1/service/bin/launcher-check.yml'), LaunchConfig.LaunchConfigInfo)
expectedCheckConfig == actualCheckConfig
expectedStaticConfig == actualStaticConfig
}


def 'produce distribution bundle that populates launcher-static.yml and launcher-check.yml with bundled jdk'() {
given:
createUntarBuildFile(buildFile)

buildFile << """
dependencies { implementation files("${EXTERNAL_JAR}") }
tasks.jar.archiveBaseName = "internal"
distribution {
javaVersion 11
jdks.put(JavaVersion.VERSION_11, fileTree('build/fake-jdk'))
javaHome 'foo'
args 'myArg1', 'myArg2'
checkArgs 'myCheckArg1', 'myCheckArg2'
env "key1": "val1",
"key2": "val2"
}""".stripIndent()
file('src/main/java/test/Test.java') << "package test;\npublic class Test {}"

when:
runTasks(':build', ':distTar', ':untar')

then:
def expectedStaticConfig = LaunchConfig.LaunchConfigInfo.builder()
.mainClass("test.Test")
.serviceName("service-name")
.javaHome("foo")
.args(["myArg1", "myArg2"])
.classpath(['service/lib/internal-0.0.1.jar', 'service/lib/external.jar'])
.jvmOpts([
'-XX:+CrashOnOutOfMemoryError',
'-Djava.io.tmpdir=var/data/tmp',
'-XX:ErrorFile=var/log/hs_err_pid%p.log',
'-XX:HeapDumpPath=var/log',
'-Dsun.net.inetaddr.ttl=20',
'-XX:+UnlockDiagnosticVMOptions',
'-XX:+IgnoreUnrecognizedVMOptions',
'-XX:NativeMemoryTracking=summary',
'-XX:FlightRecorderOptions=stackdepth=256',
'-XX:CompileCommand=exclude,sun/security/ssl/SSLEngineInputRecord.decodeInputRecord',
'-XX:-UseBiasedLocking',
'-XX:+IgnoreUnrecognizedVMOptions',
'-XX:+UseContainerCpuShares',
'-XX:+UseParallelGC',
'-Xmx4M',
'-Djavax.net.ssl.trustStore=truststore.jks'])
.env(LaunchConfig.defaultEnvironment + [
"key1": "val1",
"key2": "val2",
"JAVA_11_HOME": "service/service-name-jdks/jdk11"])
.dirs(["var/data/tmp"])
.build()
def actualStaticConfig = OBJECT_MAPPER.readValue(
file('dist/service-name-0.0.1/service/bin/launcher-static.yml'), LaunchConfig.LaunchConfigInfo)

def expectedCheckConfig = LaunchConfig.LaunchConfigInfo.builder()
.mainClass(actualStaticConfig.mainClass())
.serviceName(actualStaticConfig.serviceName())
.javaHome(actualStaticConfig.javaHome())
.args(["myCheckArg1", "myCheckArg2"])
.classpath(actualStaticConfig.classpath())
.jvmOpts([
'-XX:+CrashOnOutOfMemoryError',
'-Djava.io.tmpdir=var/data/tmp',
'-XX:ErrorFile=var/log/hs_err_pid%p.log',
'-XX:HeapDumpPath=var/log',
'-Dsun.net.inetaddr.ttl=20',
'-XX:+UnlockDiagnosticVMOptions',
'-XX:+IgnoreUnrecognizedVMOptions',
'-XX:NativeMemoryTracking=summary',
'-XX:FlightRecorderOptions=stackdepth=256',
'-Xmx4M',
'-Djavax.net.ssl.trustStore=truststore.jks'])
.env(LaunchConfig.defaultEnvironment)
.dirs(actualStaticConfig.dirs())
.build()

def actualCheckConfig = OBJECT_MAPPER.readValue(
file('dist/service-name-0.0.1/service/bin/launcher-check.yml'), LaunchConfig.LaunchConfigInfo)
expectedCheckConfig == actualCheckConfig
expectedStaticConfig == actualStaticConfig
}

def 'produce distribution with java 8 gc logging'() {
Expand Down Expand Up @@ -474,7 +563,6 @@ class JavaServiceDistributionPluginTests extends GradleIntegrationSpec {
'-Dsun.net.inetaddr.ttl=20',
'-XX:+UnlockDiagnosticVMOptions',
'-XX:+IgnoreUnrecognizedVMOptions',
'-XX:UseAVX=2',
'-XX:NativeMemoryTracking=summary',
'-XX:FlightRecorderOptions=stackdepth=256',
"-XX:+PrintGCDateStamps",
Expand Down

0 comments on commit e82d0e5

Please sign in to comment.