Skip to content

Commit

Permalink
2.3.11 - Snapshot/Volume capabilities, GroovyBuilderDeployment, TimeM…
Browse files Browse the repository at this point in the history 
…achine fixes etc (#31)

* GroovyContainer.groovy
 * Added createGroovyBuildContainer()
 * Added setUseLocalM2Cache(), installMaven(), installGit(), cloneGitRepo(), resolveMavenDependencies(), generateEffectivePom(), installMavenProject()

Container.groovy
 * Added prepareVolumeMount()
 * Improved createTar(), added ignorePaths parameter, now supports long file paths

GroovyBuilderDeployment.groovy WIP
 * New deployment, intended for building and optionally hosting groovy jars

DockerClientDS.groovy
 * added getVolumesWithLabel(), getVolumesWithName(), getContainersUsingVolume(), createVolume(), cloneVolume()

* Method for getting time from external, modified TravelToNow to allow external time

(cherry picked from commit fc28cdf)

* added constructor to TimeMachine

(cherry picked from commit aa467b0)

* shading kotlinx

(cherry picked from commit ed9002f)

* TimeMachine.groovy
 * Improved getExternalTime(), removed unused unirest field

TimeMachineSpec.groovy
 * WIP proved traveling forward in time works for a companion container

* Container.groovy
 * !NOTE! potentially breaking change: mounts field renamed preparedMounts
 * Added getMounts()

JsmContainer.groovy
 * Added cloneJiraHome, snapshotJiraHome, restoreJiraHomeSnapshot, getJiraHomeMountPoint

DockerClientDS.groovy
 * Added overwriteVolume

* JsmContainer.groovy
 * Added getSnapshotVolume()

TimeMachineSpec.groovy
 * Improved testing for JSM time travel

* JsmContainer.groovy
 * Solved bug that caused the DB to become corrupt during snapshot due to killing container forcefully.
Container.groovy
 * stopContainer(),
   * Added parameter for timeout and improved logging
JsmH2Deployment.groovy
 * The map appsToInstall can now have keys that are either URLs to apps or MarketplaceApp.Version
 * setupDeployment()
  * Added parameters to setupDeployment() to allow creation and reuse of snapshots

* pom.xml
 * Bumped to 2.3.11
 * Bumped jirainstancemanager to 2.0.3

---------

Co-authored-by: spatil <Serenad12#nacka>
  • Loading branch information
@farthinder
farthinder authored Oct 13, 2023
1 parent ed98b0b commit e2559f6
Show file tree
Hide file tree
Showing 23 changed files with 1,235 additions and 140 deletions.
4 changes: 4 additions & 0 deletions .github/buildScripts/shadingConf.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,10 @@
<pattern>okhttp3</pattern>
<shadedPattern>com.eficode.shaded.okhttp3</shadedPattern>
</relocation>
<relocation>
<pattern>kotlinx.coroutines</pattern>
<shadedPattern>com.eficode.shaded.kotlinx.coroutines</shadedPattern>
</relocation>
</relocations>

<!-- NOTE: Any dependencies of the project will not show up in the standalone pom.
Expand Down
4 changes: 2 additions & 2 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

<groupId>com.eficode</groupId>
<artifactId>devstack</artifactId>
<version>2.3.9-SNAPSHOT</version>
<version>2.3.11-SNAPSHOT</version>
<packaging>jar</packaging>

<name>DevStack</name>
Expand Down Expand Up @@ -100,7 +100,7 @@
<dependency>
<groupId>com.eficode.atlassian</groupId>
<artifactId>jirainstancemanager</artifactId>
<version>2.0.1-SNAPSHOT</version>
<version>2.0.3-SNAPSHOT</version>
</dependency>


Expand Down
141 changes: 105 additions & 36 deletions src/main/groovy/com/eficode/devstack/container/Container.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ import de.gesellix.docker.remote.api.ContainerSummary
import de.gesellix.docker.remote.api.EndpointSettings
import de.gesellix.docker.remote.api.ExecConfig
import de.gesellix.docker.remote.api.HostConfig
import de.gesellix.docker.remote.api.IdResponse
import de.gesellix.docker.remote.api.Mount
import de.gesellix.docker.remote.api.MountPoint
import de.gesellix.docker.remote.api.Network
import de.gesellix.docker.remote.api.NetworkCreateRequest
import de.gesellix.docker.remote.api.PortBinding
Expand Down Expand Up @@ -52,7 +52,7 @@ trait Container {
ArrayList<String> customEnvVar = []
String defaultShell = "/bin/bash"
String containerId
ArrayList<Mount> mounts = []
ArrayList<Mount> preparedMounts = [] //Mounts that will be added at creation


/**
Expand All @@ -70,11 +70,40 @@ trait Container {
m.type = Mount.Type.Bind
}

if (!self.mounts.find { it.source == sourceAbs && it.target == target }) {
self.mounts.add(newMount)
if (!self.preparedMounts.find { it.source == sourceAbs && it.target == target }) {
self.preparedMounts.add(newMount)
}
}

/**
* Prepare mounting of an existing or new volume
* @param volumeName The name of the volume to create, or an existing one to mount
* @param target Where to mount it in the container
* @param readOnly If it should be read only or not
*/
void prepareVolumeMount(String volumeName, String target, boolean readOnly = true) {

Mount newMount = new Mount().tap { m ->
m.source = volumeName
m.target = target
m.readOnly = readOnly
m.type = Mount.Type.Volume
}

if (!self.preparedMounts.find { it.source == volumeName && it.target == target }) {
self.preparedMounts.add(newMount)
}
}

/**
* Get MountPoints currently attached to container
* @return
*/
ArrayList<MountPoint> getMounts() {

ContainerInspectResponse response = inspectContainer()
return response.mounts
}

ContainerCreateRequest setupContainerCreateRequest() {

Expand All @@ -91,7 +120,7 @@ trait Container {
if (self.containerMainPort) {
h.portBindings = [(self.containerMainPort + "/tcp"): [new PortBinding("0.0.0.0", (self.containerMainPort))]]
}
h.mounts = self.mounts
h.mounts = self.preparedMounts
}
c.hostname = self.containerName
c.env = self.customEnvVar
Expand Down Expand Up @@ -124,7 +153,7 @@ trait Container {
ContainerCreateRequest containerCreateRequest = setupContainerCreateRequest()

if (cmd.size()) {
containerCreateRequest.cmd = cmd.collect {it.toString()}
containerCreateRequest.cmd = cmd.collect { it.toString() }
}

if (entrypoint.size()) {
Expand Down Expand Up @@ -201,8 +230,16 @@ trait Container {

}

/**
* Returnes the common short form of the container ID
* @return
*/
String getShortId() {
return getContainerId().substring(0,12)
}

String getId() {
return containerId
return getContainerId()
}

Container getSelf() {
Expand Down Expand Up @@ -329,29 +366,33 @@ trait Container {

}

boolean stopContainer() {
boolean stopContainer(Integer timeoutS = 15) {
log.info("Stopping container:" + self.containerId)
running ? dockerClient.stop(self.containerId, 15) : ""
long start = System.currentTimeSeconds()
running ? dockerClient.stop(self.containerId, timeoutS) : ""

if (running) {
log.warn("\tFailed to stop container" + self.containerId)
log.warn("Gave up waiting to shutdown ${shortId} after ${System.currentTimeSeconds() - start} seconds")
return false
} else {
log.info("\tContainer stopped")
log.debug("\t\tContainer ${shortId} stopped after ${System.currentTimeSeconds() - start} seconds")
return true
}
}


File createTar(ArrayList<String> filePaths, String outputPath) {
File createTar(ArrayList<String> filePaths, String outputPath, ArrayList<String> ignorePaths = []) {


log.info("Creating tar file:" + outputPath)
log.debug("\tUsing source paths:")
filePaths.each { log.debug("\t\t$it") }


File outputFile = new File(outputPath)
TarArchiveOutputStream tarArchive = new TarArchiveOutputStream(Files.newOutputStream(outputFile.toPath()))
tarArchive.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX)

log.info("\tProcessing files")
filePaths.each { filePath ->
Expand All @@ -368,28 +409,40 @@ trait Container {

String path = ResourceGroovyMethods.relativePath(newEntryFile, subFile)
log.trace("\t" * 4 + "Processing sub file:" + path)
TarArchiveEntry entry = new TarArchiveEntry(subFile, path)
entry.setSize(subFile.size())
tarArchive.putArchiveEntry(entry)
tarArchive.write(subFile.bytes)
tarArchive.closeArchiveEntry()
log.trace("\t" * 5 + "Added to archive")
if (ignorePaths.any { subFile.absolutePath.matches(it) }) {
log.trace("\t" * 5 + "File matches a path that is to be ignored, will not process further")
} else {
TarArchiveEntry entry = new TarArchiveEntry(subFile, path)
entry.setSize(subFile.size())
tarArchive.putArchiveEntry(entry)
tarArchive.write(subFile.bytes)
tarArchive.closeArchiveEntry()
log.trace("\t" * 5 + "Added to archive")
}


}
} else {
log.trace("\t" * 4 + "Processing file:" + newEntryFile.name)
TarArchiveEntry entry = new TarArchiveEntry(newEntryFile, newEntryFile.name)
entry.setSize(newEntryFile.size())
tarArchive.putArchiveEntry(entry)
tarArchive.write(newEntryFile.bytes)
tarArchive.closeArchiveEntry()
log.trace("\t" * 5 + "Added to archive")

if (ignorePaths.any { newEntryFile.absolutePath.matches(it) }) {
log.trace("\t" * 5 + "File matches a path that is to be ignored, will not process further")
}else {
TarArchiveEntry entry = new TarArchiveEntry(newEntryFile, newEntryFile.name)
entry.setSize(newEntryFile.size())
tarArchive.putArchiveEntry(entry)
tarArchive.write(newEntryFile.bytes)
tarArchive.closeArchiveEntry()
log.trace("\t" * 5 + "Added to archive")
}
}


}

tarArchive.finish()
log.info("\tFinished creating TAR file:" + outputFile.absolutePath)
log.debug("\t\t" + (outputFile.size() / (1024 * 1024)).round() + "MB")

return outputFile

Expand Down Expand Up @@ -429,6 +482,15 @@ trait Container {
}


/**
* Gets the home path for the containers default user
* @return ex: /home/user
*/
String getHomePath() {
runBashCommandInContainer("pwd").find {true}
}


/**
* Creates a network of the type bridge, or returns an existing one if one with the same name exists
* @param networkName name of the network
Expand Down Expand Up @@ -637,12 +699,12 @@ trait Container {
boolean replaceFileInContainer(String content, String filePath, boolean verify = false) {
ArrayList<String> out = runBashCommandInContainer("cat > $filePath <<- 'EOF'\n" + content + "\nEOF")

assert out.isEmpty() : "Unexpected output when replacing file $filePath: " + out.join("\n")
assert out.isEmpty(): "Unexpected output when replacing file $filePath: " + out.join("\n")

if (verify) {
ArrayList<String>rawOut = runBashCommandInContainer("cat " + filePath)
ArrayList<String> rawOut = runBashCommandInContainer("cat " + filePath)
String readOut = rawOut.join()
assert readOut.trim() == content.trim() : "Error when verifying that the file $filePath was replaced"
assert readOut.trim() == content.trim(): "Error when verifying that the file $filePath was replaced"
return true
}

Expand Down Expand Up @@ -672,10 +734,18 @@ trait Container {

}

boolean copyFileToContainer(String srcFilePath, String destinationDirectory) {
/**
* Creates a temporary tar, copies it to the container and extracts it there
* @param srcFilePath Local path to copy, will find directories/files recursively
* @param destinationDirectory The destination path in the container, must already exist and be absolut
* @param ignorePaths If these regex patterns matches the path/name of a file it wont be copied over.
* ex: [".*\\.git.*"]
* @return true on success
*/
boolean copyFileToContainer(String srcFilePath, String destinationDirectory, ArrayList<String> ignorePaths = []) {


File tarFile = createTar([srcFilePath], Files.createTempFile("docker_upload", ".tar").toString())
File tarFile = createTar([srcFilePath], Files.createTempFile("docker_upload", ".tar").toString(), ignorePaths)
dockerClient.putArchive(self.containerId, destinationDirectory, tarFile.newDataInputStream())

return tarFile.delete()
Expand All @@ -684,6 +754,8 @@ trait Container {

static class ContainerCallback<T> implements StreamCallback<T> {


Logger log = LoggerFactory.getLogger(ContainerCallback.class)
ArrayList<String> output = []

@Override
Expand All @@ -693,6 +765,7 @@ trait Container {
} else {
output.add(o.toString())
}
log.debug(output.last())

}
}
Expand Down Expand Up @@ -759,7 +832,6 @@ trait Container {
}



/**
* Creates a temporary container, runs a command, exits and removes container
* @param container a container object that hasnt yet been created
Expand All @@ -774,20 +846,18 @@ trait Container {
* @param dockerCertPath
* @return An array of the container logs, or just an array containing container id if timeoutMs == 0
*/
static ArrayList<String> runCmdAndRm( ArrayList<String> cmd, long timeoutMs, ArrayList<Map> mounts = [], String dockerHost = "", String dockerCertPath = "") {
static ArrayList<String> runCmdAndRm(ArrayList<String> cmd, long timeoutMs, ArrayList<Map> mounts = [], String dockerHost = "", String dockerCertPath = "") {


Container container = this.getConstructor(String, String).newInstance(dockerHost, dockerCertPath)

Logger log = LoggerFactory.getLogger(this.class)



log.info("Creating a $container.class.simpleName and running:")
log.info("\tCmd:" + cmd)



try {

container.containerName = container.containerName + "-cmd-" + System.currentTimeMillis().toString()[-5..-1]
Expand Down Expand Up @@ -821,15 +891,14 @@ trait Container {
log.info("\tContainer finisehd or timed out after ${System.currentTimeMillis() - start}ms")

if (container.running) {
log.info("\t"*2 + "Stopping container forcefully.")
log.info("\t" * 2 + "Stopping container forcefully.")
ArrayList<String> containerOut = container.containerLogs
assert container.stopAndRemoveContainer(1): "Error stopping and removing CMD container after it timed out"

throw new TimeoutException("CMD container timed out, was forcefully stopped and removed. Container logs:" + containerOut?.join("\n"))
}



ArrayList<String> containerOut = container.containerLogs

log.info("\tReturning ${containerOut.size()} log lines")
Expand All @@ -843,8 +912,8 @@ trait Container {

try {
container.stopAndRemoveContainer(1)
} catch (ignored){}

} catch (ignored) {
}


throw ex
Expand Down
3 changes: 1 addition & 2 deletions src/main/groovy/com/eficode/devstack/container/impl/BitbucketContainer.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package com.eficode.devstack.container.impl

import com.eficode.devstack.container.Container
import com.eficode.devstack.util.ImageBuilder
import de.gesellix.docker.client.EngineResponseContent
import de.gesellix.docker.remote.api.ContainerCreateRequest
import de.gesellix.docker.remote.api.HostConfig
import de.gesellix.docker.remote.api.ImageSummary
Expand Down Expand Up @@ -71,7 +70,7 @@ class BitbucketContainer implements Container {
c.exposedPorts = [(containerMainPort + "/tcp"): [:]]
c.hostConfig = new HostConfig().tap { h ->
h.portBindings = [(containerMainPort + "/tcp"): [new PortBinding("0.0.0.0", (containerMainPort))]]
h.mounts = this.mounts
h.mounts = this.preparedMounts
}
c.hostname = containerName
c.env = ["JVM_MAXIMUM_MEMORY=" + jvmMaxRam + "m", "JVM_MINIMUM_MEMORY=" + ((jvmMaxRam / 2) as String) + "m", "SETUP_BASEURL=" + baseUrl, "SERVER_PORT=" + containerMainPort] + customEnvVar
Expand Down
Loading

0 comments on commit e2559f6

Please sign in to comment.