Skip to content
This repository has been archived by the owner on Sep 21, 2021. It is now read-only.

Commit

Permalink
Cache the selenium-node image in cluster (#764)
Browse files Browse the repository at this point in the history 
* Cache the selenium-node image in cluster

Sets the `imagePullPolicy` to `IfNotPresent` instead of `Always`. 
This should allow caching in the cluster so that k8s does not have to make a request to the docker hub every time a node is to be created, thereby lowering overhead and network IO.
Apart from this, I have observed that scaling up nodes might present issues for docker hub - likely resource constraints from their side.

To avoid readiness probe errors from the nodes, it could be beneficial setting up the timeout from 1 second to 5. There isn't a lot of room for error with just one second; at the least, it's been observed that this made it slightly more stable from our side.

* Add ImagePullPolicy as an enum

* Add ImagePullPolicy to the config. Still needs to be read as a parameter if provided, or default to "Always". Right now, it is hard-coded as "Always".

* Fix timeout settings(mistakenly put 10 instead of 5)

* Fix auto-formatting of tabs

* Fix white-space auto-formatting

* Default imagePullPolicy to Always if environment "ZALENIUM_KUBERNETES_IMAGE_PULL_POLICY" is not set.

* Whitespace in ENV variable.

* Fix IntelliJ auto formatting(whitespace issues)

* Simplify logic for setting ENV variable for "ZALENIUM_KUBERNETES_IMAGE_PULL_POLICY"
  • Loading branch information
@cbll @diemol
cbll authored and diemol committed Nov 13, 2018
1 parent 15fc5e7 commit ae38e1f
Show file tree
Hide file tree
Showing 2 changed files with 31 additions and 16 deletions.
38 changes: 23 additions & 15 deletions src/main/java/de/zalando/ep/zalenium/container/kubernetes/KubernetesContainerClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ public class KubernetesContainerClient implements ContainerClient {
private List<HostAlias> hostAliases = new ArrayList<>();
private Map<String, String> nodeSelector = new HashMap<>();
private List<Toleration> tolerations = new ArrayList<>();
private String imagePullPolicy;

private final Map<String, Quantity> seleniumPodLimits = new HashMap<>();
private final Map<String, Quantity> seleniumPodRequests = new HashMap<>();
Expand Down Expand Up @@ -139,6 +140,8 @@ private void buildResourceMaps() {
}
}
}
// Default to imagePullPolicy: Always if ENV variable "ZALENIUM_KUBERNETES_IMAGE_PULL_POLICY" is not provided
imagePullPolicy = environment.getStringEnvVariable("ZALENIUM_KUBERNETES_IMAGE_PULL_POLICY", ImagePullPolicyType.Always.name());
}

private void discoverHostAliases() {
Expand Down Expand Up @@ -308,7 +311,7 @@ public int getRunningContainers(String image) {

@Override
public ContainerCreationStatus createContainer(String zaleniumContainerName, String image, Map<String, String> envVars,
String nodePort) {
String nodePort) {
String containerIdPrefix = String.format("%s-%s-", zaleniumAppName, nodePort);

// Convert the environment variables into the Kubernetes format.
Expand All @@ -329,6 +332,7 @@ public ContainerCreationStatus createContainer(String zaleniumContainerName, Str
labels.putAll(appLabelMap);
labels.putAll(podSelector);
config.setLabels(labels);
config.setImagePullPolicy(imagePullPolicy);
config.setMountedSharedFoldersMap(mountedSharedFoldersMap);
config.setHostAliases(hostAliases);
config.setNodeSelector(nodeSelector);
Expand Down Expand Up @@ -365,7 +369,7 @@ public String getContainerIp(String containerName) {
return null;
}
}

public boolean isReady(ContainerCreationStatus container) {
Pod pod = client.pods().withName(container.getContainerName()).get();
if (pod == null) {
Expand All @@ -383,19 +387,19 @@ public boolean isReady(ContainerCreationStatus container) {
public boolean isTerminated(ContainerCreationStatus container) {
Pod pod = client.pods().withName(container.getContainerName()).get();
if (pod == null) {
logger.info("Container {} has no pod - terminal.", container);
logger.info("Container {} has no pod - terminal.", container);
return true;
}
else {
List<ContainerStatus> containerStatuses = pod.getStatus().getContainerStatuses();
Optional<ContainerStateTerminated> terminated = containerStatuses.stream()
.flatMap(status -> Optional.ofNullable(status.getState()).map(Stream::of).orElse(Stream.empty()))
.flatMap(state -> Optional.ofNullable(state.getTerminated()).map(Stream::of).orElse(Stream.empty()))
.findFirst();
terminated.ifPresent(state -> logger.info("Container {} is {} - terminal.", container, state));
return terminated.isPresent();
List<ContainerStatus> containerStatuses = pod.getStatus().getContainerStatuses();
Optional<ContainerStateTerminated> terminated = containerStatuses.stream()
.flatMap(status -> Optional.ofNullable(status.getState()).map(Stream::of).orElse(Stream.empty()))
.flatMap(state -> Optional.ofNullable(state.getTerminated()).map(Stream::of).orElse(Stream.empty()))
.findFirst();

terminated.ifPresent(state -> logger.info("Container {} is {} - terminal.", container, state));

return terminated.isPresent();
}
}

Expand Down Expand Up @@ -537,6 +541,10 @@ private enum ResourceType {
REQUEST, LIMIT
}

private enum ImagePullPolicyType {
Always, IfNotPresent
}

public static DoneablePod createDoneablePodDefaultImpl(PodConfiguration config) {
DoneablePod doneablePod = config.getClient().pods()
.createNew()
Expand All @@ -557,7 +565,7 @@ public static DoneablePod createDoneablePodDefaultImpl(PodConfiguration config)
.addNewContainer()
.withName("selenium-node")
.withImage(config.getImage())
.withImagePullPolicy("Always")
.withImagePullPolicy(config.getImagePullPolicy())
.addAllToEnv(config.getEnvVars())
.addNewVolumeMount()
.withName("dshm")
Expand All @@ -571,13 +579,13 @@ public static DoneablePod createDoneablePodDefaultImpl(PodConfiguration config)
// so then we can initiate a registration.
.withNewReadinessProbe()
.withNewExec()
.addToCommand(new String[] {"/bin/sh", "-c", "http_proxy=\"\" curl -s http://`getent hosts ${HOSTNAME} | awk '{ print $1 }'`:"
.addToCommand(new String[] {"/bin/sh", "-c", "http_proxy=\"\" curl -s http://`getent hosts ${HOSTNAME} | awk '{ print $1 }'`:"
+ config.getNodePort() + "/wd/hub/status | jq .value.ready | grep true"})
.endExec()
.withInitialDelaySeconds(5)
.withFailureThreshold(60)
.withPeriodSeconds(1)
.withTimeoutSeconds(1)
.withTimeoutSeconds(5)
.withSuccessThreshold(1)
.endReadinessProbe()
.endContainer()
Expand Down
9 changes: 8 additions & 1 deletion src/main/java/de/zalando/ep/zalenium/container/kubernetes/PodConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ public class PodConfiguration {
private KubernetesClient client;
private String containerIdPrefix;
private String image;
private String imagePullPolicy;
private List<EnvVar> envVars;
private List<HostAlias> hostAliases;
private Map<String, String> labels;
Expand All @@ -25,7 +26,7 @@ public class PodConfiguration {
private Map<String, Quantity> podRequests;
private Map<String, String> nodeSelector;
private List<Toleration> tolerations;

private String nodePort;

public String getNodePort() {
Expand All @@ -52,6 +53,12 @@ public String getImage() {
public void setImage(String image) {
this.image = image;
}
public String getImagePullPolicy() {
return imagePullPolicy;
}
public void setImagePullPolicy(String imagePullPolicy) {
this.imagePullPolicy = imagePullPolicy;
}
public List<EnvVar> getEnvVars() {
return envVars;
}
Expand Down

0 comments on commit ae38e1f

Please sign in to comment.