Merge pull request #74 from mesosphere/offer_storage

better offer lifecycle management

DEVELOP.md

@@ -3,6 +3,8 @@ Development of Kubernetes-Mesos
 
 * [Prerequisites](#prerequisites)
 * [Build](README.md#build) the framework
+* [Testing](#testing)
+* [Profiling](#profiling)
 
 ### Prerequisites
 To get started with development you'll need to install some prerequisites:
@@ -66,5 +68,37 @@ $ mkdir -pv /opt && (export GOPATH=/opt; cd /opt &&
     ln -sv /opt/bin/godep /usr/local/bin/godep)
 ```
 
+### Testing
+
+There is a Makefile target called `test` that will execute unit tests for packages that have them.
+```shell
+$ make test
+```
+
+### Profiling
+
+The project Makefile supports the inclusion of arbitrary Go build flags.
+To generate a build with profiling enabled, include `TAGS=profile`.
+If you're not using the Makefile then you must supply `-tags 'profile'` as part of your `go build` command.
+If you're using the dockerbuilder then read the [instructions][3] for profiling in the accompanying documentation.
+
+```shell
+$ make TAGS=profile
+```
+
+Profiling, when enabled, is supported for both the `kubernetes-mesos` (framework) and `kubernetes-executor` binaries:
+```shell
+$ ts=$(date +'%Y%m%d%H%M%S')
+$ curl http://${servicehost}:9000/debug/pprof/heap >framework.heap.$ts
+$ curl http://10.132.189.${slave}:10250/debug/pprof/heap >${slave}.heap.$ts
+```
+
+If you have captured two or more heaps it's trivial to use the Go pprof tooling to generate reports:
+```shell
+$ go tool pprof --base=./${slave}.heap.20141117175634 --inuse_objects --pdf \
+  ./bin/kubernetes-executor ./${slave}.heap.20141120162503 >${slave}-20141120a.pdf
+```
+
 [1]: https://github.com/mesosphere/kubernetes-mesos#build
 [2]: https://github.com/tools/godep
+[3]: hack/dockerbuild/README.md#profiling

Makefile

@@ -11,14 +11,24 @@ K8S_CMD		:= \
                    github.com/GoogleCloudPlatform/kubernetes/cmd/kubecfg		\
                    github.com/GoogleCloudPlatform/kubernetes/cmd/proxy
 FRAMEWORK_CMD	:= \
-                   github.com/mesosphere/kubernetes-mesos/kubernetes-mesos		\
+                   github.com/mesosphere/kubernetes-mesos/kubernetes-mesos	\
                    github.com/mesosphere/kubernetes-mesos/kubernetes-executor
+FRAMEWORK_LIB	:= \
+		   github.com/mesosphere/kubernetes-mesos/scheduler	\
+		   github.com/mesosphere/kubernetes-mesos/service	\
+		   github.com/mesosphere/kubernetes-mesos/executor	\
+		   github.com/mesosphere/kubernetes-mesos/queue
 
+# a list of upstream projects for which we test the availability of patches
+PATCH_SCRIPT	:= $(current_dir)/hack/patches/apply.sh
 
 # TODO: make this something more reasonable
 DESTDIR		?= /target
 
-.PHONY: all error require-godep framework require-vendor proxy install info bootstrap require-gopath format
+# default build tags
+TAGS		?=
+
+.PHONY: all error require-godep framework require-vendor proxy install info bootstrap require-gopath format test patch
 
 ifneq ($(WITH_MESOS_DIR),)
 
@@ -40,7 +50,7 @@ WITH_MESOS_CGO_FLAGS :=  \
 
 endif
 
-all: proxy framework
+all: patch proxy framework
 
 error:
 	echo -E "$@: ${MSG}" >&2
@@ -58,14 +68,13 @@ proxy: require-godep
 require-vendor:
 
 framework: require-godep
-	env $(WITH_MESOS_CGO_FLAGS) go install $${WITH_RACE:+-race} $(FRAMEWORK_CMD)
+	env $(WITH_MESOS_CGO_FLAGS) go install -v -x -tags '$(TAGS)' $${WITH_RACE:+-race} $(FRAMEWORK_CMD)
 
 format: require-gopath
-	go fmt	github.com/mesosphere/kubernetes-mesos/kubernetes-mesos \
-		github.com/mesosphere/kubernetes-mesos/kubernetes-executor \
-		github.com/mesosphere/kubernetes-mesos/scheduler \
-		github.com/mesosphere/kubernetes-mesos/service \
-		github.com/mesosphere/kubernetes-mesos/executor
+	go fmt $(FRAMEWORK_CMD) $(FRAMEWORK_LIB)
+
+test: require-gopath
+	go test $(FRAMEWORK_LIB)
 
 install: all
 	mkdir -p $(DESTDIR)
@@ -79,6 +88,13 @@ info:
 	@echo CGO_CXXFLAGS="$(CGO_CXXFLAGS)"
 	@echo CGO_LDFLAGS="$(CGO_LDFLAGS)"
 	@echo RACE_FLAGS=$${WITH_RACE:+-race}
+	@echo TAGS=$(TAGS)
 
 bootstrap: require-godep
 	godep restore
+
+patch: $(PATCH_SCRIPT)
+	$(PATCH_SCRIPT)
+
+$(PATCH_SCRIPT):
+	test -x $@ || chmod +x $@

executor/executor.go

@@ -187,30 +187,38 @@ func (k *KubernetesExecutor) LaunchTask(driver mesos.ExecutorDriver, taskInfo *m
 
 			return
 		}
-		// TODO(jdefelice) remove the task (and pod state) here?
-		k.sendStatusUpdate(taskInfo.GetTaskId(), mesos.TaskState_TASK_LOST, "Task lost: launch failed")
+
+		k.lock.Lock()
+		defer k.lock.Unlock()
+		k.reportLostTask(taskId, "Task lost: launch failed")
 	}()
 }
 
 // Intended to be executed as part of the pod monitoring loop, this fn (ultimately) checks with Docker
-// whether the pod is running and will return true if the pod becomes unkown. If there's still a task
-// record on file, but no pod in Docker, then we'll also send a TASK_LOST event.
+// whether the pod is running. It will only return false if the task is still registered and the pod is
+// registered in Docker. Otherwise it returns true. If there's still a task record on file, but no pod
+// in Docker, then we'll also send a TASK_LOST event.
 func (k *KubernetesExecutor) checkForLostPodTask(taskInfo *mesos.TaskInfo, isKnownPod func() bool) bool {
 	// TODO (jdefelice) don't send false alarms for deleted pods (KILLED tasks)
-	k.lock.RLock()
-	defer k.lock.RUnlock()
+	k.lock.Lock()
+	defer k.lock.Unlock()
 
-	if !isKnownPod() {
-		taskId := taskInfo.GetTaskId().GetValue()
-		if _, ok := k.tasks[taskId]; !ok {
-			log.Infof("Ignoring lost container: task not present")
+	// TODO(jdef) we should really consider k.pods here, along with what docker is reporting, since the kubelet
+	// may constantly attempt to instantiate a pod as long as it's in the pod state that we're handing to it.
+	// otherwise, we're probably reporting a TASK_LOST prematurely. Should probably consult RestartPolicy to
+	// determine appropriate behavior. Should probably also gracefully handle docker daemon restarts.
+	taskId := taskInfo.GetTaskId().GetValue()
+	if _, ok := k.tasks[taskId]; ok {
+		if isKnownPod() {
+			return false
 		} else {
-			// TODO(jdefelice) remove the task (and pod state) here?
-			k.sendStatusUpdate(taskInfo.GetTaskId(), mesos.TaskState_TASK_LOST, "Task lost: container disappeared")
+			log.Warningf("Detected lost pod, reporting lost task %v", taskId)
+			k.reportLostTask(taskId, "Task lost: container disappeared")
 		}
-		return true
+	} else {
+		log.V(2).Infof("Task %v no longer registered, stop monitoring for lost pods", taskId)
 	}
-	return false
+	return true
 }
 
 // KillTask is called when the executor receives a request to kill a task.
@@ -243,8 +251,9 @@ func (k *KubernetesExecutor) killPodForTask(tid, reason string) {
 	// to be deprecated.
 	pid := task.containerManifest.ID
 	if _, found := k.pods[pid]; !found {
-		log.Warningf("Cannot remove Unknown pod %v\n", pid)
+		log.Warningf("Cannot remove Unknown pod %v for task %v", pid, tid)
 	} else {
+		log.V(2).Infof("Deleting pod %v for task %v", pid, tid)
 		delete(k.pods, pid)
 
 		// Send the pod updates to the channel.
@@ -259,6 +268,37 @@ func (k *KubernetesExecutor) killPodForTask(tid, reason string) {
 	k.sendStatusUpdate(task.mesosTaskInfo.GetTaskId(), mesos.TaskState_TASK_KILLED, reason)
 }
 
+// Reports a lost task to the slave and updates internal task and pod tracking state.
+// Assumes that the caller is locking around pod and task state.
+func (k *KubernetesExecutor) reportLostTask(tid, reason string) {
+	task, ok := k.tasks[tid]
+	if !ok {
+		log.Infof("Failed to report lost task, unknown task %v\n", tid)
+		return
+	}
+	delete(k.tasks, tid)
+
+	// TODO(nnielsen): Verify this assumption. Manifest ID's has been marked
+	// to be deprecated.
+	pid := task.containerManifest.ID
+	if _, found := k.pods[pid]; !found {
+		log.Warningf("Cannot remove Unknown pod %v for lost task %v", pid, tid)
+	} else {
+		log.V(2).Infof("Deleting pod %v for lost task %v", pid, tid)
+		delete(k.pods, pid)
+
+		// Send the pod updates to the channel.
+		update := kubelet.PodUpdate{Op: kubelet.SET}
+		for _, p := range k.pods {
+			update.Pods = append(update.Pods, *p)
+		}
+		k.updateChan <- update
+	}
+	// TODO(yifan): Check the result of the kill event.
+
+	k.sendStatusUpdate(task.mesosTaskInfo.GetTaskId(), mesos.TaskState_TASK_LOST, reason)
+}
+
 // FrameworkMessage is called when the framework sends some message to the executor
 func (k *KubernetesExecutor) FrameworkMessage(driver mesos.ExecutorDriver, message string) {
 	log.Infof("Receives message from framework %v\n", message)

executor/server.go

@@ -36,6 +36,7 @@ import (
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/dockertools"
 	"github.com/golang/glog"
 	"github.com/google/cadvisor/info"
+	"github.com/mesosphere/kubernetes-mesos/profile"
 )
 
 // Server is a http.Handler which exposes kubelet functionality over HTTP.
@@ -85,6 +86,7 @@ func NewServer(host HostInterface, updates chan<- interface{}, ns string) Server
 // InstallDefaultHandlers registers the set of supported HTTP request patterns with the mux.
 func (s *Server) InstallDefaultHandlers() {
 	healthz.InstallHandler(s.mux)
+	profile.InstallHandler(s.mux)
 	s.mux.HandleFunc("/podInfo", s.handlePodInfo)
 	s.mux.HandleFunc("/stats/", s.handleStats)
 	s.mux.HandleFunc("/logs/", s.handleLogs)
@@ -98,6 +100,7 @@ func (s *Server) error(w http.ResponseWriter, err error) {
 
 // handlePodInfo handles podInfo requests against the Kubelet.
 func (s *Server) handlePodInfo(w http.ResponseWriter, req *http.Request) {
+	req.Close = true
 	u, err := url.ParseRequestURI(req.RequestURI)
 	if err != nil {
 		s.error(w, err)

hack/dockerbuild/README.md

@@ -21,7 +21,7 @@ This example copies the resulting binaries into the host-mounted volume `/tmp/ta
 
 To build and copy the binaries:
 
-    $ docker run -rm -v /tmp/target:/target k8s-mesos-builder
+    $ docker run --rm -v /tmp/target:/target k8s-mesos-builder
     ...
     git clone https://${GOPKG}.git .
     Cloning into '.'...
@@ -43,7 +43,7 @@ To build and copy the binaries:
 
 Alternatively, it can be used to generate binaries from a branch:
 
-    $ docker run -rm -v /tmp/target:/target -e GIT_BRANCH=default_port k8s-mesos-builder
+    $ docker run --rm -v /tmp/target:/target -e GIT_BRANCH=default_port k8s-mesos-builder
 
 Want a quick-and-dirty development environment to start hacking?
 
@@ -52,8 +52,34 @@ Want a quick-and-dirty development environment to start hacking?
 
 Need to build the project, but from a forked git repo?
 
-    $ docker run -rm -v /tmp/target:/target -e GIT_REPO=https://github.com/whoami/kubernetes-mesos k8s-mesos-builder
+    $ docker run --rm -v /tmp/target:/target -e GIT_REPO=https://github.com/whoami/kubernetes-mesos k8s-mesos-builder
 
 To hack in your currently checked out repo mount the root of the github repo to `/snapshot`:
 
     $ docker run -ti -v /tmp/target:/target -v /home/jdef/kubernetes-mesos:/snapshot k8s-mesos-builder bash
+
+## Profiling
+
+Profiling in the cloud with Kubernetes-Mesos is easy!
+First, ssh into your Mesos cluster and generate a set of project binaries with profiling enabled (the `TAGS` variable is important here):
+
+    $ docker run --rm -ti -e GIT_BRANCH=offer_storage -e TAGS=profile \
+        -v $(pwd)/bin:/target jdef/kubernetes-mesos:dockerbuild
+
+Next, [start the framework](https://github.com/mesosphere/kubernetes-mesos/#start-the-framework) and schedule some pods.
+Once the framework and executors are up and running you can start capturing heaps:
+
+    $ ts=$(date +'%Y%m%d%H%M%S')
+    $ curl http://${servicehost}:9000/debug/pprof/heap >framework.heap.$ts
+    $ for slave in 240 242 243; do
+          curl http://10.132.189.${slave}:10250/debug/pprof/heap >${slave}.heap.$ts;
+      done
+
+Once you have a few heaps, you can generate reports.
+Additional packages may be required to support the reporting format you desire.
+
+    $ apt-get install ghostscript graphviz
+    $ go tool pprof --base=./framework.heap.20141117175634 --inuse_objects --pdf \
+        ./bin/kubernetes-executor ./framework.heap.20141120162503 >framework-20141120a.pdf
+
+For more details regarding profiling read the [pprof](http://golang.org/pkg/net/http/pprof/) package documentation.

hack/patches/README.md

@@ -0,0 +1,22 @@
+### Overview
+Periodically bug fixes will be needed in upstream code that this project depends upon.
+Rather than patch incorporated vendored code, this directory provides a place to isolate individual patches that may be applied at build time.
+**For now only a single HUNK should be present in each `.patch` file.**
+
+**NOTE:** This is not intended to replace proper pull-requests in other projects.
+Rather it is a short-term stop-gap solution for moving forward with patched code until the appropriate PR's are applied upstream and this project has been rebased to a revision that includes said PR's.
+
+### Naming Convention
+Patch files are named according to the upstream project they apply to and the issue number relative to **this project**.
+```
+  {patched-project}---{k8s-mesos-issue}.patch
+```
+
+For example, a file named `k8s---issue1234.patch` would indicate a patch for the Kubernetes project, tracked by issue 1234 in this project's (kubernetes-mesos) issues list.
+Issue 1234 should cross-reference any relevant PR's in the upstream project's repository.
+
+#### Projects
+
+Project Code | Project Name
+-------------|-------------
+ k8s         | [kubernetes](https://github.com/GoogleCloudPlatform/kubernetes)

hack/patches/apply.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+function die() {
+	test "x$1" = "x" || echo -E "$1" >&2
+	exit 1
+}
+
+test -n "$GOPATH" || die Missing GOPATH
+pkg="${GOPATH%%:*}"
+echo GO packages in $pkg will be patched
+
+test -n "$pkg" || die Invalid GOPATH=$GOPATH
+home=$(dirname "$0")
+home=$(readlink -f "$home")
+echo Patch directory $home
+
+# Add new k/v pairs for each project repo that may require patching
+# and update the README.md as entries are modified here
+declare -A pmap
+pmap=(
+  [k8s]=github.com/GoogleCloudPlatform/kubernetes
+)
+
+# TODO(jdef) at some point we should be able to apply patches with
+# multiple hunks, ala:
+# http://unix.stackexchange.com/questions/65698/how-to-make-patch-ignore-already-applied-hunks
+
+for k in "${!pmap[@]}"; do
+  repo="${pmap["${k}"]}"
+  echo "Checking patches for ${k}.. ($repo)"
+  find "${home}" -type f -name "${k}---issue*.patch" | while IFS= read -r f; do
+    #ff="${f%%.patch}"
+    #test "x$f" != "x$ff" || continue
+    cmd=( patch -p1 -s -r- -i"$f" )
+    echo -n -E "${cmd[@]}"
+    output=$(cd "${pkg}/src/${repo}" && "${cmd[@]}") && echo || {
+      echo -E "$output" | \
+        grep -q 'Reversed (or previously applied) patch detected' && \
+          echo " (already applied)" || \
+          { echo; die "Failed to apply patch ${f}: ${output}"; }
+    }
+  done
+done

hack/patches/k8s---issue77.patch

@@ -0,0 +1,14 @@
+diff --git a/pkg/kubelet/dockertools/docker.go b/pkg/kubelet/dockertools/docker.go
+index b42e2ce..b0f7808 100644
+--- a/pkg/kubelet/dockertools/docker.go
++++ b/pkg/kubelet/dockertools/docker.go
+@@ -215,6 +215,9 @@ func GetDockerPodInfo(client DockerInterface, podFullName, uuid string) (api.Pod
+ 	}
+
+ 	for _, value := range containers {
++		if len(value.Names) == 0 {
++			continue
++		}
+ 		dockerManifestID, dockerUUID, dockerContainerName, _ := ParseDockerName(value.Names[0])
+ 		if dockerManifestID != podFullName {
+ 			continue

kubernetes-mesos/main.go

@@ -47,6 +47,7 @@ import (
 	goetcd "github.com/coreos/go-etcd/etcd"
 	log "github.com/golang/glog"
 	"github.com/mesos/mesos-go/mesos"
+	_ "github.com/mesosphere/kubernetes-mesos/profile"
 	kmscheduler "github.com/mesosphere/kubernetes-mesos/scheduler"
 	kmendpoint "github.com/mesosphere/kubernetes-mesos/service"
 )
@@ -175,6 +176,8 @@ func main() {
 
 	driver.Init()
 	defer driver.Destroy()
+
+	mesosPodScheduler.Init()
 	go driver.Start()
 
 	log.V(2).Info("Serving executor artifacts...")