diff --git a/.goreleaser-stable.yaml b/.goreleaser-stable.yaml index 1b236db8d..3fb3e4901 100644 --- a/.goreleaser-stable.yaml +++ b/.goreleaser-stable.yaml @@ -171,11 +171,7 @@ builds: binary: #@ bin main: #@ "./cmd/{}".format(bin) env: - #@ if bin == "runu": - CGO_ENABLED=1 - #@ else: - - CGO_ENABLED=0 - #@ end - GOMOD=kraftkit.sh goos: - #@ os @@ -186,9 +182,20 @@ builds: - -X {{ .Env.GOMOD }}/internal/version.version={{ .Version }} - -X {{ .Env.GOMOD }}/internal/version.commit={{ .Commit }} - -X {{ .Env.GOMOD }}/internal/version.buildTime={{ .Date }} + #@ if bin == "kraft" and os == "linux": + - -linkmode external -extldflags "-static -lyajl -Wl,--whole-archive -llzma -lbz2 -lzstd -llzo2 + -lxenguest -Wl,--no-whole-archive -lxenevtchn -lxenlight -lxenstore -lxenctrl + -lxenforeignmemory -lxenforeignmemory -lxencall -lxentoolcore -lxenhypfs + -lxendevicemodel -lxengnttab -lxentoollog -lz -lnl-route-3 -lnl-3 -luuid -lutil" + #@ else: + - -linkmode external -extldflags "-static" + #@ tags: - containers_image_storage_stub - containers_image_openpgp + #@ if bin == "kraft" and os == "linux": + - xen + #@ end #@ end #@ end #@ end diff --git a/.goreleaser-staging.yaml b/.goreleaser-staging.yaml index 2b60ee425..58c1d9872 100644 --- a/.goreleaser-staging.yaml +++ b/.goreleaser-staging.yaml @@ -84,11 +84,7 @@ builds: binary: #@ bin main: #@ "./cmd/{}".format(bin) env: - #@ if bin == "runu": - CGO_ENABLED=1 - #@ else: - - CGO_ENABLED=0 - #@ end - GOMOD=kraftkit.sh goos: - #@ os @@ -99,9 +95,20 @@ builds: - -X {{ .Env.GOMOD }}/internal/version.version={{ .Version }} - -X {{ .Env.GOMOD }}/internal/version.commit={{ .Commit }} - -X {{ .Env.GOMOD }}/internal/version.buildTime={{ .Date }} + #@ if bin == "kraft" and os == "linux": + - -linkmode external -extldflags "-static -lyajl -Wl,--whole-archive -llzma -lbz2 -lzstd -llzo2 + -lxenguest -Wl,--no-whole-archive -lxenevtchn -lxenlight -lxenstore -lxenctrl + -lxenforeignmemory -lxenforeignmemory -lxencall -lxentoolcore -lxenhypfs + -lxendevicemodel -lxengnttab -lxentoollog -lz -lnl-route-3 -lnl-3 -luuid -lutil" + #@ else: + - -linkmode external -extldflags "-static" + #@ tags: - containers_image_storage_stub - containers_image_openpgp + #@ if bin == "kraft" and os == "linux": + - xen + #@ end #@ end #@ end #@ end diff --git a/Makefile b/Makefile index fb6f75f15..18f88386b 100644 --- a/Makefile +++ b/Makefile @@ -140,16 +140,20 @@ $(addprefix $(.PROXY), $(BIN)): GO_GCFLAGS ?= -N -l else $(addprefix $(.PROXY), $(BIN)): GO_LDFLAGS ?= -s -w endif + +ifeq ($(XEN), y) +$(addprefix $.PROXY), $(BIN)): TAGS ?= xen, +endif +$(addprefix $(.PROXY), $(BIN)): TAGS += containers_image_storage_stub,containers_image_openpgp $(addprefix $(.PROXY), $(BIN)): GO_LDFLAGS += -X "$(GOMOD)/internal/version.version=$(VERSION)" $(addprefix $(.PROXY), $(BIN)): GO_LDFLAGS += -X "$(GOMOD)/internal/version.commit=$(GIT_SHA)" $(addprefix $(.PROXY), $(BIN)): GO_LDFLAGS += -X "$(GOMOD)/internal/version.buildTime=$(shell date)" -$(addprefix $(.PROXY), $(BIN)): tidy $(addprefix $(.PROXY), $(BIN)): GOOS=$(GOOS) \ GOARCH=$(GOARCH) \ $(GO) build \ -v \ - -tags "containers_image_storage_stub,containers_image_openpgp" \ + -tags '$(TAGS)' \ -buildmode=pie \ -gcflags=all='$(GO_GCFLAGS)' \ -ldflags='$(GO_LDFLAGS)' \ @@ -164,6 +168,10 @@ $(addprefix $(.PROXY), $(TOOLS)): GO_GCFLAGS ?= -N -l else $(addprefix $(.PROXY), $(TOOLS)): GO_LDFLAGS ?= -s -w endif +ifeq ($(XEN), y) +$(addprefix $.PROXY), $(TOOLS)): TAGS ?= xen, +endif +$(addprefix $(.PROXY), $(TOOLS)): TAGS += containers_image_storage_stub,containers_image_openpgp $(addprefix $(.PROXY), $(TOOLS)): GO_LDFLAGS += -X "$(GOMOD)/internal/version.version=$(VERSION)" $(addprefix $(.PROXY), $(TOOLS)): GO_LDFLAGS += -X "$(GOMOD)/internal/version.commit=$(GIT_SHA)" $(addprefix $(.PROXY), $(TOOLS)): GO_LDFLAGS += -X "$(GOMOD)/internal/version.buildTime=$(shell date)" @@ -264,6 +272,7 @@ buildenv-myself-full: ## OCI image containing the build environment for KraftKit buildenv-myself: ## OCI image containing KraftKit binaries. buildenv-qemu: ## OCI image containing a Unikraft-centric build of QEMU. buildenv-github-action: ## OCI image used when building Unikraft unikernels in GitHub Actions. +buildenv-xen: ## OCI image containing a Unikraft-centric build of Xen. tools: ## Build all tools. kraft: ## The kraft binary. runu: ## The runu binary. diff --git a/buildenvs/Makefile b/buildenvs/Makefile index 5d2497a3c..7d5f23cd9 100644 --- a/buildenvs/Makefile +++ b/buildenvs/Makefile @@ -22,6 +22,26 @@ PLATFORM ?= linux/x86_64 WITH_CACHE ?= y +.PHONY: xen +xen: XEN_VERSION ?= 4.18 +xen: MAKE_NPROC ?= $(shell nproc) +xen: ENVIRONMENT ?= xen +xen: IMAGE ?= $(REGISTRY)/xen:$(XEN_VERSION) +ifeq ($(WITH_CACHE),y) +xen: _WITH_CACHE := --cache-from $(IMAGE) +else +xen: _WITH_CACHE := --no-cache +endif +xen: + $(DOCKER) build \ + --platform $(PLATFORM) \ + --build-arg XEN_VERSION=$(XEN_VERSION) \ + --build-arg MAKE_NPROC=$(MAKE_NPROC) \ + --tag $(IMAGE) \ + $(_WITH_CACHE) \ + --file $(BUILDENVSDIR)/xen.Dockerfile \ + $(DOCKER_BUILD_EXTRA) $(WORKDIR) + .PHONY: qemu qemu: QEMU_VERSION ?= 8.2.4 qemu: MAKE_NPROC ?= $(shell nproc) @@ -47,6 +67,7 @@ myself: GO_VERSION ?= 1.22.3 myself: ENVIRONMENT ?= myself myself: IMAGE ?= $(REGISTRY)/myself:$(IMAGE_TAG) myself: TARGET ?= kraftkit +myself: XEN_VERSION ?= 4.18 ifeq ($(WITH_CACHE),y) myself: _WITH_CACHE := --cache-from $(IMAGE) else @@ -56,6 +77,7 @@ myself: $(DOCKER) build \ --platform $(PLATFORM) \ --build-arg GO_VERSION=$(GO_VERSION) \ + --build-arg XEN_VERSION=$(XEN_VERSION) \ --tag $(IMAGE) \ --target $(TARGET) \ $(_WITH_CACHE) \ @@ -72,6 +94,7 @@ base: GO_VERSION ?= 1.22.3 base: IMAGE ?= $(REGISTRY)/base:$(IMAGE_TAG) base: KRAFTKIT_VERSION ?= latest base: QEMU_VERSION ?= 8.2.4 +base: XEN_VERSION ?= 4.18 ifeq ($(WITH_CACHE),y) base: _WITH_CACHE := --cache-from $(IMAGE) else @@ -82,6 +105,7 @@ base: --build-arg GO_VERSION=$(GO_VERSION) \ --build-arg KRAFTKIT_VERSION=$(KRAFTKIT_VERSION) \ --build-arg QEMU_VERSION=$(QEMU_VERSION) \ + --build-arg XEN_VERSION=$(XEN_VERSION) \ --build-arg REGISTRY=$(REGISTRY) \ --tag $(IMAGE) \ $(_WITH_CACHE) \ @@ -94,6 +118,7 @@ base-golang: GO_VERSION ?= 1.22.3 base-golang: IMAGE ?= $(REGISTRY)/base-golang:$(IMAGE_TAG) base-golang: KRAFTKIT_VERSION ?= latest base-golang: QEMU_VERSION ?= 8.2.4 +base-golang: XEN_VERSION ?= 4.18 ifeq ($(WITH_CACHE),y) base-golang: _WITH_CACHE := --cache-from $(IMAGE) else @@ -105,6 +130,7 @@ base-golang: --build-arg GO_VERSION=$(GO_VERSION) \ --build-arg KRAFTKIT_VERSION=$(KRAFTKIT_VERSION) \ --build-arg QEMU_VERSION=$(QEMU_VERSION) \ + --build-arg XEN_VERSION=$(XEN_VERSION) \ --build-arg REGISTRY=$(REGISTRY) \ --tag $(IMAGE) \ $(_WITH_CACHE) \ diff --git a/buildenvs/base.Dockerfile b/buildenvs/base.Dockerfile index 809ecf2d1..536ecf6d4 100644 --- a/buildenvs/base.Dockerfile +++ b/buildenvs/base.Dockerfile @@ -6,8 +6,10 @@ ARG DEBIAN_VERSION=bookworm-20240513 ARG KRAFTKIT_VERSION=latest ARG QEMU_VERSION=8.2.4 ARG REGISTRY=kraftkit.sh +ARG XEN_VERSION=4.18 FROM ${REGISTRY}/qemu:${QEMU_VERSION} AS qemu +FROM ${REGISTRY}/xen:${XEN_VERSION} AS xen FROM ${REGISTRY}/myself:${KRAFTKIT_VERSION} AS kraftkit FROM debian:${DEBIAN_VERSION} AS base @@ -15,6 +17,9 @@ COPY --from=qemu /bin/ /usr/local/bin COPY --from=qemu /share/qemu/ /share/qemu COPY --from=qemu /lib/x86_64-linux-gnu/ /lib/x86_64-linux-gnu COPY --from=kraftkit /kraft /usr/local/bin +COPY --from=xen /usr/lib/x86_64-linux-gnu/*.a /lib/x86_64-linux-gnu +COPY --from=xen /usr/local/lib/libxen*.a /usr/local/lib/libxen*.so* /usr/local/lib +COPY --from=xen /usr/local/include/* /usr/local/include # Install unikraft dependencies RUN set -xe; \ diff --git a/buildenvs/myself.Dockerfile b/buildenvs/myself.Dockerfile index f68142e00..4175dbe20 100644 --- a/buildenvs/myself.Dockerfile +++ b/buildenvs/myself.Dockerfile @@ -4,8 +4,11 @@ # You may not use this file except in compliance with the License. ARG GO_VERSION=1.22.3 +ARG XEN_VERSION=4.18 +ARG REGISTRY=kraftkit.sh -FROM golang:${GO_VERSION}-bookworm AS kraftkit-full +FROM ${REGISTRY}/xen:${XEN_VERSION} AS xen +FROM golang:${GO_VERSION}-bookworm AS kraftkit-full # Install build dependencies RUN set -xe; \ @@ -32,6 +35,20 @@ RUN set -xe; \ mv cosign-linux-amd64 /usr/local/bin/cosign; \ chmod +x /usr/local/bin/cosign; +COPY --from=xen /usr/local/lib/libxen*.a /usr/local/lib/libxen*.so* /usr/local/lib +COPY --from=xen /usr/local/include/* /usr/local/include/ +COPY --from=xen /usr/lib/x86_64-linux-gnu/liblzma.a \ + /usr/lib/x86_64-linux-gnu/libbz2.a \ + /usr/lib/x86_64-linux-gnu/libzstd.a \ + /usr/lib/x86_64-linux-gnu/liblzo2.a \ + /usr/lib/x86_64-linux-gnu/libyajl.a \ + /usr/lib/x86_64-linux-gnu/libz.a \ + /usr/lib/x86_64-linux-gnu/libnl-route-3.a \ + /usr/lib/x86_64-linux-gnu/libnl-3.a \ + /usr/lib/x86_64-linux-gnu/libuuid.a \ + /usr/lib/x86_64-linux-gnu/libutil.a \ + /usr/lib/x86_64-linux-gnu + WORKDIR /go/src/kraftkit.sh COPY --from=ghcr.io/goreleaser/goreleaser-cross:v1.22.3 /usr/bin/goreleaser /usr/bin/ @@ -56,4 +73,4 @@ FROM scratch AS kraftkit COPY --from=kraftkit-build /go/src/kraftkit.sh/dist/kraft /kraft -ENTRYPOINT [ "/kraft" ] \ No newline at end of file +ENTRYPOINT [ "/kraft" ] diff --git a/buildenvs/xen.Dockerfile b/buildenvs/xen.Dockerfile new file mode 100644 index 000000000..b50c441d4 --- /dev/null +++ b/buildenvs/xen.Dockerfile @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2024, Unikraft GmbH and The KraftKit Authors. +# Licensed under the BSD-3-Clause License (the "License"). +# You may not use this file except in compliance with the License. + +ARG DEBIAN_VERSION=bookworm-20240513 + +FROM debian:${DEBIAN_VERSION} AS xenbuild + +ARG XEN_VERSION=4.18 +ARG MAKE_NPROC=1 + +# The sed line should stay here until [1] is merged or forever if it's not +# [1]: https://lists.xenproject.org/archives/html/xen-devel/2024-07/msg00295.html + +RUN set -xe; \ + apt-get update; \ + apt-get install -y \ + binutils \ + bison \ + build-essential \ + cmake \ + flex \ + gcc \ + git \ + iasl \ + libbz2-dev \ + libglib2.0-dev \ + liblzo2-dev \ + liblz-dev \ + liblzma-dev \ + libnl-3-dev \ + libnl-route-3-dev \ + libncurses5-dev \ + libpixman-1-dev \ + libslirp-dev \ + libssh2-1-dev \ + libssl-dev \ + libuuid1 \ + libyajl-dev \ + libz3-dev \ + libzstd-dev \ + make \ + ninja-build \ + perl \ + pkg-config \ + python3 \ + python3-pip \ + python3-setuptools \ + python3-wheel \ + uuid-dev; \ + pip3 install python-config --break-system-packages; \ + git clone -b stable-${XEN_VERSION} https://xenbits.xen.org/git-http/xen.git /xen; \ + sed '/xs.opic: CFLAGS += -DUSE_PTHREAD/a xs.o: CFLAGS += -DUSE_PTHREAD' /xen/tools/libs/store/Makefile; \ + cd /xen; \ + ./configure --enable-virtfs; \ + make -j ${MAKE_NPROC} build-tools; \ + make -j ${MAKE_NPROC} install-tools; \ + cp /usr/lib/x86_64-linux-gnu/libyajl_s.a /usr/lib/x86_64-linux-gnu/libyajl.a + +FROM scratch AS xen + +COPY --from=xenbuild /usr/lib/x86_64-linux-gnu/liblzma.a \ + /usr/lib/x86_64-linux-gnu/libbz2.a \ + /usr/lib/x86_64-linux-gnu/libzstd.a \ + /usr/lib/x86_64-linux-gnu/liblzo2.a \ + /usr/lib/x86_64-linux-gnu/libyajl.a \ + /usr/lib/x86_64-linux-gnu/libz.a \ + /usr/lib/x86_64-linux-gnu/libnl-route-3.a \ + /usr/lib/x86_64-linux-gnu/libnl-3.a \ + /usr/lib/x86_64-linux-gnu/libuuid.a \ + /usr/lib/x86_64-linux-gnu/libutil.a \ + /usr/lib/x86_64-linux-gnu/ +COPY --from=xenbuild /usr/local/lib/libxen*.a /usr/local/lib/ +COPY --from=xenbuild /usr/local/lib/libxen*.so* /usr/local/lib/ +COPY --from=xenbuild /usr/local/include/*.h /usr/local/include/ diff --git a/go.mod b/go.mod index 07f2082c4..d07747bfb 100644 --- a/go.mod +++ b/go.mod @@ -80,6 +80,7 @@ require ( oras.land/oras-go/v2 v2.5.0 sdk.kraft.cloud v0.5.10-0.20240627124920-7a485e75d7b4 sigs.k8s.io/kustomize/kyaml v0.17.1 + xenbits.xenproject.org/git-http/xen.git/tools/golang/xenlight v0.0.0-20240402142354-17cf285d87e2 ) require ( diff --git a/go.sum b/go.sum index 1fecaddaf..42cedb9fb 100644 --- a/go.sum +++ b/go.sum @@ -1800,3 +1800,5 @@ sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= +xenbits.xenproject.org/git-http/xen.git/tools/golang/xenlight v0.0.0-20240402142354-17cf285d87e2 h1:f3OAMM0NgzlqWqZnuTIz6B6HPK1pGGfgKH6S94kYEWY= +xenbits.xenproject.org/git-http/xen.git/tools/golang/xenlight v0.0.0-20240402142354-17cf285d87e2/go.mod h1:tbZ4iMnk8RWkXPxTiCGdAw3hCOa3feShlf3sBh50uIc= diff --git a/internal/cli/kraft/logs/logs.go b/internal/cli/kraft/logs/logs.go index 17bf6d296..c731eeee1 100644 --- a/internal/cli/kraft/logs/logs.go +++ b/internal/cli/kraft/logs/logs.go @@ -167,7 +167,9 @@ func (opts *LogOptions) Run(ctx context.Context, args []string) error { if err != nil { errGroup = append(errGroup, err) } - if opts.Follow && machine.Status.State == machineapi.MachineStateRunning { + + // Sometimes the kernel can boot and exit faster than we can start tailing the logs + if opts.Follow && (machine.Status.State == machineapi.MachineStateRunning || machine.Status.State == machineapi.MachineStateExited) { observations.Add(machine) go func(machine *machineapi.Machine) { defer func() { diff --git a/internal/logtail/logtail.go b/internal/logtail/logtail.go index 2a3973e5b..bee458309 100644 --- a/internal/logtail/logtail.go +++ b/internal/logtail/logtail.go @@ -64,7 +64,11 @@ func NewLogTail(ctx context.Context, logFile string) (chan string, chan error, e switch event.Op { case fsnotify.Write: - peekAndRead(f, reader, &logs, &errs) + for { + if peekAndRead(f, reader, &logs, &errs) { + break + } + } } } } diff --git a/machine/platform/register_linux.go b/machine/platform/register_linux.go index 3db05fc64..d86f1e387 100644 --- a/machine/platform/register_linux.go +++ b/machine/platform/register_linux.go @@ -13,6 +13,7 @@ import ( "kraftkit.sh/config" "kraftkit.sh/internal/set" "kraftkit.sh/machine/firecracker" + "kraftkit.sh/machine/xen" "kraftkit.sh/store" ) @@ -43,13 +44,44 @@ var firecrackerV1alpha1Driver = func(ctx context.Context, opts ...any) (machinev ) } +var xenV1alpha1Driver = func(ctx context.Context, opts ...any) (machinev1alpha1.MachineService, error) { + service, err := xen.NewMachineV1alpha1Service(ctx) + if err != nil { + return nil, err + } + + embeddedStore, err := store.NewEmbeddedStore[machinev1alpha1.MachineSpec, machinev1alpha1.MachineStatus]( + filepath.Join( + config.G[config.KraftKit](ctx).RuntimeDir, + "machinev1alpha1", + ), + ) + if err != nil { + return nil, err + } + + return machinev1alpha1.NewMachineServiceHandler( + ctx, + service, + zip.WithStore[machinev1alpha1.MachineSpec, machinev1alpha1.MachineStatus](embeddedStore, zip.StoreRehydrationSpecNil), + zip.WithBefore(storePlatformFilter(PlatformXen)), + ) +} + func unixVariantStrategies() map[Platform]*Strategy { // TODO(jake-ciolek): The firecracker driver has a dependency on github.com/containernetworking/plugins/pkg/ns via // github.com/firecracker-microvm/firecracker-go-sdk // Unfortunately, it doesn't support darwin. - return map[Platform]*Strategy{ + unixMap := map[Platform]*Strategy{ PlatformFirecracker: { NewMachineV1alpha1: firecrackerV1alpha1Driver, }, } + + // have to check now otherwise it will error out on any interator + if _, err := xen.NewMachineV1alpha1Service(context.TODO()); err == nil { + unixMap[PlatformXen] = &Strategy{NewMachineV1alpha1: xenV1alpha1Driver} + } + + return unixMap } diff --git a/machine/xen/init.go b/machine/xen/init.go new file mode 100644 index 000000000..126d53b15 --- /dev/null +++ b/machine/xen/init.go @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright (c) 2024, Unikraft GmbH and The KraftKit Authors. +// Licensed under the BSD-3-Clause License (the "License"). +// You may not use this file except in compliance with the License. + +//go:build xen +// +build xen + +package xen + +import ( + "encoding/gob" + + "xenbits.xenproject.org/git-http/xen.git/tools/golang/xenlight" +) + +func init() { + gob.Register(xenlight.Domid(0)) +} diff --git a/machine/xen/stub.go b/machine/xen/stub.go new file mode 100644 index 000000000..91123fb41 --- /dev/null +++ b/machine/xen/stub.go @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright (c) 2024, Unikraft GmbH and The KraftKit Authors. +// Licensed under the BSD-3-Clause License (the "License"). +// You may not use this file except in compliance with the License. + +//go:build !xen +// +build !xen + +package xen + +import ( + "context" + "fmt" + + machinev1alpha1 "kraftkit.sh/api/machine/v1alpha1" +) + +func NewMachineV1alpha1Service(ctx context.Context) (machinev1alpha1.MachineService, error) { + return nil, fmt.Errorf("xen is not supported") +} diff --git a/machine/xen/v1alpha1.go b/machine/xen/v1alpha1.go new file mode 100644 index 000000000..758b58ed0 --- /dev/null +++ b/machine/xen/v1alpha1.go @@ -0,0 +1,567 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright (c) 2024, Unikraft GmbH and The KraftKit Authors. +// Licensed under the BSD-3-Clause License (the "License"). +// You may not use this file except in compliance with the License. + +//go:build xen +// +build xen + +package xen + +import ( + "context" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "slices" + "strings" + "time" + + zip "api.zip" + "github.com/acorn-io/baaah/pkg/merr" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/uuid" + + machinev1alpha1 "kraftkit.sh/api/machine/v1alpha1" + "kraftkit.sh/config" + "kraftkit.sh/internal/logtail" + "kraftkit.sh/log" + "kraftkit.sh/machine/network/macaddr" + "kraftkit.sh/unikraft/arch" + "kraftkit.sh/unikraft/export/v0/ukargparse" + "kraftkit.sh/unikraft/export/v0/uknetdev" + "kraftkit.sh/unikraft/export/v0/vfscore" + "xenbits.xenproject.org/git-http/xen.git/tools/golang/xenlight" +) + +const ( + XenMemoryScale = 1024 + XenMemoryDefault = 64 + XenCPUsDefault = 1 +) + +type machineV1alpha1Service struct{} + +func NewMachineV1alpha1Service(ctx context.Context) (machinev1alpha1.MachineService, error) { + return &machineV1alpha1Service{}, nil +} + +func (service *machineV1alpha1Service) Create(ctx context.Context, machine *machinev1alpha1.Machine) (*machinev1alpha1.Machine, error) { + cfg, err := NewXenConfig() + if err != nil { + return nil, err + } + + if machine.Status.KernelPath == "" { + return machine, fmt.Errorf("cannot create xen instace without a kernel") + } + + if arch.ArchitectureByName(machine.Spec.Architecture) == arch.ArchitectureUnknown { + return machine, fmt.Errorf("unsupported architecture %s", machine.Spec.Architecture) + } + + if _, err := os.Stat(machine.Status.KernelPath); err != nil && os.IsNotExist(err) { + return machine, fmt.Errorf("supplied kernel path does not exist: %s", machine.Status.KernelPath) + } + + if machine.ObjectMeta.UID == "" { + machine.ObjectMeta.UID = uuid.NewUUID() + } + + machine.Status.State = machinev1alpha1.MachineStateUnknown + + if len(machine.Status.StateDir) == 0 { + machine.Status.StateDir = filepath.Join(config.G[config.KraftKit](ctx).RuntimeDir, string(machine.ObjectMeta.UID)) + } + + if err := os.MkdirAll(machine.Status.StateDir, fs.ModeSetgid|0o775); err != nil { + return machine, err + } + + if len(machine.Status.LogFile) == 0 { + machine.Status.LogFile = filepath.Join(machine.Status.StateDir, "machine.log") + } + + fd, err := os.Create(machine.Status.LogFile) + if err != nil { + return machine, err + } + defer fd.Close() + + if machine.Spec.Resources.Requests == nil { + machine.Spec.Resources.Requests = make(corev1.ResourceList, 2) + } + + if machine.Spec.Resources.Requests.Memory().Value() == 0 { + quantity, err := resource.ParseQuantity(fmt.Sprintf("%dMi", XenMemoryDefault)) + if err != nil { + machine.Status.State = machinev1alpha1.MachineStateFailed + return machine, err + } + + machine.Spec.Resources.Requests[corev1.ResourceMemory] = quantity + } + + if machine.Spec.Resources.Requests.Cpu().Value() == 0 { + quantity, err := resource.ParseQuantity(fmt.Sprintf("%d", XenCPUsDefault)) + if err != nil { + machine.Status.State = machinev1alpha1.MachineStateFailed + return machine, err + } + + machine.Spec.Resources.Requests[corev1.ResourceCPU] = quantity + } + + if machine.Spec.Resources.Requests.Memory().Value() < XenMemoryScale { + return machine, fmt.Errorf("memory must be greater than %d bytes", XenMemoryScale) + } + + cfg.BInfo.MaxVcpus = int(machine.Spec.Resources.Requests.Cpu().Value()) + cfg.BInfo.MaxMemkb = uint64(machine.Spec.Resources.Requests.Memory().Value() / XenMemoryScale) + cfg.BInfo.Kernel = machine.Status.KernelPath + cfg.CInfo.Name = string(machine.ObjectMeta.Name) + cfg.CInfo.Type = xenlight.DomainTypePv + + if machine.Status.InitrdPath != "" { + cfg.BInfo.Ramdisk = machine.Status.InitrdPath + } + + if len(machine.Spec.Ports) > 0 { + return machine, fmt.Errorf("mapping ports is not supported for xen") + } + + kernelArgs, err := ukargparse.Parse(machine.Spec.KernelArgs...) + if err != nil { + return machine, err + } + + if len(machine.Spec.Networks) > 0 { + startMac, err := macaddr.GenerateMacAddress(true) + if err != nil { + return machine, err + } + + i := 0 + for _, network := range machine.Spec.Networks { + for _, iface := range network.Interfaces { + mac := iface.Spec.MacAddress + if mac == "" { + startMac = macaddr.IncrementMacAddress(startMac) + mac = startMac.String() + } + + nic, err := xenlight.NewDeviceNic() + if err != nil { + return nil, fmt.Errorf("could not create xen nic: %v", err) + } + + // TODO(andreistan26): Check if xen accepts CIDR notation + nic.Ip = fmt.Sprintf("%s %s %s", strings.Split(iface.Spec.CIDR, "/")[0], network.Netmask, network.Gateway) + nic.Bridge = network.IfName + nic.Mac = xenlight.Mac([]byte(mac)) + + cfg.Nics = append(cfg.Nics, *nic) + kernelArgs = append(kernelArgs, + uknetdev.NewParamIp().WithValue(uknetdev.NetdevIp{ + CIDR: iface.Spec.CIDR, + Gateway: network.Gateway, + DNS0: iface.Spec.DNS0, + DNS1: iface.Spec.DNS1, + Hostname: iface.Spec.Hostname, + Domain: iface.Spec.Domain, + }), + ) + i++ + } + } + } + + var fstab []string + + // TODO(andreistan26): Check if installed xen supports 9pfs + for i, vol := range machine.Spec.Volumes { + switch vol.Spec.Driver { + case "9p": + case "9pfs": + mounttag := fmt.Sprintf("fs%d", i+1) + p9Dev, err := xenlight.NewDeviceP9() + if err != nil { + return nil, err + } + + p9Dev.Tag = mounttag + p9Dev.Path = vol.Spec.Source + p9Dev.SecurityModel = "none" + + cfg.P9S = append(cfg.P9S, *p9Dev) + + fstab = append(fstab, vfscore.NewFstabEntry( + mounttag, + vol.Spec.Destination, + vol.Spec.Driver, + "", + "", + "mkmp", + ).String()) + case "initrd": + fstab = append(fstab, vfscore.NewFstabEntry( + "initrd0", + vol.Spec.Destination, + "extract", + "", + "", + "", + ).String()) + default: + return machine, fmt.Errorf("unsupported Xen volume driver: %v", vol.Spec.Driver) + } + } + + if len(fstab) > 0 { + kernelArgs = append(kernelArgs, vfscore.ParamVfsFstab.WithValue(fstab)) + } + + args := kernelArgs.Strings() + if len(args) > 0 { + args = append(args, "--") + } + + args = append(args, machine.Spec.ApplicationArgs...) + cfg.BInfo.Cmdline = strings.Join(args, " ") + + xenCtx, err := xenlight.NewContext() + if err != nil { + return nil, fmt.Errorf("could not create xen context: %v", err) + } + defer xenCtx.Close() + + machine.CreationTimestamp = metav1.Now() + + domID, err := xenCtx.DomainCreateNew(cfg) + if err != nil { + return machine, fmt.Errorf("could not create xen domain: %v", err) + } + + machine.Status.PlatformConfig = domID + + machine.Status.State = machinev1alpha1.MachineStateCreated + + return machine, nil +} + +func (service *machineV1alpha1Service) Start(ctx context.Context, machine *machinev1alpha1.Machine) (*machinev1alpha1.Machine, error) { + if machine.Status.PlatformConfig == nil { + return machine, fmt.Errorf("machine has no platform config") + } + + domId, ok := machine.Status.PlatformConfig.(xenlight.Domid) + if !ok { + return machine, fmt.Errorf("machine has no platform config") + } + + xenCtx, err := xenlight.NewContext() + if err != nil { + return nil, fmt.Errorf("could not create xen context: %v", err) + } + defer xenCtx.Close() + + err = xenCtx.DomainUnpause(domId) + if err != nil { + return machine, fmt.Errorf("could not unpause xen domain: %v", err) + } + + machine.Status.State = machinev1alpha1.MachineStateRunning + machine.Status.StartedAt = time.Now() + + return machine, nil +} + +func (service *machineV1alpha1Service) Pause(ctx context.Context, machine *machinev1alpha1.Machine) (*machinev1alpha1.Machine, error) { + if machine.Status.PlatformConfig == nil { + return machine, fmt.Errorf("machine has no platform config") + } + + domId, ok := machine.Status.PlatformConfig.(xenlight.Domid) + if !ok { + return machine, fmt.Errorf("machine has no platform config") + } + + xenCtx, err := xenlight.NewContext() + if err != nil { + return nil, fmt.Errorf("could not create xen context: %v", err) + } + defer xenCtx.Close() + + if err := xenCtx.DomainPause(domId); err != nil { + return machine, fmt.Errorf("could not unpause xen domain: %v", err) + } + + machine.Status.State = machinev1alpha1.MachineStatePaused + + return machine, nil +} + +func (service *machineV1alpha1Service) Stop(ctx context.Context, machine *machinev1alpha1.Machine) (*machinev1alpha1.Machine, error) { + if machine.Status.State == machinev1alpha1.MachineStateExited { + return machine, nil + } + + domId, ok := machine.Status.PlatformConfig.(xenlight.Domid) + if !ok { + return machine, fmt.Errorf("machine has no platform config") + } + + xenCtx, err := xenlight.NewContext() + if err != nil { + return nil, fmt.Errorf("could not create xen context: %v", err) + } + defer xenCtx.Close() + if err := xenCtx.DomainDestroy(domId); err != nil { + return machine, fmt.Errorf("could not destroy xen domain: %v", err) + } + + machine.Status.State = machinev1alpha1.MachineStateExited + machine.Status.ExitedAt = time.Now() + + return machine, nil +} + +func (service *machineV1alpha1Service) Update(ctx context.Context, machine *machinev1alpha1.Machine) (*machinev1alpha1.Machine, error) { + panic("not implemented: kraftkit.sh/machine/xen.machineV1alpha1Service.Update") +} + +func (service *machineV1alpha1Service) Delete(ctx context.Context, machine *machinev1alpha1.Machine) (*machinev1alpha1.Machine, error) { + var errs merr.Errors + + errs = append(errs, os.RemoveAll(machine.Status.StateDir)) + errs = append(errs, os.RemoveAll(machine.Status.LogFile)) + + return nil, errs.Err() +} + +func (service *machineV1alpha1Service) Get(ctx context.Context, machine *machinev1alpha1.Machine) (*machinev1alpha1.Machine, error) { + domId, ok := machine.Status.PlatformConfig.(xenlight.Domid) + if !ok { + return machine, fmt.Errorf("machine has no platform config") + } + + machine.Status.State = machinev1alpha1.MachineStateUnknown + machine.Status.ExitCode = -1 + + xenCtx, err := xenlight.NewContext() + if err != nil { + return nil, fmt.Errorf("could not create xen context: %v", err) + } + defer xenCtx.Close() + + dominfo := &xenlight.Dominfo{} + + // Should be done with xenCtx.DomainInfo, but it currently does not work + doms := xenCtx.ListDomain() + if err != nil { + return machine, fmt.Errorf("could not list xen domains: %v", err) + } + + index := slices.IndexFunc[[]xenlight.Dominfo, xenlight.Dominfo](doms, func(dominfo xenlight.Dominfo) bool { + return dominfo.Domid == domId + }) + + // if index is not present in the list probably it crashed + if index == -1 { + dominfo = &xenlight.Dominfo{Shutdown: true, ShutdownReason: xenlight.ShutdownReasonPoweroff} + } else { + dominfo = &doms[index] + } + + machine.Status.ExitCode, machine.Status.State = getXenState(dominfo) + + if machine.Status.ExitCode != -1 && machine.Status.ExitedAt.IsZero() { + machine.Status.ExitedAt = time.Now() + } + + return machine, nil +} + +func (service *machineV1alpha1Service) List(ctx context.Context, machines *machinev1alpha1.MachineList) (*machinev1alpha1.MachineList, error) { + cached := machines.Items + machines.Items = make([]zip.Object[machinev1alpha1.MachineSpec, machinev1alpha1.MachineStatus], len(cached)) + for i, machine := range cached { + machine, err := service.Get(ctx, &machine) + if err != nil { + machines.Items = cached + return machines, err + } + + machines.Items[i] = *machine + } + + return machines, nil +} + +func (service *machineV1alpha1Service) Watch(ctx context.Context, machine *machinev1alpha1.Machine) (chan *machinev1alpha1.Machine, chan error, error) { + domId, ok := machine.Status.PlatformConfig.(xenlight.Domid) + if !ok { + return nil, nil, fmt.Errorf("machine has no platform config") + } + + w, err := NewWatcher(domId) + if err != nil { + return nil, nil, err + } + + // signals when xenstore tree was updated for domain + watch, err := w.Watch(ctx) + if err != nil { + return nil, nil, err + } + + events := make(chan *machinev1alpha1.Machine) + errs := make(chan error) + + go func() { + intialMachine, err := service.Get(ctx, machine) + if err != nil { + errs <- err + } + events <- intialMachine + + for { + select { + case <-ctx.Done(): + w.Close() + return + case <-watch: + machine, err := service.Get(ctx, machine) + if err != nil { + errs <- err + continue + } + + events <- machine + } + } + }() + + return events, errs, nil +} + +func (service *machineV1alpha1Service) Logs(ctx context.Context, machine *machinev1alpha1.Machine) (chan string, chan error, error) { + domId, ok := machine.Status.PlatformConfig.(xenlight.Domid) + if !ok { + return nil, nil, fmt.Errorf("machine has no platform config") + } + + xenCtx, err := xenlight.NewContext() + if err != nil { + return nil, nil, fmt.Errorf("could not create xen context: %v", err) + } + defer xenCtx.Close() + + pts, err := xenCtx.PrimaryConsoleGetTty(uint32(domId)) + if err != nil { + return nil, nil, fmt.Errorf("could not get xen domain pts: %v", err) + } + + // Start appending pts output to logfile: pts -> chan -> log file + go func() { + ptsChan := make(chan []byte, 10) + errChan := make(chan error) + + ptsFD, err := os.OpenFile(pts, os.O_RDONLY, 0o644) + if err != nil { + log.G(ctx).Errorf("could not open xen domain pts: %v", err) + return + } + + logFD, err := os.OpenFile(machine.Status.LogFile, os.O_APPEND|os.O_WRONLY, 0o644) + if err != nil { + log.G(ctx).Errorf("log file not found after create: %v", err) + return + } + + go func() { + for { + buf := make([]byte, 1024*8) + n, err := ptsFD.Read(buf) + if err != nil { + if err != os.ErrClosed { + errChan <- err + } + + if err == io.EOF { + continue + } + + return + } + ptsChan <- buf[:n] + } + }() + + for { + select { + case err := <-errChan: + log.G(ctx).Errorf("could not read from pts: %v", err) + case line := <-ptsChan: + _, err := logFD.Write(line) + if err != nil { + log.G(ctx).Errorf("could not write to log file: %v", err) + } + case <-ctx.Done(): + logFD.Close() + ptsFD.Close() + return + } + } + }() + + return logtail.NewLogTail(ctx, machine.Status.LogFile) +} + +func getXenState(domInfo *xenlight.Dominfo) (int, machinev1alpha1.MachineState) { + if domInfo.Blocked || domInfo.Running { + return -1, machinev1alpha1.MachineStateRunning + } else if domInfo.Paused { + return -1, machinev1alpha1.MachineStatePaused + } else if domInfo.Dying { + return 0, machinev1alpha1.MachineStateExited + } else if domInfo.Shutdown { + switch domInfo.ShutdownReason { + case xenlight.ShutdownReasonCrash: + return 1, machinev1alpha1.MachineStateErrored + case xenlight.ShutdownReasonPoweroff: + return 0, machinev1alpha1.MachineStateExited + } + } + + return -1, machinev1alpha1.MachineStateUnknown +} + +func NewXenConfig() (*xenlight.DomainConfig, error) { + xcfg, err := xenlight.NewDomainConfig() + if err != nil { + return nil, err + } + + binfoPtr, err := xenlight.NewDomainBuildInfo(xenlight.DomainTypePv) + if err != nil { + return nil, err + } + + xcfg.BInfo = *binfoPtr + + cinfoPtr, err := xenlight.NewDomainCreateInfo() + if err != nil { + return nil, err + } + + xcfg.CInfo = *cinfoPtr + + xcfg.CInfo.Type = xenlight.DomainTypePv + + return xcfg, nil +} diff --git a/machine/xen/xenstore_client.go b/machine/xen/xenstore_client.go new file mode 100644 index 000000000..20a9c823a --- /dev/null +++ b/machine/xen/xenstore_client.go @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright (c) 2024, Unikraft GmbH and The KraftKit Authors. +// Licensed under the BSD-3-Clause License (the "License"). +// You may not use this file except in compliance with the License. + +//go:build xen +// +build xen + +package xen + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "fmt" + "net" + "os" + "strings" + + "kraftkit.sh/log" + "xenbits.xenproject.org/git-http/xen.git/tools/golang/xenlight" +) + +// This client implements a subset of the xenstore protocol as defined in: +// - https://xenbits.xen.org/docs/unstable/misc/xenstore.txt +// - io/xs_wire.h + +type XenstoreOperation uint32 + +const ( + WatchOp XenstoreOperation = 4 + UnwatchOp XenstoreOperation = 5 + WatchEvent XenstoreOperation = 15 + Error XenstoreOperation = 16 + + XenStorePathFmt = "/local/domain/%d" + XenStoredDefaultPath = "/var/run/xenstored" + + NUL = byte('\x00') +) + +// Endpoint for communicating with xenstore +// This client uses only the unix socket interface provided by xenstored + +func XenstoreSocketPath() string { + xenstorepath := XenStoredDefaultPath + if path := os.Getenv("XENSTORED_PATH"); path != "" { + xenstorepath = path + } + + return strings.Join([]string{xenstorepath, "socket"}, "/") +} + +type XsHeader struct { + Op XenstoreOperation + ReqID uint32 + TxID uint32 + Length uint32 +} + +type XsPacket struct { + Header XsHeader + Data []byte +} + +type baseWatcher struct { + closeSignal chan struct{} + + domID uint32 + conn net.Conn + xsPath string + token string +} + +type Watcher interface { + Watch(ctx context.Context) (chan struct{}, error) + Close() +} + +func NewWatcher(domID xenlight.Domid) (Watcher, error) { + conn, err := net.Dial("unix", XenstoreSocketPath()) + if err != nil { + return nil, err + } + + return &baseWatcher{ + domID: uint32(domID), + conn: conn, + xsPath: fmt.Sprintf(XenStorePathFmt, uint32(domID)), + token: "kraftkit" + fmt.Sprintf("%d", uint32(domID)), + closeSignal: make(chan struct{}), + }, nil +} + +// pack serializes the XsPacket into a byte slice for sending over the wire +func (packet *XsPacket) pack() []byte { + data := make([]byte, 0) + data = binary.LittleEndian.AppendUint32(data, uint32(packet.Header.Op)) + data = binary.LittleEndian.AppendUint32(data, packet.Header.ReqID) + data = binary.LittleEndian.AppendUint32(data, packet.Header.TxID) + data = binary.LittleEndian.AppendUint32(data, packet.Header.Length) + + data = append(data, packet.Data...) + + return data +} + +// unpack deserializes a byte slice into an XsPacket +func unpack(data []byte) XsPacket { + header := XsHeader{ + Op: XenstoreOperation(binary.LittleEndian.Uint32(data[0:4])), + ReqID: binary.LittleEndian.Uint32(data[4:8]), + TxID: binary.LittleEndian.Uint32(data[8:12]), + Length: binary.LittleEndian.Uint32(data[12:16]), + } + packet := XsPacket{ + Header: header, + Data: data[16 : 16+header.Length], + } + return packet +} + +func (w *baseWatcher) xsWatchRequest() error { + // Setup a watch at the xenstore path of the domain + path := append([]byte(w.xsPath), NUL) + token := append([]byte(w.token), NUL) + + packet := XsPacket{ + Header: XsHeader{ + Op: WatchOp, + ReqID: 0, + TxID: 0, + Length: uint32(len(path) + len(token)), + }, + Data: append(path, token...), + } + + if _, err := w.conn.Write(packet.pack()); err != nil { + return err + } + + buffer := make([]byte, 4096) + if _, err := w.conn.Read(buffer); err != nil { + return err + } + + packet = unpack(buffer) + if packet.Header.Op == Error { + return fmt.Errorf("could not establish communication with xenstored") + } + + return nil +} + +func (w *baseWatcher) Watch(ctx context.Context) (chan struct{}, error) { + err := w.xsWatchRequest() + if err != nil { + return nil, err + } + + event := make(chan struct{}) + + go func() { + buffer := make([]byte, 4096) + for { + select { + case <-w.closeSignal: + close(w.closeSignal) + return + default: + if _, err := w.conn.Read(buffer); err != nil { + if !errors.Is(err, os.ErrClosed) { + log.G(ctx).Debugf("error reading from xenstore socket while listening for vm status events: %v", err) + } + continue + } + + packet := unpack(buffer) + strs := SplitData(packet) + + if packet.Header.Op != WatchEvent { + continue + } + + if w.token == strs[1] && w.xsPath == strs[0] { + event <- struct{}{} + } + } + } + }() + + return event, nil +} + +func (w *baseWatcher) Close() { + w.closeSignal <- struct{}{} + w.conn.Close() +} + +func SplitData(packet XsPacket) []string { + splitPayload := []string{} + for _, byteSl := range bytes.Split(packet.Data, []byte{NUL}) { + splitPayload = append(splitPayload, string(byteSl)) + } + + return splitPayload +}