Skip to content

Commit

Permalink
Add basic VM coordinator
Browse files Browse the repository at this point in the history
  • Loading branch information
kiootic committed Jun 13, 2022
1 parent 8a5ad22 commit 64c5adf
Show file tree
Hide file tree
Showing 18 changed files with 563 additions and 32 deletions.
37 changes: 37 additions & 0 deletions coordinator/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package main

import (
"encoding/json"
"io/ioutil"

"github.com/oursky/github-ci-support/githublib"
)

type Config struct {
Auth githublib.AuthConfig `json:"auth"`
Target string `json:"target"`
Runners []RunnerConfig `json:"runners"`
VMCtlPath string `json:"vmctlPath"`
}

type RunnerConfig struct {
BaseVMBundlePath string `json:"baseVMBundlePath"`
VMConfigPath string `json:"vmConfigPath"`

RunnerGroup string `json:"runnerGroup,omitempty"`
Labels []string `json:"labels,omitempty"`
}

func NewConfig(path string) (*Config, error) {
data, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
}

var config Config
if err := json.Unmarshal(data, &config); err != nil {
return nil, err
}

return &config, nil
}
13 changes: 13 additions & 0 deletions coordinator/go.mod
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
module github.com/oursky/github-ci-support/coordinator

go 1.18

require go.uber.org/zap v1.21.0

require (
github.com/google/go-github/v45 v45.1.0 // indirect
github.com/google/go-querystring v1.1.0 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
)
54 changes: 54 additions & 0 deletions coordinator/go.sum
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-github/v45 v45.1.0/go.mod h1:FObaZJEDSTa/WGCzZ2Z3eoCDXWJKMenWWTrd8jrta28=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8=
go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
17 changes: 17 additions & 0 deletions coordinator/mac.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package main

import (
"crypto/rand"
"fmt"
)

var buf [6]byte

func generateMACAddress() string {
rand.Read(buf[:])
// unicast
buf[0] &^= (1 << 0)
// local
buf[0] |= (1 << 1)
return fmt.Sprintf("%02x:%02x:%02x:%02x:%02x:%02x", buf[0], buf[01], buf[2], buf[3], buf[4], buf[5])
}
73 changes: 73 additions & 0 deletions coordinator/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
package main

import (
"context"
"flag"
"fmt"
"os"
"os/signal"
"syscall"

"go.uber.org/zap"
"golang.org/x/sync/errgroup"

"github.com/oursky/github-ci-support/githublib"
)

func main() {
var configPath string
flag.StringVar(&configPath, "config", "", "path to config file")

flag.Parse()

if configPath == "" {
panic("config is required")
}

l, _ := zap.NewProduction()
defer l.Sync()
logger := l.Sugar()

config, err := NewConfig(configPath)
if err != nil {
panic(fmt.Sprintf("cannot load config: %s", err))
}

client, err := config.Auth.CreateClient()
if err != nil {
panic(fmt.Sprintf("cannot create client: %s", err))
}

target, err := githublib.NewRunnerTarget(config.Target)
if err != nil {
panic(fmt.Sprintf("cannot load target: %s", err))
}

token := githublib.NewRegistrationTokenStore(target, client)

var runners []*Runner
for i, runnerConfig := range config.Runners {
runner := NewRunner(i, logger, config, runnerConfig, token)
runners = append(runners, runner)
}

ctx, cancel := context.WithCancel(context.Background())
g, ctx := errgroup.WithContext(ctx)
go start(ctx, g, runners)

sig := make(chan os.Signal, 1)
signal.Notify(sig, syscall.SIGTERM, syscall.SIGINT)
go func() {
<-sig
logger.Info("exiting...")
cancel()
}()

g.Wait()
}

func start(ctx context.Context, g *errgroup.Group, runners []*Runner) {
for _, runner := range runners {
runner.Run(ctx, g)
}
}
106 changes: 106 additions & 0 deletions coordinator/runner.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
package main

import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path/filepath"

"github.com/oursky/github-ci-support/githublib"
"go.uber.org/zap"
"golang.org/x/sync/errgroup"
)

type Runner struct {
id int
logger *zap.SugaredLogger
vmctlPath string
config *RunnerConfig
token *githublib.RegistrationTokenStore
}

func NewRunner(id int, logger *zap.SugaredLogger, config *Config, runnerConfig RunnerConfig, token *githublib.RegistrationTokenStore) *Runner {
return &Runner{
id: id,
logger: logger.Named(fmt.Sprintf("runner-%d", id)),
vmctlPath: config.VMCtlPath,
config: &runnerConfig,
token: token,
}
}

func (r *Runner) Run(ctx context.Context, g *errgroup.Group) {
g.Go(func() error {
return r.run(ctx)
})
}

func (r *Runner) run(ctx context.Context) error {
workDir, err := os.MkdirTemp("", fmt.Sprintf("runner-%d-*", r.id))
if err != nil {
return fmt.Errorf("failed to create working directory: %w", err)
}
r.logger.Infow("created working directory", "dir", workDir)

defer func() {
r.logger.Infow("deleting working directory", "dir", workDir)
os.RemoveAll(workDir)
}()

bundlePath := filepath.Join(workDir, "vm.bundle")
configPath := filepath.Join(workDir, "config.json")

ok := true
for ok {
ok, err = r.runVM(ctx, bundlePath, configPath)
if err != nil {
return fmt.Errorf("failed to run VM: %w", err)
}
if ok {
r.logger.Info("VM exited, restarting VM")
}
}

return nil
}

func (r *Runner) runVM(ctx context.Context, bundlePath, configPath string) (bool, error) {
vm := NewVM(r.logger, r.vmctlPath, bundlePath, configPath)

r.logger.Infow("cloning VM", "from", r.config.BaseVMBundlePath, "to", bundlePath)
err := vm.CloneFrom(ctx, r.config.BaseVMBundlePath)
if err != nil {
return false, fmt.Errorf("failed to clone VM: %w", err)
}

configData, err := ioutil.ReadFile(r.config.VMConfigPath)
if err != nil {
return false, fmt.Errorf("failed load VM config: %w", err)
}
var config map[string]interface{}
if err := json.Unmarshal(configData, &config); err != nil {
return false, fmt.Errorf("failed parse VM config: %w", err)
}

macAddr := generateMACAddress()
r.logger.Infow("generated MAC address", "mac", macAddr)
config["macAddress"] = macAddr

configData, err = json.Marshal(config)
if err != nil {
return false, fmt.Errorf("failed serialize VM config: %w", err)
}
if err := ioutil.WriteFile(configPath, configData, 0644); err != nil {
return false, fmt.Errorf("failed save VM config: %w", err)
}

state := RunnerState{VM: vm, MacAddress: macAddr}
err = state.Run(ctx)
if err != nil {
return false, err
}

return true, nil
}
23 changes: 23 additions & 0 deletions coordinator/runner_state.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package main

import "context"

type RunnerState struct {
MacAddress string
VM *VM
}

func (s *RunnerState) Run(ctx context.Context) error {
cmd, err := s.VM.Start(ctx)
if err != nil {
return err
}

completed := make(chan error, 1)
go func() {
completed <- cmd.Wait()
}()

err = <-completed
return err
}
36 changes: 36 additions & 0 deletions coordinator/vm.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package main

import (
"context"
"os/exec"

"go.uber.org/zap"
)

type VM struct {
logger *zap.SugaredLogger
VMCtlPath string
BundlePath string
ConfigPath string
}

func NewVM(logger *zap.SugaredLogger, vmctlPath, bundlePath, configPath string) *VM {
return &VM{
logger: logger.Named("vm"),
VMCtlPath: vmctlPath,
BundlePath: bundlePath,
ConfigPath: configPath,
}
}

func (v *VM) CloneFrom(ctx context.Context, bundlePath string) error {
cmd := exec.CommandContext(ctx, v.VMCtlPath, "clone", bundlePath, v.BundlePath)
v.logger.Debugw("cloning vm", "cmd", cmd.String())
return cmd.Run()
}

func (v *VM) Start(ctx context.Context) (*exec.Cmd, error) {
cmd := exec.CommandContext(ctx, v.VMCtlPath, "start", "--config", v.ConfigPath, "--bundle", v.BundlePath)
v.logger.Debugw("starting vm", "cmd", cmd.String())
return cmd, cmd.Start()
}
Loading

0 comments on commit 64c5adf

Please sign in to comment.